diff options
| author | Sean Brogan <sean.brogan@microsoft.com> | 2019-10-17 21:40:58 -0700 |
|---|---|---|
| committer | Michael D Kinney <michael.d.kinney@intel.com> | 2019-11-11 13:02:03 -0800 |
| commit | 9da7846c880ead64239f68157849e41a521a31fa (patch) | |
| tree | 834a115939d3316a87cdfff162f1617d2fd07f7f | |
| parent | de4ce46d6e5979f734b8f1063b9d3bdc0431bf21 (diff) | |
| download | edk2-9da7846c880ead64239f68157849e41a521a31fa.tar.gz edk2-9da7846c880ead64239f68157849e41a521a31fa.tar.bz2 edk2-9da7846c880ead64239f68157849e41a521a31fa.zip | |
.pytool/Plugin: Add CI plugins
https://bugzilla.tianocore.org/show_bug.cgi?id=2315
Add .pytool directory to the edk2 repository with the
following plugins. These plugins are in a top level
directory because that can be used with all packages
and platforms.
* CharEncodingCheck
* CompilerPlugin
* DependencyCheck
* DscCompleteCheck
* GuidCheck
* LibraryClassCheck
* SpellCheck
Cc: Sean Brogan <sean.brogan@microsoft.com>
Cc: Bret Barkelew <Bret.Barkelew@microsoft.com>
Cc: Liming Gao <liming.gao@intel.com>
Signed-off-by: Michael D Kinney <michael.d.kinney@intel.com>
Reviewed-by: Liming Gao <liming.gao@intel.com>
22 files changed, 1648 insertions, 0 deletions
diff --git a/.pytool/Plugin/CharEncodingCheck/CharEncodingCheck.py b/.pytool/Plugin/CharEncodingCheck/CharEncodingCheck.py new file mode 100644 index 0000000000..02f25ab19f --- /dev/null +++ b/.pytool/Plugin/CharEncodingCheck/CharEncodingCheck.py @@ -0,0 +1,118 @@ +# @file CharEncodingCheck.py
+#
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+
+
+import os
+import logging
+from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin
+from edk2toolext.environment.var_dict import VarDict
+
+##
+# map
+##
+EcodingMap = {
+ ".md": 'utf-8',
+ ".dsc": 'utf-8',
+ ".dec": 'utf-8',
+ ".c": 'utf-8',
+ ".h": 'utf-8',
+ ".asm": 'utf-8',
+ ".masm": 'utf-8',
+ ".nasm": 'utf-8',
+ ".s": 'utf-8',
+ ".inf": 'utf-8',
+ ".asl": 'utf-8',
+ ".uni": 'utf-8',
+ ".py": 'utf-8'
+}
+
+
+class CharEncodingCheck(ICiBuildPlugin):
+ """
+ A CiBuildPlugin that scans each file in the code tree and confirms the encoding is correct.
+
+ Configuration options:
+ "CharEncodingCheck": {
+ "IgnoreFiles": []
+ }
+ """
+
+ def GetTestName(self, packagename: str, environment: VarDict) -> tuple:
+ """ Provide the testcase name and classname for use in reporting
+ testclassname: a descriptive string for the testcase can include whitespace
+ classname: should be patterned <packagename>.<plugin>.<optionally any unique condition>
+
+ Args:
+ packagename: string containing name of package to build
+ environment: The VarDict for the test to run in
+ Returns:
+ a tuple containing the testcase name and the classname
+ (testcasename, classname)
+ """
+ return ("Check for valid file encoding for " + packagename, packagename + ".CharEncodingCheck")
+
+ ##
+ # External function of plugin. This function is used to perform the task of the ci_build_plugin Plugin
+ #
+ # - package is the edk2 path to package. This means workspace/packagepath relative.
+ # - edk2path object configured with workspace and packages path
+ # - PkgConfig Object (dict) for the pkg
+ # - EnvConfig Object
+ # - Plugin Manager Instance
+ # - Plugin Helper Obj Instance
+ # - Junit Logger
+ # - output_stream the StringIO output stream from this plugin via logging
+ def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None):
+ overall_status = 0
+ files_tested = 0
+
+ abs_pkg_path = Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath(packagename)
+
+ if abs_pkg_path is None:
+ tc.SetSkipped()
+ tc.LogStdError("No Package folder {0}".format(abs_pkg_path))
+ return 0
+
+ for (ext, enc) in EcodingMap.items():
+ files = self.WalkDirectoryForExtension([ext], abs_pkg_path)
+ files = [Edk2pathObj.GetEdk2RelativePathFromAbsolutePath(x) for x in files] # make edk2relative path so can process ignores
+
+ if "IgnoreFiles" in pkgconfig:
+ for a in pkgconfig["IgnoreFiles"]:
+ a = a.replace(os.sep, "/")
+ try:
+ tc.LogStdOut("Ignoring File {0}".format(a))
+ files.remove(a)
+ except:
+ tc.LogStdError("CharEncodingCheck.IgnoreInf -> {0} not found in filesystem. Invalid ignore file".format(a))
+ logging.info("CharEncodingCheck.IgnoreInf -> {0} not found in filesystem. Invalid ignore file".format(a))
+
+ files = [Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath(x) for x in files]
+ for a in files:
+ files_tested += 1
+ if(self.TestEncodingOk(a, enc)):
+ logging.debug("File {0} Passed Encoding Check {1}".format(a, enc))
+ else:
+ tc.LogStdError("Encoding Failure in {0}. Not {1}".format(a, enc))
+ overall_status += 1
+
+ tc.LogStdOut("Tested Encoding on {0} files".format(files_tested))
+ if overall_status is not 0:
+ tc.SetFailed("CharEncoding {0} Failed. Errors {1}".format(packagename, overall_status), "CHAR_ENCODING_CHECK_FAILED")
+ else:
+ tc.SetSuccess()
+ return overall_status
+
+ def TestEncodingOk(self, apath, encodingValue):
+ try:
+ with open(apath, "rb") as fobj:
+ fobj.read().decode(encodingValue)
+ except Exception as exp:
+ logging.error("Encoding failure: file: {0} type: {1}".format(apath, encodingValue))
+ logging.debug("EXCEPTION: while processing {1} - {0}".format(exp, apath))
+ return False
+
+ return True
diff --git a/.pytool/Plugin/CharEncodingCheck/CharEncodingCheck_plug_in.yaml b/.pytool/Plugin/CharEncodingCheck/CharEncodingCheck_plug_in.yaml new file mode 100644 index 0000000000..f3ce0b3b3d --- /dev/null +++ b/.pytool/Plugin/CharEncodingCheck/CharEncodingCheck_plug_in.yaml @@ -0,0 +1,11 @@ +## @file
+# CiBuildPlugin used to check char encoding
+#
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+{
+ "scope": "cibuild",
+ "name": "Char Encoding Check Test",
+ "module": "CharEncodingCheck"
+}
diff --git a/.pytool/Plugin/CharEncodingCheck/Readme.md b/.pytool/Plugin/CharEncodingCheck/Readme.md new file mode 100644 index 0000000000..47486e5c3c --- /dev/null +++ b/.pytool/Plugin/CharEncodingCheck/Readme.md @@ -0,0 +1,18 @@ +# Character Encoding Check Plugin
+
+This CiBuildPlugin scans all the files in a package to make sure each file is
+correctly encoded and all characters can be read. Improper encoding causes
+tools to fail in some situations especially in different locals.
+
+## Configuration
+
+The plugin can be configured to ignore certain files.
+
+``` yaml
+"CharEncodingCheck": {
+ "IgnoreFiles": []
+}
+```
+### IgnoreFiles
+
+OPTIONAL List of file to ignore.
diff --git a/.pytool/Plugin/CompilerPlugin/CompilerPlugin.py b/.pytool/Plugin/CompilerPlugin/CompilerPlugin.py new file mode 100644 index 0000000000..3b6f7c7698 --- /dev/null +++ b/.pytool/Plugin/CompilerPlugin/CompilerPlugin.py @@ -0,0 +1,102 @@ +# @file HostUnitTestCompiler_plugin.py
+##
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+
+import logging
+import os
+import re
+from edk2toollib.uefi.edk2.parsers.dsc_parser import DscParser
+from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin
+from edk2toolext.environment.uefi_build import UefiBuilder
+from edk2toolext import edk2_logging
+from edk2toolext.environment.var_dict import VarDict
+
+
+class CompilerPlugin(ICiBuildPlugin):
+ """
+ A CiBuildPlugin that compiles the package dsc
+ from the package being tested.
+
+ Configuration options:
+ "CompilerPlugin": {
+ "DscPath": "<path to dsc from root of pkg>"
+ }
+ """
+
+ def GetTestName(self, packagename: str, environment: VarDict) -> tuple:
+ """ Provide the testcase name and classname for use in reporting
+
+ Args:
+ packagename: string containing name of package to build
+ environment: The VarDict for the test to run in
+ Returns:
+ a tuple containing the testcase name and the classname
+ (testcasename, classname)
+ """
+ target = environment.GetValue("TARGET")
+ return ("Compile " + packagename + " " + target, packagename + ".Compiler." + target)
+
+ def RunsOnTargetList(self):
+ return ["DEBUG", "RELEASE"]
+
+ ##
+ # External function of plugin. This function is used to perform the task of the MuBuild Plugin
+ #
+ # - package is the edk2 path to package. This means workspace/packagepath relative.
+ # - edk2path object configured with workspace and packages path
+ # - PkgConfig Object (dict) for the pkg
+ # - EnvConfig Object
+ # - Plugin Manager Instance
+ # - Plugin Helper Obj Instance
+ # - Junit Logger
+ # - output_stream the StringIO output stream from this plugin via logging
+ def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None):
+ self._env = environment
+
+ # Parse the config for required DscPath element
+ if "DscPath" not in pkgconfig:
+ tc.SetSkipped()
+ tc.LogStdError("DscPath not found in config file. Nothing to compile.")
+ return -1
+
+ AP = Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath(packagename)
+
+ APDSC = os.path.join(AP, pkgconfig["DscPath"].strip())
+ AP_Path = Edk2pathObj.GetEdk2RelativePathFromAbsolutePath(APDSC)
+ if AP is None or AP_Path is None or not os.path.isfile(APDSC):
+ tc.SetSkipped()
+ tc.LogStdError("Package Dsc not found.")
+ return -1
+
+ logging.info("Building {0}".format(AP_Path))
+ self._env.SetValue("ACTIVE_PLATFORM", AP_Path, "Set in Compiler Plugin")
+
+ # Parse DSC to check for SUPPORTED_ARCHITECTURES
+ dp = DscParser()
+ dp.SetBaseAbsPath(Edk2pathObj.WorkspacePath)
+ dp.SetPackagePaths(Edk2pathObj.PackagePathList)
+ dp.ParseFile(AP_Path)
+ if "SUPPORTED_ARCHITECTURES" in dp.LocalVars:
+ SUPPORTED_ARCHITECTURES = dp.LocalVars["SUPPORTED_ARCHITECTURES"].split('|')
+ TARGET_ARCHITECTURES = environment.GetValue("TARGET_ARCH").split(' ')
+
+ # Skip if there is no intersection between SUPPORTED_ARCHITECTURES and TARGET_ARCHITECTURES
+ if len(set(SUPPORTED_ARCHITECTURES) & set(TARGET_ARCHITECTURES)) == 0:
+ tc.SetSkipped()
+ tc.LogStdError("No supported architecutres to build")
+ return -1
+
+ uefiBuilder = UefiBuilder()
+ # do all the steps
+ # WorkSpace, PackagesPath, PInHelper, PInManager
+ ret = uefiBuilder.Go(Edk2pathObj.WorkspacePath, os.pathsep.join(Edk2pathObj.PackagePathList), PLMHelper, PLM)
+ if ret != 0: # failure:
+ tc.SetFailed("Compile failed for {0}".format(packagename), "Compile_FAILED")
+ tc.LogStdError("{0} Compile failed with error code {1} ".format(AP_Path, ret))
+ return 1
+
+ else:
+ tc.SetSuccess()
+ return 0
diff --git a/.pytool/Plugin/CompilerPlugin/Compiler_plug_in.yaml b/.pytool/Plugin/CompilerPlugin/Compiler_plug_in.yaml new file mode 100644 index 0000000000..4d7ca326c7 --- /dev/null +++ b/.pytool/Plugin/CompilerPlugin/Compiler_plug_in.yaml @@ -0,0 +1,11 @@ +## @file
+# CiBuildPlugin used to compile each package
+#
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+{
+ "scope": "cibuild",
+ "name": "Compiler Plugin",
+ "module": "CompilerPlugin"
+}
diff --git a/.pytool/Plugin/CompilerPlugin/Readme.md b/.pytool/Plugin/CompilerPlugin/Readme.md new file mode 100644 index 0000000000..44e6a2c8d9 --- /dev/null +++ b/.pytool/Plugin/CompilerPlugin/Readme.md @@ -0,0 +1,17 @@ +# Compiler Plugin
+
+This CiBuildPlugin compiles the package DSC from the package being tested.
+
+## Configuration
+
+The package relative path of the DSC file to build.
+
+``` yaml
+"CompilerPlugin": {
+ "DscPath": "<path to dsc from root of pkg>"
+}
+```
+
+### DscPath
+
+Package relative path to the DSC file to build.
diff --git a/.pytool/Plugin/DependencyCheck/DependencyCheck.py b/.pytool/Plugin/DependencyCheck/DependencyCheck.py new file mode 100644 index 0000000000..2c3d8baf69 --- /dev/null +++ b/.pytool/Plugin/DependencyCheck/DependencyCheck.py @@ -0,0 +1,120 @@ +# @file dependency_check.py
+#
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+
+import logging
+import os
+from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin
+from edk2toollib.uefi.edk2.parsers.inf_parser import InfParser
+from edk2toolext.environment.var_dict import VarDict
+
+
+class DependencyCheck(ICiBuildPlugin):
+ """
+ A CiBuildPlugin that finds all modules (inf files) in a package and reviews the packages used
+ to confirm they are acceptable. This is to help enforce layering and identify improper
+ dependencies between packages.
+
+ Configuration options:
+ "DependencyCheck": {
+ "AcceptableDependencies": [], # Package dec files that are allowed in all INFs. Example: MdePkg/MdePkg.dec
+ "AcceptableDependencies-<MODULE_TYPE>": [], # OPTIONAL Package dependencies for INFs that are HOST_APPLICATION
+ "AcceptableDependencies-HOST_APPLICATION": [], # EXAMPLE Package dependencies for INFs that are HOST_APPLICATION
+ "IgnoreInf": [] # Ignore INF if found in filesystem
+ }
+ """
+
+ def GetTestName(self, packagename: str, environment: VarDict) -> tuple:
+ """ Provide the testcase name and classname for use in reporting
+
+ Args:
+ packagename: string containing name of package to build
+ environment: The VarDict for the test to run in
+ Returns:
+ a tuple containing the testcase name and the classname
+ (testcasename, classname)
+ testclassname: a descriptive string for the testcase can include whitespace
+ classname: should be patterned <packagename>.<plugin>.<optionally any unique condition>
+ """
+ return ("Test Package Dependencies for modules in " + packagename, packagename + ".DependencyCheck")
+
+ ##
+ # External function of plugin. This function is used to perform the task of the MuBuild Plugin
+ #
+ # - package is the edk2 path to package. This means workspace/packagepath relative.
+ # - edk2path object configured with workspace and packages path
+ # - PkgConfig Object (dict) for the pkg
+ # - EnvConfig Object
+ # - Plugin Manager Instance
+ # - Plugin Helper Obj Instance
+ # - Junit Logger
+ # - output_stream the StringIO output stream from this plugin via logging
+ def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None):
+ overall_status = 0
+
+ # Get current platform
+ abs_pkg_path = Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath(packagename)
+
+ # Get INF Files
+ INFFiles = self.WalkDirectoryForExtension([".inf"], abs_pkg_path)
+ INFFiles = [Edk2pathObj.GetEdk2RelativePathFromAbsolutePath(x) for x in INFFiles] # make edk2relative path so can compare with Ignore List
+
+ # Remove ignored INFs
+ if "IgnoreInf" in pkgconfig:
+ for a in pkgconfig["IgnoreInf"]:
+ a = a.replace(os.sep, "/") ## convert path sep in case ignore list is bad. Can't change case
+ try:
+ INFFiles.remove(a)
+ tc.LogStdOut("IgnoreInf {0}".format(a))
+ except:
+ logging.info("DependencyConfig.IgnoreInf -> {0} not found in filesystem. Invalid ignore file".format(a))
+ tc.LogStdError("DependencyConfig.IgnoreInf -> {0} not found in filesystem. Invalid ignore file".format(a))
+
+
+ # Get the AccpetableDependencies list
+ if "AcceptableDependencies" not in pkgconfig:
+ logging.info("DependencyCheck Skipped. No Acceptable Dependencies defined.")
+ tc.LogStdOut("DependencyCheck Skipped. No Acceptable Dependencies defined.")
+ tc.SetSkipped()
+ return -1
+
+ # Log dependencies
+ for k in pkgconfig.keys():
+ if k.startswith("AcceptableDependencies"):
+ pkgstring = "\n".join(pkgconfig[k])
+ if ("-" in k):
+ _, _, mod_type = k.partition("-")
+ tc.LogStdOut(f"Additional dependencies for MODULE_TYPE {mod_type}:\n {pkgstring}")
+ else:
+ tc.LogStdOut(f"Acceptable Dependencies:\n {pkgstring}")
+
+ # For each INF file
+ for file in INFFiles:
+ ip = InfParser()
+ logging.debug("Parsing " + file)
+ ip.SetBaseAbsPath(Edk2pathObj.WorkspacePath).SetPackagePaths(Edk2pathObj.PackagePathList).ParseFile(file)
+
+ if("MODULE_TYPE" not in ip.Dict):
+ tc.LogStdOut("Ignoring INF. Missing key for MODULE_TYPE {0}".format(file))
+ continue
+
+ mod_type = ip.Dict["MODULE_TYPE"].upper()
+ for p in ip.PackagesUsed:
+ if p not in pkgconfig["AcceptableDependencies"]:
+ # If not in the main acceptable dependencies list then check module specific
+ mod_specific_key = "AcceptableDependencies-" + mod_type
+ if mod_specific_key in pkgconfig and p in pkgconfig[mod_specific_key]:
+ continue
+
+ logging.error("Dependency Check: Invalid Dependency INF: {0} depends on pkg {1}".format(file, p))
+ tc.LogStdError("Dependency Check: Invalid Dependency INF: {0} depends on pkg {1}".format(file, p))
+ overall_status += 1
+
+ # If XML object exists, add results
+ if overall_status is not 0:
+ tc.SetFailed("Failed with {0} errors".format(overall_status), "DEPENDENCYCHECK_FAILED")
+ else:
+ tc.SetSuccess()
+ return overall_status
diff --git a/.pytool/Plugin/DependencyCheck/DependencyCheck_plug_in.yaml b/.pytool/Plugin/DependencyCheck/DependencyCheck_plug_in.yaml new file mode 100644 index 0000000000..c983469716 --- /dev/null +++ b/.pytool/Plugin/DependencyCheck/DependencyCheck_plug_in.yaml @@ -0,0 +1,13 @@ +## @file
+# CiBuildPlugin used to check all infs within a package
+# to confirm the packagesdependency are on the configured list of acceptable
+# dependencies.
+#
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+{
+ "scope": "cibuild",
+ "name": "Dependency Check Test",
+ "module": "DependencyCheck"
+}
diff --git a/.pytool/Plugin/DependencyCheck/Readme.md b/.pytool/Plugin/DependencyCheck/Readme.md new file mode 100644 index 0000000000..62612f6965 --- /dev/null +++ b/.pytool/Plugin/DependencyCheck/Readme.md @@ -0,0 +1,31 @@ +# Depdendency Check Plugin
+
+A CiBuildPlugin that finds all modules (inf files) in a package and reviews the
+packages used to confirm they are acceptable. This is to help enforce layering
+and identify improper dependencies between packages.
+
+## Configuration
+
+The plugin must be configured with the acceptabe package dependencies for the
+package.
+
+``` yaml
+"DependencyCheck": {
+ "AcceptableDependencies": [],
+ "AcceptableDependencies-<MODULE_TYPE>": [],
+ "IgnoreInf": []
+}
+```
+
+### AcceptableDependencies
+
+Package dec files that are allowed in all INFs. Example: MdePkg/MdePkg.dec
+
+### AcceptableDependencies-<MODULE_TYPE>
+
+OPTIONAL Package dependencies for INFs that have module type <MODULE_TYPE>.
+Example: AcceptableDependencies-HOST_APPLICATION.
+
+### IgnoreInf
+
+OPTIONAL list of INFs to ignore for this dependency check.
diff --git a/.pytool/Plugin/DscCompleteCheck/DscCompleteCheck.py b/.pytool/Plugin/DscCompleteCheck/DscCompleteCheck.py new file mode 100644 index 0000000000..e2485f5708 --- /dev/null +++ b/.pytool/Plugin/DscCompleteCheck/DscCompleteCheck.py @@ -0,0 +1,118 @@ +# @file DscCompleteCheck.py
+#
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+import logging
+import os
+from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin
+from edk2toollib.uefi.edk2.parsers.dsc_parser import DscParser
+from edk2toollib.uefi.edk2.parsers.inf_parser import InfParser
+from edk2toolext.environment.var_dict import VarDict
+
+
+class DscCompleteCheck(ICiBuildPlugin):
+ """
+ A CiBuildPlugin that scans the package dsc file and confirms all modules (inf files) are
+ listed in the components sections.
+
+ Configuration options:
+ "DscCompleteCheck": {
+ "DscPath": "<path to dsc from root of pkg>"
+ "IgnoreInf": [] # Ignore INF if found in filesystem by not dsc
+ }
+ """
+
+ def GetTestName(self, packagename: str, environment: VarDict) -> tuple:
+ """ Provide the testcase name and classname for use in reporting
+
+ Args:
+ packagename: string containing name of package to build
+ environment: The VarDict for the test to run in
+ Returns:
+ a tuple containing the testcase name and the classname
+ (testcasename, classname)
+ testclassname: a descriptive string for the testcase can include whitespace
+ classname: should be patterned <packagename>.<plugin>.<optionally any unique condition>
+ """
+ return ("Check the " + packagename + " DSC for a being complete", packagename + ".DscCompleteCheck")
+
+ ##
+ # External function of plugin. This function is used to perform the task of the MuBuild Plugin
+ #
+ # - package is the edk2 path to package. This means workspace/packagepath relative.
+ # - edk2path object configured with workspace and packages path
+ # - PkgConfig Object (dict) for the pkg
+ # - VarDict containing the shell environment Build Vars
+ # - Plugin Manager Instance
+ # - Plugin Helper Obj Instance
+ # - Junit Logger
+ # - output_stream the StringIO output stream from this plugin via logging
+ def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None):
+ overall_status = 0
+
+ # Parse the config for required DscPath element
+ if "DscPath" not in pkgconfig:
+ tc.SetSkipped()
+ tc.LogStdError("DscPath not found in config file. Nothing to check.")
+ return -1
+
+ abs_pkg_path = Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath(packagename)
+ abs_dsc_path = os.path.join(abs_pkg_path, pkgconfig["DscPath"].strip())
+ wsr_dsc_path = Edk2pathObj.GetEdk2RelativePathFromAbsolutePath(abs_dsc_path)
+
+ if abs_dsc_path is None or wsr_dsc_path is "" or not os.path.isfile(abs_dsc_path):
+ tc.SetSkipped()
+ tc.LogStdError("Package Dsc not found")
+ return 0
+
+ # Get INF Files
+ INFFiles = self.WalkDirectoryForExtension([".inf"], abs_pkg_path)
+ INFFiles = [Edk2pathObj.GetEdk2RelativePathFromAbsolutePath(x) for x in INFFiles] # make edk2relative path so can compare with DSC
+
+ # remove ignores
+
+ if "IgnoreInf" in pkgconfig:
+ for a in pkgconfig["IgnoreInf"]:
+ a = a.replace(os.sep, "/")
+ try:
+ tc.LogStdOut("Ignoring INF {0}".format(a))
+ INFFiles.remove(a)
+ except:
+ tc.LogStdError("DscCompleteCheck.IgnoreInf -> {0} not found in filesystem. Invalid ignore file".format(a))
+ logging.info("DscCompleteCheck.IgnoreInf -> {0} not found in filesystem. Invalid ignore file".format(a))
+
+ # DSC Parser
+ dp = DscParser()
+ dp.SetBaseAbsPath(Edk2pathObj.WorkspacePath)
+ dp.SetPackagePaths(Edk2pathObj.PackagePathList)
+ dp.SetInputVars(environment.GetAllBuildKeyValues())
+ dp.ParseFile(wsr_dsc_path)
+
+ # Check if INF in component section
+ for INF in INFFiles:
+ if not any(INF.strip() in x for x in dp.ThreeMods) and \
+ not any(INF.strip() in x for x in dp.SixMods) and \
+ not any(INF.strip() in x for x in dp.OtherMods):
+
+ infp = InfParser().SetBaseAbsPath(Edk2pathObj.WorkspacePath)
+ infp.SetPackagePaths(Edk2pathObj.PackagePathList)
+ infp.ParseFile(INF)
+ if("MODULE_TYPE" not in infp.Dict):
+ tc.LogStdOut("Ignoring INF. Missing key for MODULE_TYPE {0}".format(INF))
+ continue
+
+ if(infp.Dict["MODULE_TYPE"] == "HOST_APPLICATION"):
+ tc.LogStdOut("Ignoring INF. Module type is HOST_APPLICATION {0}".format(INF))
+ continue
+
+ logging.critical(INF + " not in " + wsr_dsc_path)
+ tc.LogStdError("{0} not in {1}".format(INF, wsr_dsc_path))
+ overall_status = overall_status + 1
+
+ # If XML object exists, add result
+ if overall_status is not 0:
+ tc.SetFailed("DscCompleteCheck {0} Failed. Errors {1}".format(wsr_dsc_path, overall_status), "CHECK_FAILED")
+ else:
+ tc.SetSuccess()
+ return overall_status
diff --git a/.pytool/Plugin/DscCompleteCheck/DscCompleteCheck_plug_in.yaml b/.pytool/Plugin/DscCompleteCheck/DscCompleteCheck_plug_in.yaml new file mode 100644 index 0000000000..215b8b76a9 --- /dev/null +++ b/.pytool/Plugin/DscCompleteCheck/DscCompleteCheck_plug_in.yaml @@ -0,0 +1,12 @@ +## @file
+# CiBuildPlugin used to confirm all INFs are listed in
+# the components section of package dsc
+#
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+{
+ "scope": "cibuild",
+ "name": "Dsc Complete Check Test",
+ "module": "DscCompleteCheck"
+}
diff --git a/.pytool/Plugin/DscCompleteCheck/Readme.md b/.pytool/Plugin/DscCompleteCheck/Readme.md new file mode 100644 index 0000000000..eefbb9894d --- /dev/null +++ b/.pytool/Plugin/DscCompleteCheck/Readme.md @@ -0,0 +1,27 @@ +# Dsc Complete Check Plugin
+
+This CiBuildPlugin scans all INF files from a package and confirms they are
+listed in the package level DSC file. The test considers it an error if any INF
+does not appear in the `Components` section of the package-level DSC (indicating
+that it would not be built if the package were built). This is critical because
+much of the CI infrastructure assumes that all modules will be listed in the DSC
+and compiled.
+
+## Configuration
+
+The plugin has a few configuration options to support the UEFI codebase.
+
+``` yaml
+"DscCompleteCheck": {
+ "DscPath": "", # Path to dsc from root of package
+ "IgnoreInf": [] # Ignore INF if found in filesystem by not dsc
+ }
+```
+
+### DscPath
+
+Path to DSC to consider platform dsc
+
+### IgnoreInf
+
+Ignore error if Inf file is not listed in DSC file
diff --git a/.pytool/Plugin/GuidCheck/GuidCheck.py b/.pytool/Plugin/GuidCheck/GuidCheck.py new file mode 100644 index 0000000000..f0b10beb1e --- /dev/null +++ b/.pytool/Plugin/GuidCheck/GuidCheck.py @@ -0,0 +1,251 @@ +# @file GuidCheck.py
+#
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+import logging
+from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin
+from edk2toollib.uefi.edk2.guid_list import GuidList
+from edk2toolext.environment.var_dict import VarDict
+
+
+class GuidCheck(ICiBuildPlugin):
+ """
+ A CiBuildPlugin that scans the code tree and looks for duplicate guids
+ from the package being tested.
+
+ Configuration options:
+ "GuidCheck": {
+ "IgnoreGuidName": [], # provide in format guidname=guidvalue or just guidname
+ "IgnoreGuidValue": [],
+ "IgnoreFoldersAndFiles": [],
+ "IgnoreDuplicates": [] # Provide in format guidname=guidname=guidname...
+ }
+ """
+
+ def GetTestName(self, packagename: str, environment: VarDict) -> tuple:
+ """ Provide the testcase name and classname for use in reporting
+
+ Args:
+ packagename: string containing name of package to build
+ environment: The VarDict for the test to run in
+ Returns:
+ a tuple containing the testcase name and the classname
+ (testcasename, classname)
+ testclassname: a descriptive string for the testcase can include whitespace
+ classname: should be patterned <packagename>.<plugin>.<optionally any unique condition>
+ """
+ return ("Confirm GUIDs are unique in " + packagename, packagename + ".GuidCheck")
+
+ def _FindConflictingGuidValues(self, guidlist: list) -> list:
+ """ Find all duplicate guids by guid value and report them as errors
+ """
+ # Sort the list by guid
+ guidsorted = sorted(
+ guidlist, key=lambda x: x.guid.upper(), reverse=True)
+
+ previous = None # Store previous entry for comparison
+ error = None
+ errors = []
+ for index in range(len(guidsorted)):
+ i = guidsorted[index]
+ if(previous is not None):
+ if i.guid == previous.guid: # Error
+ if(error is None):
+ # Catch errors with more than 1 conflict
+ error = ErrorEntry("guid")
+ error.entries.append(previous)
+ errors.append(error)
+ error.entries.append(i)
+ else:
+ # no match. clear error
+ error = None
+ previous = i
+ return errors
+
+ def _FindConflictingGuidNames(self, guidlist: list) -> list:
+ """ Find all duplicate guids by name and if they are not all
+ from inf files report them as errors. It is ok to have
+ BASE_NAME duplication.
+
+ Is this useful? It would catch two same named guids in dec file
+ that resolve to different values.
+ """
+ # Sort the list by guid
+ namesorted = sorted(guidlist, key=lambda x: x.name.upper())
+
+ previous = None # Store previous entry for comparison
+ error = None
+ errors = []
+ for index in range(len(namesorted)):
+ i = namesorted[index]
+ if(previous is not None):
+ # If name matches
+ if i.name == previous.name:
+ if(error is None):
+ # Catch errors with more than 1 conflict
+ error = ErrorEntry("name")
+ error.entries.append(previous)
+ errors.append(error)
+ error.entries.append(i)
+ else:
+ # no match. clear error
+ error = None
+ previous = i
+
+ # Loop thru and remove any errors where all files are infs as it is ok if
+ # they have the same inf base name.
+ for e in errors[:]:
+ if len( [en for en in e.entries if not en.absfilepath.lower().endswith(".inf")]) == 0:
+ errors.remove(e)
+
+ return errors
+
+ ##
+ # External function of plugin. This function is used to perform the task of the MuBuild Plugin
+ #
+ # - package is the edk2 path to package. This means workspace/packagepath relative.
+ # - edk2path object configured with workspace and packages path
+ # - PkgConfig Object (dict) for the pkg
+ # - EnvConfig Object
+ # - Plugin Manager Instance
+ # - Plugin Helper Obj Instance
+ # - Junit Logger
+ # - output_stream the StringIO output stream from this plugin via logging
+
+ def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None):
+ Errors = []
+
+ abs_pkg_path = Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath(
+ packagename)
+
+ if abs_pkg_path is None:
+ tc.SetSkipped()
+ tc.LogStdError("No package {0}".format(packagename))
+ return -1
+
+ All_Ignores = ["/Build", "/Conf"]
+ # Parse the config for other ignores
+ if "IgnoreFoldersAndFiles" in pkgconfig:
+ All_Ignores.extend(pkgconfig["IgnoreFoldersAndFiles"])
+
+ # Parse the workspace for all GUIDs
+ gs = GuidList.guidlist_from_filesystem(
+ Edk2pathObj.WorkspacePath, ignore_lines=All_Ignores)
+
+ # Remove ignored guidvalue
+ if "IgnoreGuidValue" in pkgconfig:
+ for a in pkgconfig["IgnoreGuidValue"]:
+ try:
+ tc.LogStdOut("Ignoring Guid {0}".format(a.upper()))
+ for b in gs[:]:
+ if b.guid == a.upper():
+ gs.remove(b)
+ except:
+ tc.LogStdError("GuidCheck.IgnoreGuid -> {0} not found. Invalid ignore guid".format(a.upper()))
+ logging.info("GuidCheck.IgnoreGuid -> {0} not found. Invalid ignore guid".format(a.upper()))
+
+ # Remove ignored guidname
+ if "IgnoreGuidName" in pkgconfig:
+ for a in pkgconfig["IgnoreGuidName"]:
+ entry = a.split("=")
+ if(len(entry) > 2):
+ tc.LogStdError("GuidCheck.IgnoreGuidName -> {0} Invalid Format.".format(a))
+ logging.info("GuidCheck.IgnoreGuidName -> {0} Invalid Format.".format(a))
+ continue
+ try:
+ tc.LogStdOut("Ignoring Guid {0}".format(a))
+ for b in gs[:]:
+ if b.name == entry[0]:
+ if(len(entry) == 1):
+ gs.remove(b)
+ elif(len(entry) == 2 and b.guid.upper() == entry[1].upper()):
+ gs.remove(b)
+ else:
+ c.LogStdError("GuidCheck.IgnoreGuidName -> {0} incomplete match. Invalid ignore guid".format(a))
+
+ except:
+ tc.LogStdError("GuidCheck.IgnoreGuidName -> {0} not found. Invalid ignore name".format(a))
+ logging.info("GuidCheck.IgnoreGuidName -> {0} not found. Invalid ignore name".format(a))
+
+ # Find conflicting Guid Values
+ Errors.extend(self._FindConflictingGuidValues(gs))
+
+ # Check if there are expected duplicates and remove it from the error list
+ if "IgnoreDuplicates" in pkgconfig:
+ for a in pkgconfig["IgnoreDuplicates"]:
+ names = a.split("=")
+ if len(names) < 2:
+ tc.LogStdError("GuidCheck.IgnoreDuplicates -> {0} invalid format".format(a))
+ logging.info("GuidCheck.IgnoreDuplicates -> {0} invalid format".format(a))
+ continue
+
+ for b in Errors[:]:
+ if b.type != "guid":
+ continue
+ ## Make a list of the names that are not in the names list. If there
+ ## are any in the list then this error should not be ignored.
+ t = [x for x in b.entries if x.name not in names]
+ if(len(t) == len(b.entries)):
+ ## did not apply to any entry
+ continue
+ elif(len(t) == 0):
+ ## full match - ignore duplicate
+ tc.LogStdOut("GuidCheck.IgnoreDuplicates -> {0}".format(a))
+ Errors.remove(b)
+ elif(len(t) < len(b.entries)):
+ ## partial match
+ tc.LogStdOut("GuidCheck.IgnoreDuplicates -> {0} incomplete match".format(a))
+ logging.info("GuidCheck.IgnoreDuplicates -> {0} incomplete match".format(a))
+ else:
+ tc.LogStdOut("GuidCheck.IgnoreDuplicates -> {0} unknown error.".format(a))
+ logging.info("GuidCheck.IgnoreDuplicates -> {0} unknown error".format(a))
+
+
+
+ # Find conflicting Guid Names
+ Errors.extend(self._FindConflictingGuidNames(gs))
+
+ # Log errors for anything within the package under test
+ for er in Errors[:]:
+ InMyPackage = False
+ for a in er.entries:
+ if abs_pkg_path in a.absfilepath:
+ InMyPackage = True
+ break
+ if(not InMyPackage):
+ Errors.remove(er)
+ else:
+ logging.error(str(er))
+ tc.LogStdError(str(er))
+
+ # add result to test case
+ overall_status = len(Errors)
+ if overall_status is not 0:
+ tc.SetFailed("GuidCheck {0} Failed. Errors {1}".format(
+ packagename, overall_status), "CHECK_FAILED")
+ else:
+ tc.SetSuccess()
+ return overall_status
+
+
+class ErrorEntry():
+ """ Custom/private class for reporting errors in the GuidList
+ """
+
+ def __init__(self, errortype):
+ self.type = errortype # 'guid' or 'name' depending on error type
+ self.entries = [] # GuidListEntry that are in error condition
+
+ def __str__(self):
+ a = f"Error Duplicate {self.type}: "
+ if(self.type == "guid"):
+ a += f" {self.entries[0].guid}"
+ elif(self.type == "name"):
+ a += f" {self.entries[0].name}"
+
+ a += f" ({len(self.entries)})\n"
+
+ for e in self.entries:
+ a += "\t" + str(e) + "\n"
+ return a
diff --git a/.pytool/Plugin/GuidCheck/GuidCheck_plug_in.yaml b/.pytool/Plugin/GuidCheck/GuidCheck_plug_in.yaml new file mode 100644 index 0000000000..b16f6c16b6 --- /dev/null +++ b/.pytool/Plugin/GuidCheck/GuidCheck_plug_in.yaml @@ -0,0 +1,11 @@ +## @file
+# CiBuildPlugin used to check guid uniqueness
+#
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+{
+ "scope": "cibuild",
+ "name": "Guid Check Test",
+ "module": "GuidCheck"
+}
diff --git a/.pytool/Plugin/GuidCheck/Readme.md b/.pytool/Plugin/GuidCheck/Readme.md new file mode 100644 index 0000000000..4e2c353cc4 --- /dev/null +++ b/.pytool/Plugin/GuidCheck/Readme.md @@ -0,0 +1,80 @@ +# Guid Check Plugin
+
+This CiBuildPlugin scans all the files in a code tree to find all the GUID
+definitions. After collection it will then look for duplication in the package
+under test. Uniqueness of all GUIDs are critical within the UEFI environment.
+Duplication can cause numerous issues including locating the wrong data
+structure, calling the wrong function, or decoding the wrong data members.
+
+Currently Scanned:
+
+* INF files are scanned for there Module guid
+* DEC files are scanned for all of their Protocols, PPIs, and Guids as well as
+ the one package GUID.
+
+Any GUID value being equal to two names or even just defined in two files is
+considered an error unless in the ignore list.
+
+Any GUID name that is found more than once is an error unless all occurrences
+are Module GUIDs. Since the Module GUID is assigned to the Module name it is
+common to have numerous versions of the same module named the same.
+
+## Configuration
+
+The plugin has numerous configuration options to support the UEFI codebase.
+
+``` yaml
+"GuidCheck": {
+ "IgnoreGuidName": [],
+ "IgnoreGuidValue": [],
+ "IgnoreFoldersAndFiles": [],
+ "IgnoreDuplicates": []
+ }
+```
+
+### IgnoreGuidName
+
+This list allows strings in two formats.
+
+* _GuidName_
+ * This will remove any entry with this GuidName from the list of GUIDs
+ therefore ignoring any error associated with this name.
+* _GuidName=GuidValue_
+ * This will also ignore the GUID by name but only if the value equals the
+ GuidValue.
+ * GuidValue should be in registry format.
+ * This is the suggested format to use as it will limit the ignore to only the
+ defined case.
+
+### IgnoreGuidValue
+
+This list allows strings in guid registry format _GuidValue_.
+
+* This will remove any entry with this GuidValue from the list of GUIDs
+ therefore ignoring any error associated with this value.
+* GuidValue must be in registry format xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+
+### IgnoreFoldersAndFiles
+
+This supports .gitignore file and folder matching strings including wildcards
+
+* Any folder or file ignored will not be parsed and therefore any GUID defined
+ will be ignored.
+* The plugin will always ignores the following ["/Build", "/Conf"]
+
+### IgnoreDuplicates
+
+This supports strings in the format of _GuidName_=_GuidName_=_GuidName_
+
+* For the error with the GuidNames to be ignored the list must match completely
+ with what is found during the code scan.
+ * For example if there are two GUIDs that are by design equal within the code
+ tree then it should be _GuidName_=_GuidName_
+ * If instead there are three GUIDs then it must be
+ _GuidName_=_GuidName_=_GuidName_
+* This is the best ignore list to use because it is the most strict and will
+ catch new problems when new conflicts are introduced.
+* There are numerous places in the UEFI specification in which two GUID names
+ are assigned the same value. These names should be set in this ignore list so
+ that they don't cause an error but any additional duplication would still be
+ caught.
diff --git a/.pytool/Plugin/LibraryClassCheck/LibraryClassCheck.py b/.pytool/Plugin/LibraryClassCheck/LibraryClassCheck.py new file mode 100644 index 0000000000..5dafcbc13f --- /dev/null +++ b/.pytool/Plugin/LibraryClassCheck/LibraryClassCheck.py @@ -0,0 +1,153 @@ +# @file LibraryClassCheck.py
+#
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+import logging
+import os
+from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin
+from edk2toollib.uefi.edk2.parsers.dec_parser import DecParser
+from edk2toollib.uefi.edk2.parsers.inf_parser import InfParser
+from edk2toolext.environment.var_dict import VarDict
+
+
+class LibraryClassCheck(ICiBuildPlugin):
+ """
+ A CiBuildPlugin that scans the code tree and library classes for undeclared
+ files
+
+ Configuration options:
+ "LibraryClassCheck": {
+ IgnoreHeaderFile: [], # Ignore a file found on disk
+ IgnoreLibraryClass: [] # Ignore a declaration found in dec file
+ }
+ """
+
+ def GetTestName(self, packagename: str, environment: VarDict) -> tuple:
+ """ Provide the testcase name and classname for use in reporting
+ testclassname: a descriptive string for the testcase can include whitespace
+ classname: should be patterned <packagename>.<plugin>.<optionally any unique condition>
+
+ Args:
+ packagename: string containing name of package to build
+ environment: The VarDict for the test to run in
+ Returns:
+ a tuple containing the testcase name and the classname
+ (testcasename, classname)
+ """
+ return ("Check library class declarations in " + packagename, packagename + ".LibraryClassCheck")
+
+ def __GetPkgDec(self, rootpath):
+ try:
+ allEntries = os.listdir(rootpath)
+ for entry in allEntries:
+ if entry.lower().endswith(".dec"):
+ return(os.path.join(rootpath, entry))
+ except Exception:
+ logging.error("Unable to find DEC for package:{0}".format(rootpath))
+
+ return None
+
+ ##
+ # External function of plugin. This function is used to perform the task of the MuBuild Plugin
+ #
+ # - package is the edk2 path to package. This means workspace/packagepath relative.
+ # - edk2path object configured with workspace and packages path
+ # - PkgConfig Object (dict) for the pkg
+ # - EnvConfig Object
+ # - Plugin Manager Instance
+ # - Plugin Helper Obj Instance
+ # - Junit Logger
+ # - output_stream the StringIO output stream from this plugin via logging
+ def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None):
+ overall_status = 0
+ LibraryClassIgnore = []
+
+ abs_pkg_path = Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath(packagename)
+ abs_dec_path = self.__GetPkgDec(abs_pkg_path)
+ wsr_dec_path = Edk2pathObj.GetEdk2RelativePathFromAbsolutePath(abs_dec_path)
+
+ if abs_dec_path is None or wsr_dec_path is "" or not os.path.isfile(abs_dec_path):
+ tc.SetSkipped()
+ tc.LogStdError("No DEC file {0} in package {1}".format(abs_dec_path, abs_pkg_path))
+ return -1
+
+ # Get all include folders
+ dec = DecParser()
+ dec.SetBaseAbsPath(Edk2pathObj.WorkspacePath).SetPackagePaths(Edk2pathObj.PackagePathList)
+ dec.ParseFile(wsr_dec_path)
+
+ AllHeaderFiles = []
+
+ for includepath in dec.IncludePaths:
+ ## Get all header files in the library folder
+ AbsLibraryIncludePath = os.path.join(abs_pkg_path, includepath, "Library")
+ if(not os.path.isdir(AbsLibraryIncludePath)):
+ continue
+
+ hfiles = self.WalkDirectoryForExtension([".h"], AbsLibraryIncludePath)
+ hfiles = [os.path.relpath(x,abs_pkg_path) for x in hfiles] # make package root relative path
+ hfiles = [x.replace("\\", "/") for x in hfiles] # make package relative path
+
+ AllHeaderFiles.extend(hfiles)
+
+ if len(AllHeaderFiles) == 0:
+ tc.SetSkipped()
+ tc.LogStdError(f"No Library include folder in any Include path")
+ return -1
+
+ # Remove ignored paths
+ if "IgnoreHeaderFile" in pkgconfig:
+ for a in pkgconfig["IgnoreHeaderFile"]:
+ try:
+ tc.LogStdOut("Ignoring Library Header File {0}".format(a))
+ AllHeaderFiles.remove(a)
+ except:
+ tc.LogStdError("LibraryClassCheck.IgnoreHeaderFile -> {0} not found. Invalid Header File".format(a))
+ logging.info("LibraryClassCheck.IgnoreHeaderFile -> {0} not found. Invalid Header File".format(a))
+
+ if "IgnoreLibraryClass" in pkgconfig:
+ LibraryClassIgnore = pkgconfig["IgnoreLibraryClass"]
+
+
+ ## Attempt to find library classes
+ for lcd in dec.LibraryClasses:
+ ## Check for correct file path separator
+ if "\\" in lcd.path:
+ tc.LogStdError("LibraryClassCheck.DecFilePathSeparator -> {0} invalid.".format(lcd.path))
+ logging.error("LibraryClassCheck.DecFilePathSeparator -> {0} invalid.".format(lcd.path))
+ overall_status += 1
+ continue
+
+ if lcd.name in LibraryClassIgnore:
+ tc.LogStdOut("Ignoring Library Class Name {0}".format(lcd.name))
+ LibraryClassIgnore.remove(lcd.name)
+ continue
+
+ logging.debug(f"Looking for Library Class {lcd.path}")
+ try:
+ AllHeaderFiles.remove(lcd.path)
+
+ except ValueError:
+ tc.LogStdError(f"Library {lcd.name} with path {lcd.path} not found in package filesystem")
+ logging.error(f"Library {lcd.name} with path {lcd.path} not found in package filesystem")
+ overall_status += 1
+
+ ## any remaining AllHeaderFiles are not described in DEC
+ for h in AllHeaderFiles:
+ tc.LogStdError(f"Library Header File {h} not declared in package DEC {wsr_dec_path}")
+ logging.error(f"Library Header File {h} not declared in package DEC {wsr_dec_path}")
+ overall_status += 1
+
+ ## Warn about any invalid library class names in the ignore list
+ for r in LibraryClassIgnore:
+ tc.LogStdError("LibraryClassCheck.IgnoreLibraryClass -> {0} not found. Library Class not found".format(r))
+ logging.info("LibraryClassCheck.IgnoreLibraryClass -> {0} not found. Library Class not found".format(r))
+
+
+ # If XML object exists, add result
+ if overall_status is not 0:
+ tc.SetFailed("LibraryClassCheck {0} Failed. Errors {1}".format(wsr_dec_path, overall_status), "CHECK_FAILED")
+ else:
+ tc.SetSuccess()
+ return overall_status
diff --git a/.pytool/Plugin/LibraryClassCheck/LibraryClassCheck_plug_in.yaml b/.pytool/Plugin/LibraryClassCheck/LibraryClassCheck_plug_in.yaml new file mode 100644 index 0000000000..f90eec32ff --- /dev/null +++ b/.pytool/Plugin/LibraryClassCheck/LibraryClassCheck_plug_in.yaml @@ -0,0 +1,11 @@ +## @file
+# CiBuildPlugin used to check that all library classes are declared correctly in dec file
+#
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+{
+ "scope": "cibuild",
+ "name": "Library Class Check Test",
+ "module": "LibraryClassCheck"
+}
diff --git a/.pytool/Plugin/LibraryClassCheck/Readme.md b/.pytool/Plugin/LibraryClassCheck/Readme.md new file mode 100644 index 0000000000..7fa2b202f2 --- /dev/null +++ b/.pytool/Plugin/LibraryClassCheck/Readme.md @@ -0,0 +1,25 @@ +# Library Class Check Plugin
+
+This CiBuildPlugin scans at all library header files found in the `Library`
+folders in all of the package's declared include directories and ensures that
+all files have a matching LibraryClass declaration in the DEC file for the
+package. Any missing declarations will cause a failure.
+
+## Configuration
+
+The plugin has a few configuration options to support the UEFI codebase.
+
+``` yaml
+"LibraryClassCheck": {
+ IgnoreHeaderFile: [], # Ignore a file found on disk
+ IgnoreLibraryClass: [] # Ignore a declaration found in dec file
+}
+```
+
+### IgnoreHeaderFile
+
+Ignore a file found on disk
+
+### IgnoreLibraryClass
+
+Ignore a declaration found in dec file
diff --git a/.pytool/Plugin/SpellCheck/Readme.md b/.pytool/Plugin/SpellCheck/Readme.md new file mode 100644 index 0000000000..394bb8effc --- /dev/null +++ b/.pytool/Plugin/SpellCheck/Readme.md @@ -0,0 +1,127 @@ +# Spell Check Plugin
+
+This CiBuildPlugin scans all the files in a given package and checks for
+spelling errors.
+
+This plugin requires NodeJs and cspell. If the plugin doesn't find its required
+tools then it will mark the test as skipped.
+
+* NodeJS: https://nodejs.org/en/
+* cspell: https://www.npmjs.com/package/cspell
+ * Src and doc available: https://github.com/streetsidesoftware/cspell
+
+## Configuration
+
+The plugin has a few configuration options to support the UEFI codebase.
+
+``` yaml
+ "SpellCheck": {
+ "AuditOnly": False, # If True, log all errors and then mark as skipped
+ "IgnoreFiles": [], # use gitignore syntax to ignore errors in matching files
+ "ExtendWords": [], # words to extend to the dictionary for this package
+ "IgnoreStandardPaths": [], # Standard Plugin defined paths that should be ignore
+ "AdditionalIncludePaths": [] # Additional paths to spell check (wildcards supported)
+ }
+```
+
+### AuditOnly
+
+Boolean - Default is False.
+If True run the test in an Audit only mode which will log all errors but instead
+of failing the build it will set the test as skipped. This allows visibility
+into the failures without breaking the build.
+
+### IgnoreFiles
+
+This supports .gitignore file and folder matching strings including wildcards
+
+* All files will be parsed regardless but then any spelling errors found within
+ ignored files will not be reported as an error.
+* Errors in ignored files will still be output to the test results as
+ informational comments.
+
+### ExtendWords
+
+This list allows words to be added to the dictionary for the spell checker when
+this package is tested. These follow the rules of the cspell config words field.
+
+### IgnoreStandardPaths
+
+This plugin by default will check the below standard paths. If the package
+would like to ignore any of them list that here.
+
+```python
+[
+# C source
+"*.c",
+"*.h",
+
+# Assembly files
+"*.nasm",
+"*.asm",
+"*.masm",
+"*.s",
+
+# ACPI source language
+"*.asl",
+
+# Edk2 build files
+"*.dsc", "*.dec", "*.fdf", "*.inf",
+
+# Documentation files
+"*.md", "*.txt"
+]
+```
+
+### AdditionalIncludePaths
+
+If the package would to add additional path patterns to be included in
+spellchecking they can be defined here.
+
+## Other configuration
+
+In the cspell.base.json there are numerous other settings configured. There is
+no support to override these on a per package basis but future features could
+make this available. One interesting configuration option is `minWordLength`.
+Currently it is set to _5_ which means all 2,3, and 4 letter words will be
+ignored. This helps minimize the number of technical acronyms, register names,
+and other UEFI specific values that must be ignored.
+
+## False positives
+
+The cspell dictionary is not perfect and there are cases where technical words
+or acronyms are not found in the dictionary. There are three ways to resolve
+false positives and the choice for which method should be based on how broadly
+the word should be accepted.
+
+### CSpell Base Config file
+
+If the change should apply to all UEFI code and documentation then it should be
+added to the base config file `words` section. The base config file is adjacent
+to this file and titled `cspell.base.json`. This is a list of accepted words
+for all spell checking operations on all packages.
+
+### Package Config
+
+In the package `*.ci.yaml` file there is a `SpellCheck` config section. This
+section allows files to be ignored as well as words that should be considered
+valid for all files within this package. Add the desired words to the
+"ExtendedWords" member.
+
+### In-line File
+
+CSpell supports numerous methods to annotate your files to ignore words,
+sections, etc. This can be found in CSpell documentation. Suggestion here is
+to use a c-style comment at the top of the file to add words that should be
+ignored just for this file. Obviously this has the highest maintenance cost so
+it should only be used for file unique words.
+
+``` c
+// spell-checker:ignore unenroll, word2, word3
+```
+
+or
+
+```ini
+# spell-checker:ignore unenroll, word2, word3
+```
diff --git a/.pytool/Plugin/SpellCheck/SpellCheck.py b/.pytool/Plugin/SpellCheck/SpellCheck.py new file mode 100644 index 0000000000..43365441b9 --- /dev/null +++ b/.pytool/Plugin/SpellCheck/SpellCheck.py @@ -0,0 +1,216 @@ +# @file SpellCheck.py
+#
+# An edk2-pytool based plugin wrapper for cspell
+#
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+import logging
+import json
+import yaml
+from io import StringIO
+import os
+from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin
+from edk2toollib.utility_functions import RunCmd
+from edk2toolext.environment.var_dict import VarDict
+from edk2toollib.gitignore_parser import parse_gitignore_lines
+from edk2toolext.environment import version_aggregator
+
+
+class SpellCheck(ICiBuildPlugin):
+ """
+ A CiBuildPlugin that uses the cspell node module to scan the files
+ from the package being tested for spelling errors. The plugin contains
+ the base cspell.json file then thru the configuration options other settings
+ can be changed or extended.
+
+ Configuration options:
+ "SpellCheck": {
+ "AuditOnly": False, # Don't fail the build if there are errors. Just log them
+ "IgnoreFiles": [], # use gitignore syntax to ignore errors in matching files
+ "ExtendWords": [], # words to extend to the dictionary for this package
+ "IgnoreStandardPaths": [], # Standard Plugin defined paths that should be ignore
+ "AdditionalIncludePaths": [] # Additional paths to spell check (wildcards supported)
+ }
+ """
+
+ #
+ # A package can remove any of these using IgnoreStandardPaths
+ #
+ STANDARD_PLUGIN_DEFINED_PATHS = ["*.c", "*.h",
+ "*.nasm", "*.asm", "*.masm", "*.s",
+ "*.asl",
+ "*.dsc", "*.dec", "*.fdf", "*.inf",
+ "*.md", "*.txt"
+ ]
+
+ def GetTestName(self, packagename: str, environment: VarDict) -> tuple:
+ """ Provide the testcase name and classname for use in reporting
+
+ Args:
+ packagename: string containing name of package to build
+ environment: The VarDict for the test to run in
+ Returns:
+ a tuple containing the testcase name and the classname
+ (testcasename, classname)
+ testclassname: a descriptive string for the testcase can include whitespace
+ classname: should be patterned <packagename>.<plugin>.<optionally any unique condition>
+ """
+ return ("Spell check files in " + packagename, packagename + ".SpellCheck")
+
+ ##
+ # External function of plugin. This function is used to perform the task of the CiBuild Plugin
+ #
+ # - package is the edk2 path to package. This means workspace/packagepath relative.
+ # - edk2path object configured with workspace and packages path
+ # - PkgConfig Object (dict) for the pkg
+ # - EnvConfig Object
+ # - Plugin Manager Instance
+ # - Plugin Helper Obj Instance
+ # - Junit Logger
+ # - output_stream the StringIO output stream from this plugin via logging
+
+ def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None):
+ Errors = []
+
+ abs_pkg_path = Edk2pathObj.GetAbsolutePathOnThisSytemFromEdk2RelativePath(
+ packagename)
+
+ if abs_pkg_path is None:
+ tc.SetSkipped()
+ tc.LogStdError("No package {0}".format(packagename))
+ return -1
+
+ # check for node
+ return_buffer = StringIO()
+ ret = RunCmd("node", "--version", outstream=return_buffer)
+ if (ret != 0):
+ tc.SetSkipped()
+ tc.LogStdError("NodeJs not installed. Test can't run")
+ logging.warning("NodeJs not installed. Test can't run")
+ return -1
+ node_version = return_buffer.getvalue().strip() # format vXX.XX.XX
+ tc.LogStdOut(f"Node version: {node_version}")
+ version_aggregator.GetVersionAggregator().ReportVersion(
+ "NodeJs", node_version, version_aggregator.VersionTypes.INFO)
+
+ # Check for cspell
+ return_buffer = StringIO()
+ ret = RunCmd("cspell", "--version", outstream=return_buffer)
+ if (ret != 0):
+ tc.SetSkipped()
+ tc.LogStdError("cspell not installed. Test can't run")
+ logging.warning("cspell not installed. Test can't run")
+ return -1
+ cspell_version = return_buffer.getvalue().strip() # format XX.XX.XX
+ tc.LogStdOut(f"CSpell version: {cspell_version}")
+ version_aggregator.GetVersionAggregator().ReportVersion(
+ "CSpell", cspell_version, version_aggregator.VersionTypes.INFO)
+
+ package_relative_paths_to_spell_check = SpellCheck.STANDARD_PLUGIN_DEFINED_PATHS
+
+ #
+ # Allow the ci.yaml to remove any of the above standard paths
+ #
+ if("IgnoreStandardPaths" in pkgconfig):
+ for a in pkgconfig["IgnoreStandardPaths"]:
+ if(a in package_relative_paths_to_spell_check):
+ tc.LogStdOut(
+ f"ignoring standard path due to ci.yaml ignore: {a}")
+ package_relative_paths_to_spell_check.remove(a)
+ else:
+ tc.LogStdOut(f"Invalid IgnoreStandardPaths value: {a}")
+
+ #
+ # check for any additional include paths defined by package config
+ #
+ if("AdditionalIncludePaths" in pkgconfig):
+ package_relative_paths_to_spell_check.extend(
+ pkgconfig["AdditionalIncludePaths"])
+
+ #
+ # Make the path string for cspell to check
+ #
+ relpath = os.path.relpath(abs_pkg_path)
+ cpsell_paths = " ".join(
+ [f"{relpath}/**/{x}" for x in package_relative_paths_to_spell_check])
+
+ # Make the config file
+ config_file_path = os.path.join(
+ Edk2pathObj.WorkspacePath, "Build", packagename, "cspell_actual_config.json")
+ mydir = os.path.dirname(os.path.abspath(__file__))
+ # load as yaml so it can have comments
+ base = os.path.join(mydir, "cspell.base.yaml")
+ with open(base, "r") as i:
+ config = yaml.safe_load(i)
+
+ if("ExtendWords" in pkgconfig):
+ config["words"].extend(pkgconfig["ExtendWords"])
+ with open(config_file_path, "w") as o:
+ json.dump(config, o) # output as json so compat with cspell
+
+ All_Ignores = []
+ # Parse the config for other ignores
+ if "IgnoreFiles" in pkgconfig:
+ All_Ignores.extend(pkgconfig["IgnoreFiles"])
+
+ # spell check all the files
+ ignore = parse_gitignore_lines(All_Ignores, os.path.join(
+ abs_pkg_path, "nofile.txt"), abs_pkg_path)
+
+ # result is a list of strings like this
+ # C:\src\sp-edk2\edk2\FmpDevicePkg\FmpDevicePkg.dec:53:9 - Unknown word (Capule)
+ EasyFix = []
+ results = self._check_spelling(cpsell_paths, config_file_path)
+ for r in results:
+ path, _, word = r.partition(" - Unknown word ")
+ if len(word) == 0:
+ # didn't find pattern
+ continue
+
+ pathinfo = path.rsplit(":", 2) # remove the line no info
+ if(ignore(pathinfo[0])): # check against ignore list
+ tc.LogStdOut(f"ignoring error due to ci.yaml ignore: {r}")
+ continue
+
+ # real error
+ EasyFix.append(word.strip().strip("()"))
+ Errors.append(r)
+
+ # Log all errors tc StdError
+ for l in Errors:
+ tc.LogStdError(l.strip())
+
+ # Helper - Log the syntax needed to add these words to dictionary
+ if len(EasyFix) > 0:
+ EasyFix = sorted(set(a.lower() for a in EasyFix))
+ tc.LogStdOut("\n Easy fix:")
+ OneString = "If these are not errors add this to your ci.yaml file.\n"
+ OneString += '"SpellCheck": {\n "ExtendWords": ['
+ for a in EasyFix:
+ tc.LogStdOut(f'\n"{a}",')
+ OneString += f'\n "{a}",'
+ logging.info(OneString.rstrip(",") + '\n ]\n}')
+
+ # add result to test case
+ overall_status = len(Errors)
+ if overall_status != 0:
+ if "AuditOnly" in pkgconfig and pkgconfig["AuditOnly"]:
+ # set as skipped if AuditOnly
+ tc.SetSkipped()
+ return -1
+ else:
+ tc.SetFailed("SpellCheck {0} Failed. Errors {1}".format(
+ packagename, overall_status), "CHECK_FAILED")
+ else:
+ tc.SetSuccess()
+ return overall_status
+
+ def _check_spelling(self, abs_file_to_check: str, abs_config_file_to_use: str) -> []:
+ output = StringIO()
+ ret = RunCmd(
+ "cspell", f"--config {abs_config_file_to_use} {abs_file_to_check}", outstream=output)
+ if ret == 0:
+ return []
+ else:
+ return output.getvalue().strip().splitlines()
diff --git a/.pytool/Plugin/SpellCheck/SpellCheck_plug_in.yaml b/.pytool/Plugin/SpellCheck/SpellCheck_plug_in.yaml new file mode 100644 index 0000000000..d438199f60 --- /dev/null +++ b/.pytool/Plugin/SpellCheck/SpellCheck_plug_in.yaml @@ -0,0 +1,11 @@ +## @file
+# CiBuildPlugin used to check spelling
+#
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+{
+ "scope": "cibuild",
+ "name": "Spell Check Test",
+ "module": "SpellCheck"
+}
diff --git a/.pytool/Plugin/SpellCheck/cspell.base.yaml b/.pytool/Plugin/SpellCheck/cspell.base.yaml new file mode 100644 index 0000000000..53000fc381 --- /dev/null +++ b/.pytool/Plugin/SpellCheck/cspell.base.yaml @@ -0,0 +1,165 @@ +## @file
+# CSpell configuration
+#
+# Copyright (c) Microsoft Corporation
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+{
+ "version": "0.1",
+ "language": "en",
+ "dictionaries": [
+ "companies ",
+ "softwareTerms",
+ "python",
+ "cpp"
+ ],
+ "ignorePaths": [
+ "*.pdb",
+ "**/*_extdep/**",
+ "*.pdf",
+ "*.exe",
+ "*.jpg"
+ ],
+ "minWordLength": 5,
+ "allowCompoundWords": false,
+ "ignoreWords": [
+ "muchange"
+ ],
+ "words": [
+ "MTRRs",
+ "Microarchitecture",
+ "Goldmont",
+ "cpuid",
+ "mwait",
+ "cstate",
+ "smram",
+ "scrtm",
+ "smbus",
+ "selftest",
+ "socket",
+ "MMRAM",
+ "qword",
+ "ENDBR",
+ "SMBASE",
+ "FXSAVE",
+ "FXRSTOR",
+ "RDRAND",
+ "IOAPIC",
+ "ATAPI",
+ "movsb",
+ "iretw",
+ "XENSTORE",
+ "cdrom",
+ "oprom",
+ "oproms",
+ "varstore",
+ "EKU",
+ "ascii",
+ "nmake",
+ "NVDIMM",
+ "nasmb",
+ "Mtftp",
+ "Hypercall",
+ "hypercalls",
+ "IOMMU",
+ "QEMU",
+ "qemus",
+ "OVMF",
+ "tiano",
+ "tianocore",
+ "edkii",
+ "coreboot",
+ "uefipayload",
+ "bootloader",
+ "bootloaders",
+ "mdepkg",
+ "skuid",
+ "dxefv",
+ "toolchain",
+ "libraryclass",
+ "preboot",
+ "pythonpath",
+ "cygpath",
+ "nuget",
+ "basetools",
+ "prepi",
+ "OPTEE",
+ "stringid",
+ "peims",
+ "memmap",
+ "guids",
+ "uuids",
+ "smbios",
+ "certdb",
+ "certdbv",
+ "EfiSigList",
+ "depex",
+ "IHANDLE",
+ "Virtio",
+ "Mbytes",
+ "Citrix",
+ "initrd",
+ "semihost",
+ "Semihosting",
+ "Trustzone",
+ "Fastboot",
+ "framebuffer",
+ "genfw",
+ "TTYTERM",
+ "miniport",
+ "LFENCE",
+ "PCANSI",
+ "submodule",
+ "submodules",
+ "brotli",
+ "PCCTS",
+ "softfloat",
+ "whitepaper",
+ "ACPICA",
+ "plugfest",
+ "bringup",
+ "formset", #VFR
+ "ideqvallist",
+ "numberof",
+ "oneof",
+ "endformset",
+ "endnumeric",
+ "endoneof",
+ "disableif",
+ "guidid",
+ "classguid",
+ "efivarstore",
+ "formsetguid",
+ "formid",
+ "suppressif",
+ "grayoutif",
+ "ideqval",
+ "endform",
+ "endcheckbox",
+ "questionid",
+ "questionref",
+ "enddate",
+ "endstring",
+ "guidop",
+ "endguidop",
+ "langdef",
+ "dynamicex",
+ "tokenspace",
+ "tokenguid",
+ "pcd's", #seems like cspell bug
+ "peim's",
+ "autogen",
+ "Disasm",
+ "Torito",
+ "SRIOV",
+ "MRIOV",
+ "UARTs",
+ "Consplitter", # common module in UEFI
+ "FIFOs",
+ "ACPINVS",
+ "Endof", # due to of not being uppercase
+ "bootability",
+ "Sdhci",
+ "inmodule",
+ ]
+}
|