BaseTools: Enhance Basetool for incremental build

BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=2311

Include dependency file in Makefile to enhance
incremental build

Signed-off-by: Bob Feng <bob.c.feng@intel.com>

Cc: Liming Gao <liming.gao@intel.com>
Cc: Steven Shi <steven.shi@intel.com>
Reviewed-by: Liming Gao <liming.gao@intel.com>

3 files changed, 331 insertions, 59 deletions

diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py b/BaseTools/Source/Python/AutoGen/GenMake.py
index 59a01a7f24..fe94f9a4c2 100755
--- a/BaseTools/Source/Python/AutoGen/GenMake.py
+++ b/BaseTools/Source/Python/AutoGen/GenMake.py
@@ -185,6 +185,12 @@ class BuildFile(object):
         self._FileType = FileType

         FileContent = self._TEMPLATE_.Replace(self._TemplateDict)

         FileName = self._FILE_NAME_[FileType]

+        if not os.path.exists(os.path.join(self._AutoGenObject.MakeFileDir, "deps.txt")):

+            with open(os.path.join(self._AutoGenObject.MakeFileDir, "deps.txt"),"w+") as fd:

+                fd.write("")

+        if not os.path.exists(os.path.join(self._AutoGenObject.MakeFileDir, "dependency")):

+            with open(os.path.join(self._AutoGenObject.MakeFileDir, "dependency"),"w+") as fd:

+                fd.write("")

         return SaveFileOnChange(os.path.join(self._AutoGenObject.MakeFileDir, FileName), FileContent, False)

 

     ## Return a list of directory creation command string

@@ -304,9 +310,6 @@ MAKE_FILE = ${makefile_path}
 ${BEGIN}${file_macro}

 ${END}

 

-COMMON_DEPS = ${BEGIN}${common_dependency_file} \\

-              ${END}

-

 #

 # Overridable Target Macro Definitions

 #

@@ -382,6 +385,8 @@ gen_fds:
 \t@"$(MAKE)" $(MAKE_FLAGS) -f $(BUILD_DIR)${separator}${makefile_name} fds

 \t@cd $(MODULE_BUILD_DIR)

 

+${INCLUDETAG}

+

 #

 # Individual Object Build Targets

 #

@@ -515,9 +520,6 @@ cleanlib:
                     # Remove duplicated include path, if any

                     if Attr == "FLAGS":

                         Value = RemoveDupOption(Value, IncPrefix, MyAgo.IncludePathList)

-                        if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and Tool == 'CC' and '/GM' in Value:

-                            Value = Value.replace(' /MP', '')

-                            MyAgo.BuildOption[Tool][Attr] = Value

                         if Tool == "OPTROM" and PCI_COMPRESS_Flag:

                             ValueList = Value.split()

                             if ValueList:

@@ -540,7 +542,7 @@ cleanlib:
                 UnexpandMacro = []

                 NewStr = []

                 for Str in StrList:

-                    if '$' in Str:

+                    if '$' in Str or '-MMD' in Str or '-MF' in Str:

                         UnexpandMacro.append(Str)

                     else:

                         NewStr.append(Str)

@@ -590,10 +592,9 @@ cleanlib:
                                                 )

         FileMacroList.append(FileMacro)

         # Add support when compiling .nasm source files

-        for File in self.FileCache.keys():

-            if not str(File).endswith('.nasm'):

-                continue

-            IncludePathList = []

+        IncludePathList = []

+        asmsource = [item for item in MyAgo.SourceFileList if item.File.upper().endswith((".NASM",".ASM",".NASMB","S"))]

+        if asmsource:

             for P in  MyAgo.IncludePathList:

                 IncludePath = self._INC_FLAG_['NASM'] + self.PlaceMacro(P, self.Macros)

                 if IncludePath.endswith(os.sep):

@@ -606,7 +607,6 @@ cleanlib:
                     IncludePath = os.path.join(IncludePath, '')

                 IncludePathList.append(IncludePath)

             FileMacroList.append(self._FILE_MACRO_TEMPLATE.Replace({"macro_name": "NASM_INC", "source_file": IncludePathList}))

-            break

 

         # Generate macros used to represent files containing list of input files

         for ListFileMacro in self.ListFileMacros:

@@ -696,6 +696,7 @@ cleanlib:
             "file_macro"                : FileMacroList,

             "file_build_target"         : self.BuildTargetList,

             "backward_compatible_target": BcTargetList,

+            "INCLUDETAG"                   : self._INCLUDE_CMD_[self._FileType] + " " + os.path.join("$(MODULE_BUILD_DIR)","dependency")

         }

 

         return MakefileTemplateDict

@@ -903,16 +904,10 @@ cleanlib:
                 if Item in SourceFileList:

                     SourceFileList.remove(Item)

 

-        FileDependencyDict = self.GetFileDependency(

-                                    SourceFileList,

-                                    ForceIncludedFile,

-                                    self._AutoGenObject.IncludePathList + self._AutoGenObject.BuildOptionIncPathList

-                                    )

-

+        FileDependencyDict = {item:ForceIncludedFile for item in SourceFileList}

 

-        if FileDependencyDict:

-            for Dependency in FileDependencyDict.values():

-                self.DependencyHeaderFileSet.update(set(Dependency))

+        for Dependency in FileDependencyDict.values():

+            self.DependencyHeaderFileSet.update(set(Dependency))

 

         # Get a set of unique package includes from MetaFile

         parentMetaFileIncludes = set()

@@ -972,42 +967,16 @@ cleanlib:
                         ExtraData = "Local Header: " + aFile + " not found in " + self._AutoGenObject.MetaFile.Path

                         )

 

-        DepSet = None

         for File,Dependency in FileDependencyDict.items():

             if not Dependency:

-                FileDependencyDict[File] = ['$(FORCE_REBUILD)']

                 continue

 

             self._AutoGenObject.AutoGenDepSet |= set(Dependency)

 

-            # skip non-C files

-            if File.Ext not in [".c", ".C"] or File.Name == "AutoGen.c":

-                continue

-            elif DepSet is None:

-                DepSet = set(Dependency)

-            else:

-                DepSet &= set(Dependency)

-        # in case nothing in SourceFileList

-        if DepSet is None:

-            DepSet = set()

-        #

-        # Extract common files list in the dependency files

-        #

-        for File in DepSet:

-            self.CommonFileDependency.append(self.PlaceMacro(File.Path, self.Macros))

-

         CmdSumDict = {}

         CmdTargetDict = {}

         CmdCppDict = {}

         DependencyDict = FileDependencyDict.copy()

-        for File in FileDependencyDict:

-            # skip non-C files

-            if File.Ext not in [".c", ".C"] or File.Name == "AutoGen.c":

-                continue

-            NewDepSet = set(FileDependencyDict[File])

-            NewDepSet -= DepSet

-            FileDependencyDict[File] = ["$(COMMON_DEPS)"] + list(NewDepSet)

-            DependencyDict[File] = list(NewDepSet)

 

         # Convert target description object to target string in makefile

         if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and TAB_C_CODE_FILE in self._AutoGenObject.Targets:

@@ -1080,17 +1049,13 @@ cleanlib:
                     else:

                         CmdCppDict[item.Target.SubDir] = ['$(MAKE_FILE)', Path]

                     if CppPath.Path in DependencyDict:

-                        if '$(FORCE_REBUILD)' in DependencyDict[CppPath.Path]:

-                            if '$(FORCE_REBUILD)' not in (self.CommonFileDependency + CmdCppDict[item.Target.SubDir]):

-                                CmdCppDict[item.Target.SubDir].append('$(FORCE_REBUILD)')

-                        else:

-                            for Temp in DependencyDict[CppPath.Path]:

-                                try:

-                                    Path = self.PlaceMacro(Temp.Path, self.Macros)

-                                except:

-                                    continue

-                                if Path not in (self.CommonFileDependency + CmdCppDict[item.Target.SubDir]):

-                                    CmdCppDict[item.Target.SubDir].append(Path)

+                        for Temp in DependencyDict[CppPath.Path]:

+                            try:

+                                Path = self.PlaceMacro(Temp.Path, self.Macros)

+                            except:

+                                continue

+                            if Path not in (self.CommonFileDependency + CmdCppDict[item.Target.SubDir]):

+                                CmdCppDict[item.Target.SubDir].append(Path)

         if T.Commands:

             CommandList = T.Commands[:]

             for Item in CommandList[:]:

@@ -1109,7 +1074,7 @@ cleanlib:
                     CommandList.pop(Index)

                     if SingleCommandList[-1].endswith("%s%s.c" % (TAB_SLASH, CmdSumDict[CmdSign[3:].rsplit(TAB_SLASH, 1)[0]])):

                         Cpplist = CmdCppDict[T.Target.SubDir]

-                        Cpplist.insert(0, '$(OBJLIST_%d): $(COMMON_DEPS)' % list(self.ObjTargetDict.keys()).index(T.Target.SubDir))

+                        Cpplist.insert(0, '$(OBJLIST_%d): ' % list(self.ObjTargetDict.keys()).index(T.Target.SubDir))

                         T.Commands[Index] = '%s\n\t%s' % (' \\\n\t'.join(Cpplist), CmdTargetDict[CmdSign])

                     else:

                         T.Commands.pop(Index)

diff --git a/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py
new file mode 100644
index 0000000000..bb6e883d84
--- /dev/null
+++ b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py
@@ -0,0 +1,284 @@
+## @file

+# Build cache intermediate result and state

+#

+# Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>

+# SPDX-License-Identifier: BSD-2-Clause-Patent

+#

+from Common.caching import cached_property

+import Common.EdkLogger as EdkLogger

+import Common.LongFilePathOs as os

+from Common.BuildToolError import *

+from Common.Misc import SaveFileOnChange, PathClass

+from Common.Misc import TemplateString

+import sys

+gIsFileMap = {}

+if sys.platform == "win32":

+    _INCLUDE_DEPS_TEMPLATE = TemplateString('''

+${BEGIN}

+!IF EXIST(${deps_file})

+!INCLUDE ${deps_file}

+!ENDIF

+${END}

+        ''')

+else:

+    _INCLUDE_DEPS_TEMPLATE = TemplateString('''

+${BEGIN}

+-include ${deps_file}

+${END}

+        ''')

+

+DEP_FILE_TAIL = "# Updated \n"

+

+class IncludesAutoGen():

+    """ This class is to manage the dependent files witch are used in Makefile to support incremental build.

+        1. C files:

+            1. MSVS.

+               cl.exe has a build option /showIncludes to display include files on stdout. Build tool captures

+               that messages and generate dependency files, .deps files.

+            2. CLANG and GCC

+               -MMD -MF build option are used to generate dependency files by compiler. Build tool updates the

+               .deps files.

+        2. ASL files:

+            1. Trim find out all the included files with asl specific include format and generate .trim.deps file.

+            2. ASL PP use c preprocessor to find out all included files with #include format and generate a .deps file

+            3. build tool updates the .deps file

+        3. ASM files (.asm, .s or .nasm):

+            1. Trim find out all the included files with asl specific include format and generate .trim.deps file.

+            2. ASM PP use c preprocessor to find out all included files with #include format and generate a deps file

+            3. build tool updates the .deps file

+    """

+    def __init__(self, makefile_folder, ModuleAuto):

+        self.d_folder = makefile_folder

+        self.makefile_folder = makefile_folder

+        self.module_autogen = ModuleAuto

+        self.ToolChainFamily = ModuleAuto.ToolChainFamily

+        self.workspace = ModuleAuto.WorkspaceDir

+

+    def CreateModuleDeps(self):

+        SaveFileOnChange(os.path.join(self.makefile_folder,"deps.txt"),"\n".join(self.DepsCollection),False)

+

+    def CreateDepsInclude(self):

+        deps_file = {'deps_file':self.deps_files}

+        try:

+            deps_include_str = _INCLUDE_DEPS_TEMPLATE.Replace(deps_file)

+        except Exception as e:

+            print(e)

+        SaveFileOnChange(os.path.join(self.makefile_folder,"dependency"),deps_include_str,False)

+

+    @cached_property

+    def deps_files(self):

+        """ Get all .deps file under module build folder. """

+        deps_files = []

+        for root, _, files in os.walk(self.d_folder, topdown=False):

+            for name in files:

+                if not name.endswith(".deps"):

+                    continue

+                abspath = os.path.join(root, name)

+                deps_files.append(abspath)

+        return deps_files

+

+    @cached_property

+    def DepsCollection(self):

+        """ Collect all the dependency files list from all .deps files under a module's build folder """

+        includes = set()

+        targetname = [item[0].Name for item in self.TargetFileList.values()]

+        for abspath in self.deps_files:

+            try:

+                with open(abspath,"r") as fd:

+                    lines = fd.readlines()

+

+                firstlineitems = lines[0].split(": ")

+                dependency_file = firstlineitems[1].strip(" \\\n")

+                dependency_file = dependency_file.strip('''"''')

+                if dependency_file:

+                    if os.path.normpath(dependency_file +".deps") == abspath:

+                        continue

+                    filename = os.path.basename(dependency_file).strip()

+                    if filename not in self.SourceFileList and filename not in targetname:

+                        includes.add(dependency_file.strip())

+

+                for item in lines[1:]:

+                    if item == DEP_FILE_TAIL:

+                        continue

+                    dependency_file = item.strip(" \\\n")

+                    dependency_file = dependency_file.strip('''"''')

+                    if os.path.normpath(dependency_file +".deps") == abspath:

+                        continue

+                    filename = os.path.basename(dependency_file).strip()

+                    if filename in self.SourceFileList:

+                        continue

+                    if filename in targetname:

+                        continue

+                    includes.add(dependency_file.strip())

+            except Exception as e:

+                EdkLogger.error("build",FILE_NOT_FOUND, "%s doesn't exist" % abspath, ExtraData=str(e), RaiseError=False)

+                continue

+        rt = sorted(list(set([item.strip(' " \\\n') for item in includes])))

+        return rt

+

+    @cached_property

+    def SourceFileList(self):

+        """ Get a map of module's source files name to module's source files path """

+        source = {os.path.basename(item.File):item.Path for item in self.module_autogen.SourceFileList}

+        middle_file = {}

+        for afile in source:

+            if afile.upper().endswith(".VFR"):

+                middle_file.update({afile.split(".")[0]+".c":os.path.join(self.module_autogen.DebugDir,afile.split(".")[0]+".c")})

+            if afile.upper().endswith((".S","ASM")):

+                middle_file.update({afile.split(".")[0]+".i":os.path.join(self.module_autogen.OutputDir,afile.split(".")[0]+".i")})

+            if afile.upper().endswith(".ASL"):

+                middle_file.update({afile.split(".")[0]+".i":os.path.join(self.module_autogen.OutputDir,afile.split(".")[0]+".i")})

+        source.update({"AutoGen.c":os.path.join(self.module_autogen.OutputDir,"AutoGen.c")})

+        source.update(middle_file)

+        return source

+

+    @cached_property

+    def HasNamesakeSourceFile(self):

+        source_base_name = set([os.path.basename(item.File) for item in self.module_autogen.SourceFileList])

+        rt = len(source_base_name) != len(self.module_autogen.SourceFileList)

+        return rt

+    @cached_property

+    def CcPPCommandPathSet(self):

+        rt = set()

+        rt.add(self.module_autogen.BuildOption.get('CC',{}).get('PATH'))

+        rt.add(self.module_autogen.BuildOption.get('ASLCC',{}).get('PATH'))

+        rt.add(self.module_autogen.BuildOption.get('ASLPP',{}).get('PATH'))

+        rt.add(self.module_autogen.BuildOption.get('VFRPP',{}).get('PATH'))

+        rt.add(self.module_autogen.BuildOption.get('PP',{}).get('PATH'))

+        rt.add(self.module_autogen.BuildOption.get('APP',{}).get('PATH'))

+        rt.discard(None)

+        return rt

+    @cached_property

+    def TargetFileList(self):

+        """ Get a map of module's target name to a tuple of module's targets path and whose input file path """

+        targets = {}

+        targets["AutoGen.obj"] = (PathClass(os.path.join(self.module_autogen.OutputDir,"AutoGen.obj")),PathClass(os.path.join(self.module_autogen.DebugDir,"AutoGen.c")))

+        for item in self.module_autogen.Targets.values():

+            for block in item:

+                targets[block.Target.Path] = (block.Target,block.Inputs[0])

+        return targets

+

+    def GetRealTarget(self,source_file_abs):

+        """ Get the final target file based on source file abspath """

+        source_target_map = {item[1].Path:item[0].Path for item in self.TargetFileList.values()}

+        source_name_map = {item[1].File:item[0].Path for item in self.TargetFileList.values()}

+        target_abs = source_target_map.get(source_file_abs)

+        if target_abs is None:

+            if source_file_abs.strip().endswith(".i"):

+                sourcefilename = os.path.basename(source_file_abs.strip())

+                for sourcefile in source_name_map:

+                    if sourcefilename.split(".")[0] == sourcefile.split(".")[0]:

+                        target_abs = source_name_map[sourcefile]

+                        break

+                else:

+                    target_abs = source_file_abs

+            else:

+                target_abs = source_file_abs

+        return target_abs

+

+    def CreateDepsFileForMsvc(self, DepList):

+        """ Generate dependency files, .deps file from /showIncludes output message """

+        if not DepList:

+            return

+        ModuleDepDict = {}

+        current_source = ""

+        SourceFileAbsPathMap = self.SourceFileList

+        for line in DepList:

+            line = line.strip()

+            if self.HasNamesakeSourceFile:

+                for cc_cmd in self.CcPPCommandPathSet:

+                    if cc_cmd in line:

+                        if '''"'''+cc_cmd+'''"''' in line:

+                            cc_options = line[len(cc_cmd)+2:].split()

+                        else:

+                            cc_options = line[len(cc_cmd):].split()

+                        SourceFileAbsPathMap = {os.path.basename(item):item for item in cc_options if not item.startswith("/") and os.path.exists(item)}

+            if line in SourceFileAbsPathMap:

+                current_source = line

+                if current_source not in ModuleDepDict:

+                    ModuleDepDict[SourceFileAbsPathMap[current_source]] = []

+            elif "Note: including file:" ==  line.lstrip()[:21]:

+                if not current_source:

+                    EdkLogger.error("build",BUILD_ERROR, "Parse /showIncludes output failed. line: %s. \n" % line, RaiseError=False)

+                else:

+                    ModuleDepDict[SourceFileAbsPathMap[current_source]].append(line.lstrip()[22:].strip())

+

+        for source_abs in ModuleDepDict:

+            if ModuleDepDict[source_abs]:

+                target_abs = self.GetRealTarget(source_abs)

+                dep_file_name = os.path.basename(source_abs) + ".deps"

+                SaveFileOnChange(os.path.join(os.path.dirname(target_abs),dep_file_name)," \\\n".join([target_abs+":"] + ['''"''' + item +'''"''' for item in ModuleDepDict[source_abs]]),False)

+

+    def UpdateDepsFileforNonMsvc(self):

+        """ Update .deps files.

+            1. Update target path to absolute path.

+            2. Update middle target to final target.

+        """

+

+        for abspath in self.deps_files:

+            if abspath.endswith(".trim.deps"):

+                continue

+            try:

+                newcontent = []

+                with open(abspath,"r") as fd:

+                    lines = fd.readlines()

+                if lines[-1] == DEP_FILE_TAIL:

+                    continue

+                firstlineitems = lines[0].strip().split(" ")

+

+                if len(firstlineitems) > 2:

+                    sourceitem = firstlineitems[1]

+                else:

+                    sourceitem = lines[1].strip().split(" ")[0]

+

+                source_abs = self.SourceFileList.get(sourceitem,sourceitem)

+                firstlineitems[0] = self.GetRealTarget(source_abs)

+                p_target = firstlineitems

+                if not p_target[0].strip().endswith(":"):

+                    p_target[0] += ": "

+

+                if len(p_target) == 2:

+                    p_target[0] += lines[1]

+                    newcontent.append(p_target[0])

+                    newcontent.extend(lines[2:])

+                else:

+                    line1 = " ".join(p_target).strip()

+                    line1 += "\n"

+                    newcontent.append(line1)

+                    newcontent.extend(lines[1:])

+

+                newcontent.append("\n")

+                newcontent.append(DEP_FILE_TAIL)

+                with open(abspath,"w") as fw:

+                    fw.write("".join(newcontent))

+            except Exception as e:

+                EdkLogger.error("build",FILE_NOT_FOUND, "%s doesn't exist" % abspath, ExtraData=str(e), RaiseError=False)

+                continue

+

+    def UpdateDepsFileforTrim(self):

+        """ Update .deps file which generated by trim. """

+

+        for abspath in self.deps_files:

+            if not abspath.endswith(".trim.deps"):

+                continue

+            try:

+                newcontent = []

+                with open(abspath,"r") as fd:

+                    lines = fd.readlines()

+                if lines[-1] == DEP_FILE_TAIL:

+                    continue

+

+                source_abs = lines[0].strip().split(" ")[0]

+                targetitem = self.GetRealTarget(source_abs.strip(" :"))

+

+                targetitem += ": "

+                targetitem += lines[1]

+                newcontent.append(targetitem)

+                newcontent.extend(lines[2:])

+                newcontent.append("\n")

+                newcontent.append(DEP_FILE_TAIL)

+                with open(abspath,"w") as fw:

+                    fw.write("".join(newcontent))

+            except Exception as e:

+                EdkLogger.error("build",FILE_NOT_FOUND, "%s doesn't exist" % abspath, ExtraData=str(e), RaiseError=False)

+                continue

diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
index e6d6c43810..1111d5de25 100755
--- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
@@ -1129,9 +1129,32 @@ class ModuleAutoGen(AutoGen):
             for Inc in IncludesList:

                 if Inc not in RetVal:

                     RetVal.append(str(Inc))

+        RetVal.extend(self.IncPathFromBuildOptions)

         return RetVal

 

     @cached_property

+    def IncPathFromBuildOptions(self):

+        IncPathList = []

+        for tool in self.BuildOption:

+            if 'FLAGS' in self.BuildOption[tool]:

+                flags = self.BuildOption[tool]['FLAGS']

+                whitespace = False

+                for flag in flags.split(" "):

+                    flag = flag.strip()

+                    if flag.startswith(("/I","-I")):

+                        if len(flag)>2:

+                            if os.path.exists(flag[2:]):

+                                IncPathList.append(flag[2:])

+                        else:

+                            whitespace = True

+                            continue

+                    if whitespace and flag:

+                        if os.path.exists(flag):

+                            IncPathList.append(flag)

+                            whitespace = False

+        return IncPathList

+

+    @cached_property

     def IncludePathLength(self):

         return sum(len(inc)+1 for inc in self.IncludePathList)