BaseTools: Enable Multiple Process AutoGen

BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=1875

Assign the Module AutoGen tasks into multiple
sub process.

Cc: Liming Gao <liming.gao@intel.com>
Signed-off-by: Bob Feng <bob.c.feng@intel.com>
Acked-by: Laszlo Ersek <lersek@redhat.com>
Tested-by: Laszlo Ersek <lersek@redhat.com>
Acked-by: Liming Gao <liming.gao@intel.com>

1 files changed, 83 insertions, 44 deletions

diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py
index 3d083f4eaa..af1be8d281 100644
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -30,6 +30,7 @@ from optparse import OptionParser
 from AutoGen.PlatformAutoGen import PlatformAutoGen

 from AutoGen.ModuleAutoGen import ModuleAutoGen

 from AutoGen.WorkspaceAutoGen import WorkspaceAutoGen

+from AutoGen.AutoGenWorker import AutoGenWorkerInProcess,AutoGenManager

 from AutoGen import GenMake

 from Common import Misc as Utils

 

@@ -50,7 +51,7 @@ from PatchPcdValue.PatchPcdValue import PatchBinaryFile
 

 import Common.GlobalData as GlobalData

 from GenFds.GenFds import GenFds, GenFdsApi

-

+import multiprocessing as mp

 

 # Version and Copyright

 VersionNumber = "0.60" + ' ' + gBUILD_VERSION

@@ -343,9 +344,9 @@ class ModuleMakeUnit(BuildUnit):
     #   @param  Obj         The ModuleAutoGen object the build is working on

     #   @param  Target      The build target name, one of gSupportedTarget

     #

-    def __init__(self, Obj, Target):

-        Dependency = [ModuleMakeUnit(La, Target) for La in Obj.LibraryAutoGenList]

-        BuildUnit.__init__(self, Obj, Obj.BuildCommand, Target, Dependency, Obj.MakeFileDir)

+    def __init__(self, Obj, BuildCommand,Target):

+        Dependency = [ModuleMakeUnit(La, BuildCommand,Target) for La in Obj.LibraryAutoGenList]

+        BuildUnit.__init__(self, Obj, BuildCommand, Target, Dependency, Obj.MakeFileDir)

         if Target in [None, "", "all"]:

             self.Target = "tbuild"

 

@@ -364,10 +365,10 @@ class PlatformMakeUnit(BuildUnit):
     #   @param  Obj         The PlatformAutoGen object the build is working on

     #   @param  Target      The build target name, one of gSupportedTarget

     #

-    def __init__(self, Obj, Target):

-        Dependency = [ModuleMakeUnit(Lib, Target) for Lib in self.BuildObject.LibraryAutoGenList]

-        Dependency.extend([ModuleMakeUnit(Mod, Target) for Mod in self.BuildObject.ModuleAutoGenList])

-        BuildUnit.__init__(self, Obj, Obj.BuildCommand, Target, Dependency, Obj.MakeFileDir)

+    def __init__(self, Obj, BuildCommand, Target):

+        Dependency = [ModuleMakeUnit(Lib, BuildCommand, Target) for Lib in self.BuildObject.LibraryAutoGenList]

+        Dependency.extend([ModuleMakeUnit(Mod, BuildCommand,Target) for Mod in self.BuildObject.ModuleAutoGenList])

+        BuildUnit.__init__(self, Obj, BuildCommand, Target, Dependency, Obj.MakeFileDir)

 

 ## The class representing the task of a module build or platform build

 #

@@ -824,8 +825,31 @@ class Build():
         if not (self.LaunchPrebuildFlag and os.path.exists(self.PlatformBuildPath)):

             self.InitBuild()

 

+        self.AutoGenMgr = None

         EdkLogger.info("")

         os.chdir(self.WorkspaceDir)

+    def StartAutoGen(self,mqueue, DataPipe,SkipAutoGen,PcdMaList):

+        try:

+            if SkipAutoGen:

+                return True,0

+            feedback_q = mp.Queue()

+            file_lock = mp.Lock()

+            error_event = mp.Event()

+            auto_workers = [AutoGenWorkerInProcess(mqueue,DataPipe.dump_file,feedback_q,file_lock,error_event) for _ in range(self.ThreadNumber)]

+            self.AutoGenMgr = AutoGenManager(auto_workers,feedback_q,error_event)

+            self.AutoGenMgr.start()

+            for w in auto_workers:

+                w.start()

+            if PcdMaList is not None:

+                for PcdMa in PcdMaList:

+                    PcdMa.CreateCodeFile(False)

+                    PcdMa.CreateMakeFile(False,GenFfsList = DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.File, PcdMa.Arch),[]))

+

+            self.AutoGenMgr.join()

+            rt = self.AutoGenMgr.Status

+            return rt, 0

+        except Exception as e:

+            return False,e.errcode

 

     ## Load configuration

     #

@@ -1190,30 +1214,34 @@ class Build():
     #   @param  CreateDepModuleMakeFile     Flag used to indicate creating makefile

     #                                       for dependent modules/Libraries

     #

-    def _BuildPa(self, Target, AutoGenObject, CreateDepsCodeFile=True, CreateDepsMakeFile=True, BuildModule=False, FfsCommand={}):

+    def _BuildPa(self, Target, AutoGenObject, CreateDepsCodeFile=True, CreateDepsMakeFile=True, BuildModule=False, FfsCommand=None, PcdMaList=None):

         if AutoGenObject is None:

             return False

-

+        if FfsCommand is None:

+            FfsCommand = {}

         # skip file generation for cleanxxx targets, run and fds target

         if Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:

             # for target which must generate AutoGen code and makefile

-            if not self.SkipAutoGen or Target == 'genc':

-                self.Progress.Start("Generating code")

-                AutoGenObject.CreateCodeFile(CreateDepsCodeFile)

-                self.Progress.Stop("done!")

-            if Target == "genc":

-                return True

-

-            if not self.SkipAutoGen or Target == 'genmake':

-                self.Progress.Start("Generating makefile")

-                AutoGenObject.CreateMakeFile(CreateDepsMakeFile, FfsCommand)

-                self.Progress.Stop("done!")

-            if Target == "genmake":

-                return True

-        else:

-            # always recreate top/platform makefile when clean, just in case of inconsistency

+            mqueue = mp.Queue()

+            for m in AutoGenObject.GetAllModuleInfo:

+                mqueue.put(m)

+

+            AutoGenObject.DataPipe.DataContainer = {"FfsCommand":FfsCommand}

+            self.Progress.Start("Generating makefile and code")

+            data_pipe_file = os.path.join(AutoGenObject.BuildDir, "GlobalVar_%s_%s.bin" % (str(AutoGenObject.Guid),AutoGenObject.Arch))

+            AutoGenObject.DataPipe.dump(data_pipe_file)

+            autogen_rt,errorcode = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList)

+            self.Progress.Stop("done!")

+            if not autogen_rt:

+                self.AutoGenMgr.TerminateWorkers()

+                self.AutoGenMgr.join(0.1)

+                raise FatalError(errorcode)

             AutoGenObject.CreateCodeFile(False)

             AutoGenObject.CreateMakeFile(False)

+        else:

+            # always recreate top/platform makefile when clean, just in case of inconsistency

+            AutoGenObject.CreateCodeFile(True)

+            AutoGenObject.CreateMakeFile(True)

 

         if EdkLogger.GetLevel() == EdkLogger.QUIET:

             EdkLogger.quiet("Building ... %s" % repr(AutoGenObject))

@@ -1334,8 +1362,8 @@ class Build():
                 return True

         else:

             # always recreate top/platform makefile when clean, just in case of inconsistency

-            AutoGenObject.CreateCodeFile(False)

-            AutoGenObject.CreateMakeFile(False)

+            AutoGenObject.CreateCodeFile(True)

+            AutoGenObject.CreateMakeFile(True)

 

         if EdkLogger.GetLevel() == EdkLogger.QUIET:

             EdkLogger.quiet("Building ... %s" % repr(AutoGenObject))

@@ -1713,9 +1741,10 @@ class Build():
                             continue

                         if Ma.PcdIsDriver:

                             Ma.PlatformInfo = Pa

+                            Ma.Workspace = Wa

                             PcdMaList.append(Ma)

                         self.BuildModules.append(Ma)

-                    self._BuildPa(self.Target, Pa, FfsCommand=CmdListDict)

+                    self._BuildPa(self.Target, Pa, FfsCommand=CmdListDict,PcdMaList=PcdMaList)

 

                 # Create MAP file when Load Fix Address is enabled.

                 if self.Target in ["", "all", "fds"]:

@@ -1850,7 +1879,7 @@ class Build():
                     MakeStart = time.time()

                     for Ma in self.BuildModules:

                         if not Ma.IsBinaryModule:

-                            Bt = BuildTask.New(ModuleMakeUnit(Ma, self.Target))

+                            Bt = BuildTask.New(ModuleMakeUnit(Ma, Pa.BuildCommand,self.Target))

                         # Break build if any build thread has error

                         if BuildTask.HasError():

                             # we need a full version of makefile for platform

@@ -1980,7 +2009,7 @@ class Build():
                 Wa.CreateMakeFile(False)

 

                 # Add ffs build to makefile

-                CmdListDict = None

+                CmdListDict = {}

                 if GlobalData.gEnableGenfdsMultiThread and self.Fdf:

                     CmdListDict = self._GenFfsCmd(Wa.ArchList)

 

@@ -1988,6 +2017,7 @@ class Build():
                 ExitFlag = threading.Event()

                 ExitFlag.clear()

                 self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime)))

+                self.BuildModules = []

                 for Arch in Wa.ArchList:

                     PcdMaList    = []

                     AutoGenStart = time.time()

@@ -2005,6 +2035,8 @@ class Build():
                             if Inf in Pa.Platform.Modules:

                                 continue

                             ModuleList.append(Inf)

+                    Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict}

+                    Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp}

                     for Module in ModuleList:

                         # Get ModuleAutoGen object to generate C code file and makefile

                         Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)

@@ -2013,6 +2045,7 @@ class Build():
                             continue

                         if Ma.PcdIsDriver:

                             Ma.PlatformInfo = Pa

+                            Ma.Workspace = Wa

                             PcdMaList.append(Ma)

                         if Ma.CanSkipbyHash():

                             self.HashSkipModules.append(Ma)

@@ -2024,34 +2057,32 @@ class Build():
                                 EdkLogger.quiet("cache miss: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))

 

                         # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds'

-                        if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:

                             # for target which must generate AutoGen code and makefile

-                            if not self.SkipAutoGen or self.Target == 'genc':

-                                Ma.CreateCodeFile(True)

-                            if self.Target == "genc":

-                                continue

 

-                            if not self.SkipAutoGen or self.Target == 'genmake':

-                                if CmdListDict and self.Fdf and (Module.File, Arch) in CmdListDict:

-                                    Ma.CreateMakeFile(True, CmdListDict[Module.File, Arch])

-                                    del CmdListDict[Module.File, Arch]

-                                else:

-                                    Ma.CreateMakeFile(True)

-                            if self.Target == "genmake":

-                                continue

                         self.BuildModules.append(Ma)

                         # Initialize all modules in tracking to 'FAIL'

                         if Ma.Arch not in GlobalData.gModuleBuildTracking:

                             GlobalData.gModuleBuildTracking[Ma.Arch] = dict()

                         if Ma not in GlobalData.gModuleBuildTracking[Ma.Arch]:

                             GlobalData.gModuleBuildTracking[Ma.Arch][Ma] = 'FAIL'

+                    mqueue = mp.Queue()

+                    for m in Pa.GetAllModuleInfo:

+                        mqueue.put(m)

+                    data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch))

+                    Pa.DataPipe.dump(data_pipe_file)

+                    autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList)

                     self.Progress.Stop("done!")

                     self.AutoGenTime += int(round((time.time() - AutoGenStart)))

+                    if not autogen_rt:

+                        self.AutoGenMgr.TerminateWorkers()

+                        self.AutoGenMgr.join(0.1)

+                        raise FatalError(errorcode)

+                for Arch in Wa.ArchList:

                     MakeStart = time.time()

                     for Ma in self.BuildModules:

                         # Generate build task for the module

                         if not Ma.IsBinaryModule:

-                            Bt = BuildTask.New(ModuleMakeUnit(Ma, self.Target))

+                            Bt = BuildTask.New(ModuleMakeUnit(Ma, Pa.BuildCommand,self.Target))

                         # Break build if any build thread has error

                         if BuildTask.HasError():

                             # we need a full version of makefile for platform

@@ -2498,6 +2529,10 @@ def Main():
             EdkLogger.error(X.ToolName, FORMAT_INVALID, File=X.FileName, Line=X.LineNumber, ExtraData=X.Message, RaiseError=False)

         ReturnCode = FORMAT_INVALID

     except KeyboardInterrupt:

+        if MyBuild is not None:

+

+            # for multi-thread build exits safely

+            MyBuild.Relinquish()

         ReturnCode = ABORT_ERROR

         if Option is not None and Option.debug is not None:

             EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())

@@ -2554,6 +2589,10 @@ def Main():
     return ReturnCode

 

 if __name__ == '__main__':

+    try:

+        mp.set_start_method('spawn')

+    except:

+        pass

     r = Main()

     ## 0-127 is a safe return range, and 1 is a standard default error

     if r < 0 or r > 127: r = 1