BaseTools: Enable Multiple Process AutoGen

BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=1875

Assign the Module AutoGen tasks into multiple
sub process.

Cc: Liming Gao <liming.gao@intel.com>
Signed-off-by: Bob Feng <bob.c.feng@intel.com>
Acked-by: Laszlo Ersek <lersek@redhat.com>
Tested-by: Laszlo Ersek <lersek@redhat.com>
Acked-by: Liming Gao <liming.gao@intel.com>

7 files changed, 307 insertions, 56 deletions

diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
new file mode 100644
index 0000000000..6c6ef49647
--- /dev/null
+++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
@@ -0,0 +1,191 @@
+## @file

+# Create makefile for MS nmake and GNU make

+#

+# Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>

+# SPDX-License-Identifier: BSD-2-Clause-Patent

+#

+from __future__ import absolute_import

+import multiprocessing as mp

+import threading

+from Common.Misc import PathClass

+from AutoGen.ModuleAutoGen import ModuleAutoGen

+from AutoGen.ModuleAutoGenHelper import WorkSpaceInfo,AutoGenInfo

+import Common.GlobalData as GlobalData

+import Common.EdkLogger as EdkLogger

+import os

+from Common.MultipleWorkspace import MultipleWorkspace as mws

+from AutoGen.AutoGen import AutoGen

+from Workspace.WorkspaceDatabase import BuildDB

+from queue import Empty

+import traceback

+import sys

+from AutoGen.DataPipe import MemoryDataPipe

+def clearQ(q):

+    try:

+        while True:

+            q.get_nowait()

+    except Empty:

+        pass

+class AutoGenManager(threading.Thread):

+    def __init__(self,autogen_workers, feedback_q,error_event):

+        super(AutoGenManager,self).__init__()

+        self.autogen_workers = autogen_workers

+        self.feedback_q = feedback_q

+        self.terminate = False

+        self.Status = True

+        self.error_event = error_event

+    def run(self):

+        try:

+            fin_num = 0

+            while True:

+                badnews = self.feedback_q.get()

+                if badnews is None:

+                    break

+                if badnews == "Done":

+                    fin_num += 1

+                else:

+                    self.Status = False

+                    self.TerminateWorkers()

+                if fin_num == len(self.autogen_workers):

+                    self.clearQueue()

+                    for w in self.autogen_workers:

+                        w.join()

+                    break

+        except Exception:

+            return

+

+    def clearQueue(self):

+        taskq = self.autogen_workers[0].module_queue

+        clearQ(taskq)

+        clearQ(self.feedback_q)

+

+    def TerminateWorkers(self):

+        self.error_event.set()

+    def kill(self):

+        self.feedback_q.put(None)

+class AutoGenWorkerInProcess(mp.Process):

+    def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_lock,error_event):

+        mp.Process.__init__(self)

+        self.module_queue = module_queue

+        self.data_pipe_file_path =data_pipe_file_path

+        self.data_pipe = None

+        self.feedback_q = feedback_q

+        self.PlatformMetaFileSet = {}

+        self.file_lock = file_lock

+        self.error_event = error_event

+    def GetPlatformMetaFile(self,filepath,root):

+        try:

+            return self.PlatformMetaFileSet[(filepath,root)]

+        except:

+            self.PlatformMetaFileSet[(filepath,root)]  = filepath

+            return self.PlatformMetaFileSet[(filepath,root)]

+    def run(self):

+        try:

+            taskname = "Init"

+            with self.file_lock:

+                if not os.path.exists(self.data_pipe_file_path):

+                    self.feedback_q.put(taskname + ":" + "load data pipe %s failed." % self.data_pipe_file_path)

+                self.data_pipe = MemoryDataPipe()

+                self.data_pipe.load(self.data_pipe_file_path)

+            EdkLogger.Initialize()

+            loglevel = self.data_pipe.Get("LogLevel")

+            if not loglevel:

+                loglevel = EdkLogger.INFO

+            EdkLogger.SetLevel(loglevel)

+            logfile = self.data_pipe.Get("LogFile")

+            if logfile:

+                EdkLogger.SetLogFile(logfile)

+            target = self.data_pipe.Get("P_Info").get("Target")

+            toolchain = self.data_pipe.Get("P_Info").get("ToolChain")

+            archlist = self.data_pipe.Get("P_Info").get("ArchList")

+

+            active_p = self.data_pipe.Get("P_Info").get("ActivePlatform")

+            workspacedir = self.data_pipe.Get("P_Info").get("WorkspaceDir")

+            PackagesPath = os.getenv("PACKAGES_PATH")

+            mws.setWs(workspacedir, PackagesPath)

+            self.Wa = WorkSpaceInfo(

+                workspacedir,active_p,target,toolchain,archlist

+                )

+            self.Wa._SrcTimeStamp = self.data_pipe.Get("Workspace_timestamp")

+            GlobalData.gGlobalDefines = self.data_pipe.Get("G_defines")

+            GlobalData.gCommandLineDefines = self.data_pipe.Get("CL_defines")

+            os.environ._data = self.data_pipe.Get("Env_Var")

+            GlobalData.gWorkspace = workspacedir

+            GlobalData.gDisableIncludePathCheck = False

+            GlobalData.gFdfParser = self.data_pipe.Get("FdfParser")

+            GlobalData.gDatabasePath = self.data_pipe.Get("DatabasePath")

+            pcd_from_build_option = []

+            for pcd_tuple in self.data_pipe.Get("BuildOptPcd"):

+                pcd_id = ".".join((pcd_tuple[0],pcd_tuple[1]))

+                if pcd_tuple[2].strip():

+                    pcd_id = ".".join((pcd_id,pcd_tuple[2]))

+                pcd_from_build_option.append("=".join((pcd_id,pcd_tuple[3])))

+            GlobalData.BuildOptionPcd = pcd_from_build_option

+            module_count = 0

+            FfsCmd = self.data_pipe.Get("FfsCommand")

+            if FfsCmd is None:

+                FfsCmd = {}

+            PlatformMetaFile = self.GetPlatformMetaFile(self.data_pipe.Get("P_Info").get("ActivePlatform"),

+                                             self.data_pipe.Get("P_Info").get("WorkspaceDir"))

+            libConstPcd = self.data_pipe.Get("LibConstPcd")

+            Refes = self.data_pipe.Get("REFS")

+            while True:

+                if self.module_queue.empty():

+                    break

+                if self.error_event.is_set():

+                    break

+                module_count += 1

+                module_file,module_root,module_path,module_basename,module_originalpath,module_arch,IsLib = self.module_queue.get_nowait()

+                modulefullpath = os.path.join(module_root,module_file)

+                taskname = " : ".join((modulefullpath,module_arch))

+                module_metafile = PathClass(module_file,module_root)

+                if module_path:

+                    module_metafile.Path = module_path

+                if module_basename:

+                    module_metafile.BaseName = module_basename

+                if module_originalpath:

+                    module_metafile.OriginalPath = PathClass(module_originalpath,module_root)

+                arch = module_arch

+                target = self.data_pipe.Get("P_Info").get("Target")

+                toolchain = self.data_pipe.Get("P_Info").get("ToolChain")

+                Ma = ModuleAutoGen(self.Wa,module_metafile,target,toolchain,arch,PlatformMetaFile,self.data_pipe)

+                Ma.IsLibrary = IsLib

+                if IsLib:

+                    if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in libConstPcd:

+                        Ma.ConstPcd = libConstPcd[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]

+                    if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in Refes:

+                        Ma.ReferenceModules = Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]

+                Ma.CreateCodeFile(False)

+                Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.File, Ma.Arch),[]))

+        except Empty:

+            pass

+        except:

+            traceback.print_exc(file=sys.stdout)

+            self.feedback_q.put(taskname)

+        finally:

+            self.feedback_q.put("Done")

+    def printStatus(self):

+        print("Processs ID: %d Run %d modules in AutoGen " % (os.getpid(),len(AutoGen.Cache())))

+        print("Processs ID: %d Run %d modules in AutoGenInfo " % (os.getpid(),len(AutoGenInfo.GetCache())))

+        groupobj = {}

+        for buildobj in BuildDB.BuildObject.GetCache().values():

+            if str(buildobj).lower().endswith("dec"):

+                try:

+                    groupobj['dec'].append(str(buildobj))

+                except:

+                    groupobj['dec'] = [str(buildobj)]

+            if str(buildobj).lower().endswith("dsc"):

+                try:

+                    groupobj['dsc'].append(str(buildobj))

+                except:

+                    groupobj['dsc'] = [str(buildobj)]

+

+            if str(buildobj).lower().endswith("inf"):

+                try:

+                    groupobj['inf'].append(str(buildobj))

+                except:

+                    groupobj['inf'] = [str(buildobj)]

+

+        print("Processs ID: %d Run %d pkg in WDB " % (os.getpid(),len(groupobj.get("dec",[]))))

+        print("Processs ID: %d Run %d pla in WDB " % (os.getpid(),len(groupobj.get("dsc",[]))))

+        print("Processs ID: %d Run %d inf in WDB " % (os.getpid(),len(groupobj.get("inf",[]))))

diff --git a/BaseTools/Source/Python/AutoGen/DataPipe.py b/BaseTools/Source/Python/AutoGen/DataPipe.py
index 5bcc39bd38..6299208056 100644
--- a/BaseTools/Source/Python/AutoGen/DataPipe.py
+++ b/BaseTools/Source/Python/AutoGen/DataPipe.py
@@ -11,6 +11,7 @@ import Common.GlobalData as GlobalData
 import os

 import pickle

 from pickle import HIGHEST_PROTOCOL

+from Common import EdkLogger

 

 class PCD_DATA():

     def __init__(self,TokenCName,TokenSpaceGuidCName,Type,DatumType,SkuInfoList,DefaultValue,

@@ -34,6 +35,7 @@ class DataPipe(object):
     def __init__(self, BuildDir=None):

         self.data_container = {}

         self.BuildDir = BuildDir

+        self.dump_file = ""

 

 class MemoryDataPipe(DataPipe):

 

@@ -41,6 +43,7 @@ class MemoryDataPipe(DataPipe):
         return self.data_container.get(key)

 

     def dump(self,file_path):

+        self.dump_file = file_path

         with open(file_path,'wb') as fd:

             pickle.dump(self.data_container,fd,pickle.HIGHEST_PROTOCOL)

 

@@ -71,7 +74,7 @@ class MemoryDataPipe(DataPipe):
         for m in PlatformInfo.Platform.Modules:

             m_pcds =  PlatformInfo.Platform.Modules[m].Pcds

             if m_pcds:

-                ModulePcds[(m.File,m.Root)] = [PCD_DATA(

+                ModulePcds[(m.File,m.Root,m.Arch)] = [PCD_DATA(

             pcd.TokenCName,pcd.TokenSpaceGuidCName,pcd.Type,

             pcd.DatumType,pcd.SkuInfoList,pcd.DefaultValue,

             pcd.MaxDatumSize,pcd.UserDefinedDefaultStoresFlag,pcd.validateranges,

@@ -83,11 +86,18 @@ class MemoryDataPipe(DataPipe):
 

         #Module's Library Instance

         ModuleLibs = {}

+        libModules = {}

         for m in PlatformInfo.Platform.Modules:

             module_obj = BuildDB.BuildObject[m,PlatformInfo.Arch,PlatformInfo.BuildTarget,PlatformInfo.ToolChain]

             Libs = GetModuleLibInstances(module_obj, PlatformInfo.Platform, BuildDB.BuildObject, PlatformInfo.Arch,PlatformInfo.BuildTarget,PlatformInfo.ToolChain)

-            ModuleLibs[(m.File,m.Root,module_obj.Arch)] = [(l.MetaFile.File,l.MetaFile.Root,l.Arch) for l in Libs]

+            for lib in Libs:

+                try:

+                    libModules[(lib.MetaFile.File,lib.MetaFile.Root,lib.Arch,lib.MetaFile.Path)].append((m.File,m.Root,module_obj.Arch,m.Path))

+                except:

+                    libModules[(lib.MetaFile.File,lib.MetaFile.Root,lib.Arch,lib.MetaFile.Path)] = [(m.File,m.Root,module_obj.Arch,m.Path)]

+            ModuleLibs[(m.File,m.Root,module_obj.Arch,m.Path)] = [(l.MetaFile.File,l.MetaFile.Root,l.Arch,l.MetaFile.Path) for l in Libs]

         self.DataContainer = {"DEPS":ModuleLibs}

+        self.DataContainer = {"REFS":libModules}

 

         #Platform BuildOptions

 

@@ -143,5 +153,8 @@ class MemoryDataPipe(DataPipe):
 

         self.DataContainer = {"GuidDict": PlatformInfo.Platform._GuidDict}

 

+        self.DataContainer = {"DatabasePath":GlobalData.gDatabasePath}

         self.DataContainer = {"FdfParser": True if GlobalData.gFdfParser else False}

 

+        self.DataContainer = {"LogLevel": EdkLogger.GetLevel()}

+        self.DataContainer = {"LogFile": GlobalData.gOptions.LogFile if GlobalData.gOptions.LogFile is not None else ""}

diff --git a/BaseTools/Source/Python/AutoGen/GenC.py b/BaseTools/Source/Python/AutoGen/GenC.py
index 4c3f4e3e55..910c8fe370 100644
--- a/BaseTools/Source/Python/AutoGen/GenC.py
+++ b/BaseTools/Source/Python/AutoGen/GenC.py
@@ -1472,8 +1472,8 @@ def CreateModuleEntryPointCode(Info, AutoGenC, AutoGenH):
 

     if Info.ModuleType in [SUP_MODULE_PEI_CORE, SUP_MODULE_DXE_CORE, SUP_MODULE_SMM_CORE, SUP_MODULE_MM_CORE_STANDALONE]:

         if Info.SourceFileList:

-          if NumEntryPoints != 1:

-              EdkLogger.error(

+            if NumEntryPoints != 1:

+                EdkLogger.error(

                   "build",

                   AUTOGEN_ERROR,

                   '%s must have exactly one entry point' % Info.ModuleType,

diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
index d19c038620..ed6822334e 100644
--- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
@@ -1686,6 +1686,7 @@ class ModuleAutoGen(AutoGen):
         if not self.IsLibrary and CreateLibraryMakeFile:

             for LibraryAutoGen in self.LibraryAutoGenList:

                 LibraryAutoGen.CreateMakeFile()

+

         # Don't enable if hash feature enabled, CanSkip uses timestamps to determine build skipping

         if not GlobalData.gUseHashCache and self.CanSkip():

             return

@@ -1729,7 +1730,6 @@ class ModuleAutoGen(AutoGen):
         if not self.IsLibrary and CreateLibraryCodeFile:

             for LibraryAutoGen in self.LibraryAutoGenList:

                 LibraryAutoGen.CreateCodeFile()

-

         # Don't enable if hash feature enabled, CanSkip uses timestamps to determine build skipping

         if not GlobalData.gUseHashCache and self.CanSkip():

             return

diff --git a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py
index 6c947eca2b..4abfc6c29d 100644
--- a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py
@@ -133,6 +133,12 @@ class PlatformAutoGen(AutoGen):
         self.DataPipe.FillData(self)

 

         return True

+    def FillData_LibConstPcd(self):

+        libConstPcd = {}

+        for LibAuto in self.LibraryAutoGenList:

+            if LibAuto.ConstPcd:

+                libConstPcd[(LibAuto.MetaFile.File,LibAuto.MetaFile.Root,LibAuto.Arch,LibAuto.MetaFile.Path)] = LibAuto.ConstPcd

+        self.DataPipe.DataContainer = {"LibConstPcd":libConstPcd}

     ## hash() operator of PlatformAutoGen

     #

     #  The platform file path and arch string will be used to represent

@@ -162,7 +168,7 @@ class PlatformAutoGen(AutoGen):
             return

 

         for Ma in self.ModuleAutoGenList:

-            Ma.CreateCodeFile(True)

+            Ma.CreateCodeFile(CreateModuleCodeFile)

 

     ## Generate Fds Command

     @cached_property

@@ -179,9 +185,9 @@ class PlatformAutoGen(AutoGen):
             for Ma in self._MaList:

                 key = (Ma.MetaFile.File, self.Arch)

                 if key in FfsCommand:

-                    Ma.CreateMakeFile(True, FfsCommand[key])

+                    Ma.CreateMakeFile(CreateModuleMakeFile, FfsCommand[key])

                 else:

-                    Ma.CreateMakeFile(True)

+                    Ma.CreateMakeFile(CreateModuleMakeFile)

 

         # no need to create makefile for the platform more than once

         if self.IsMakeFileCreated:

@@ -1086,10 +1092,10 @@ class PlatformAutoGen(AutoGen):
                 Libs = GetModuleLibInstances(module_obj, self.Platform, self.BuildDatabase, self.Arch,self.BuildTarget,self.ToolChain)

             else:

                 Libs = []

-            ModuleLibs.update( set([(l.MetaFile.File,l.MetaFile.Root,l.Arch,True) for l in Libs]))

+            ModuleLibs.update( set([(l.MetaFile.File,l.MetaFile.Root,l.MetaFile.Path,l.MetaFile.BaseName,l.MetaFile.OriginalPath,l.Arch,True) for l in Libs]))

             if WithoutPcd and module_obj.PcdIsDriver:

                 continue

-            ModuleLibs.add((m.File,m.Root,module_obj.Arch,False))

+            ModuleLibs.add((m.File,m.Root,m.Path,m.BaseName,m.OriginalPath,module_obj.Arch,bool(module_obj.LibraryClass)))

 

         return ModuleLibs

 

diff --git a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
index 22a7d996fd..ea0d8f8bfb 100644
--- a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
@@ -113,6 +113,8 @@ class WorkspaceAutoGen(AutoGen):
         self.ProcessMixedPcd()

         self.VerifyPcdsFromFDF()

         self.CollectAllPcds()

+        for Pa in self.AutoGenObjectList:

+            Pa.FillData_LibConstPcd()

         self.GeneratePkgLevelHash()

         #

         # Check PCDs token value conflict in each DEC file.

@@ -881,7 +883,7 @@ class WorkspaceAutoGen(AutoGen):
         if not CreateDepsMakeFile:

             return

         for Pa in self.AutoGenObjectList:

-            Pa.CreateMakeFile(True)

+            Pa.CreateMakeFile(CreateDepsMakeFile)

 

     ## Create autogen code for platform and modules

     #

@@ -895,7 +897,7 @@ class WorkspaceAutoGen(AutoGen):
         if not CreateDepsCodeFile:

             return

         for Pa in self.AutoGenObjectList:

-            Pa.CreateCodeFile(True)

+            Pa.CreateCodeFile(CreateDepsCodeFile)

 

     ## Create AsBuilt INF file the platform

     #

diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py
index 3d083f4eaa..af1be8d281 100644
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -30,6 +30,7 @@ from optparse import OptionParser
 from AutoGen.PlatformAutoGen import PlatformAutoGen

 from AutoGen.ModuleAutoGen import ModuleAutoGen

 from AutoGen.WorkspaceAutoGen import WorkspaceAutoGen

+from AutoGen.AutoGenWorker import AutoGenWorkerInProcess,AutoGenManager

 from AutoGen import GenMake

 from Common import Misc as Utils

 

@@ -50,7 +51,7 @@ from PatchPcdValue.PatchPcdValue import PatchBinaryFile
 

 import Common.GlobalData as GlobalData

 from GenFds.GenFds import GenFds, GenFdsApi

-

+import multiprocessing as mp

 

 # Version and Copyright

 VersionNumber = "0.60" + ' ' + gBUILD_VERSION

@@ -343,9 +344,9 @@ class ModuleMakeUnit(BuildUnit):
     #   @param  Obj         The ModuleAutoGen object the build is working on

     #   @param  Target      The build target name, one of gSupportedTarget

     #

-    def __init__(self, Obj, Target):

-        Dependency = [ModuleMakeUnit(La, Target) for La in Obj.LibraryAutoGenList]

-        BuildUnit.__init__(self, Obj, Obj.BuildCommand, Target, Dependency, Obj.MakeFileDir)

+    def __init__(self, Obj, BuildCommand,Target):

+        Dependency = [ModuleMakeUnit(La, BuildCommand,Target) for La in Obj.LibraryAutoGenList]

+        BuildUnit.__init__(self, Obj, BuildCommand, Target, Dependency, Obj.MakeFileDir)

         if Target in [None, "", "all"]:

             self.Target = "tbuild"

 

@@ -364,10 +365,10 @@ class PlatformMakeUnit(BuildUnit):
     #   @param  Obj         The PlatformAutoGen object the build is working on

     #   @param  Target      The build target name, one of gSupportedTarget

     #

-    def __init__(self, Obj, Target):

-        Dependency = [ModuleMakeUnit(Lib, Target) for Lib in self.BuildObject.LibraryAutoGenList]

-        Dependency.extend([ModuleMakeUnit(Mod, Target) for Mod in self.BuildObject.ModuleAutoGenList])

-        BuildUnit.__init__(self, Obj, Obj.BuildCommand, Target, Dependency, Obj.MakeFileDir)

+    def __init__(self, Obj, BuildCommand, Target):

+        Dependency = [ModuleMakeUnit(Lib, BuildCommand, Target) for Lib in self.BuildObject.LibraryAutoGenList]

+        Dependency.extend([ModuleMakeUnit(Mod, BuildCommand,Target) for Mod in self.BuildObject.ModuleAutoGenList])

+        BuildUnit.__init__(self, Obj, BuildCommand, Target, Dependency, Obj.MakeFileDir)

 

 ## The class representing the task of a module build or platform build

 #

@@ -824,8 +825,31 @@ class Build():
         if not (self.LaunchPrebuildFlag and os.path.exists(self.PlatformBuildPath)):

             self.InitBuild()

 

+        self.AutoGenMgr = None

         EdkLogger.info("")

         os.chdir(self.WorkspaceDir)

+    def StartAutoGen(self,mqueue, DataPipe,SkipAutoGen,PcdMaList):

+        try:

+            if SkipAutoGen:

+                return True,0

+            feedback_q = mp.Queue()

+            file_lock = mp.Lock()

+            error_event = mp.Event()

+            auto_workers = [AutoGenWorkerInProcess(mqueue,DataPipe.dump_file,feedback_q,file_lock,error_event) for _ in range(self.ThreadNumber)]

+            self.AutoGenMgr = AutoGenManager(auto_workers,feedback_q,error_event)

+            self.AutoGenMgr.start()

+            for w in auto_workers:

+                w.start()

+            if PcdMaList is not None:

+                for PcdMa in PcdMaList:

+                    PcdMa.CreateCodeFile(False)

+                    PcdMa.CreateMakeFile(False,GenFfsList = DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.File, PcdMa.Arch),[]))

+

+            self.AutoGenMgr.join()

+            rt = self.AutoGenMgr.Status

+            return rt, 0

+        except Exception as e:

+            return False,e.errcode

 

     ## Load configuration

     #

@@ -1190,30 +1214,34 @@ class Build():
     #   @param  CreateDepModuleMakeFile     Flag used to indicate creating makefile

     #                                       for dependent modules/Libraries

     #

-    def _BuildPa(self, Target, AutoGenObject, CreateDepsCodeFile=True, CreateDepsMakeFile=True, BuildModule=False, FfsCommand={}):

+    def _BuildPa(self, Target, AutoGenObject, CreateDepsCodeFile=True, CreateDepsMakeFile=True, BuildModule=False, FfsCommand=None, PcdMaList=None):

         if AutoGenObject is None:

             return False

-

+        if FfsCommand is None:

+            FfsCommand = {}

         # skip file generation for cleanxxx targets, run and fds target

         if Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:

             # for target which must generate AutoGen code and makefile

-            if not self.SkipAutoGen or Target == 'genc':

-                self.Progress.Start("Generating code")

-                AutoGenObject.CreateCodeFile(CreateDepsCodeFile)

-                self.Progress.Stop("done!")

-            if Target == "genc":

-                return True

-

-            if not self.SkipAutoGen or Target == 'genmake':

-                self.Progress.Start("Generating makefile")

-                AutoGenObject.CreateMakeFile(CreateDepsMakeFile, FfsCommand)

-                self.Progress.Stop("done!")

-            if Target == "genmake":

-                return True

-        else:

-            # always recreate top/platform makefile when clean, just in case of inconsistency

+            mqueue = mp.Queue()

+            for m in AutoGenObject.GetAllModuleInfo:

+                mqueue.put(m)

+

+            AutoGenObject.DataPipe.DataContainer = {"FfsCommand":FfsCommand}

+            self.Progress.Start("Generating makefile and code")

+            data_pipe_file = os.path.join(AutoGenObject.BuildDir, "GlobalVar_%s_%s.bin" % (str(AutoGenObject.Guid),AutoGenObject.Arch))

+            AutoGenObject.DataPipe.dump(data_pipe_file)

+            autogen_rt,errorcode = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList)

+            self.Progress.Stop("done!")

+            if not autogen_rt:

+                self.AutoGenMgr.TerminateWorkers()

+                self.AutoGenMgr.join(0.1)

+                raise FatalError(errorcode)

             AutoGenObject.CreateCodeFile(False)

             AutoGenObject.CreateMakeFile(False)

+        else:

+            # always recreate top/platform makefile when clean, just in case of inconsistency

+            AutoGenObject.CreateCodeFile(True)

+            AutoGenObject.CreateMakeFile(True)

 

         if EdkLogger.GetLevel() == EdkLogger.QUIET:

             EdkLogger.quiet("Building ... %s" % repr(AutoGenObject))

@@ -1334,8 +1362,8 @@ class Build():
                 return True

         else:

             # always recreate top/platform makefile when clean, just in case of inconsistency

-            AutoGenObject.CreateCodeFile(False)

-            AutoGenObject.CreateMakeFile(False)

+            AutoGenObject.CreateCodeFile(True)

+            AutoGenObject.CreateMakeFile(True)

 

         if EdkLogger.GetLevel() == EdkLogger.QUIET:

             EdkLogger.quiet("Building ... %s" % repr(AutoGenObject))

@@ -1713,9 +1741,10 @@ class Build():
                             continue

                         if Ma.PcdIsDriver:

                             Ma.PlatformInfo = Pa

+                            Ma.Workspace = Wa

                             PcdMaList.append(Ma)

                         self.BuildModules.append(Ma)

-                    self._BuildPa(self.Target, Pa, FfsCommand=CmdListDict)

+                    self._BuildPa(self.Target, Pa, FfsCommand=CmdListDict,PcdMaList=PcdMaList)

 

                 # Create MAP file when Load Fix Address is enabled.

                 if self.Target in ["", "all", "fds"]:

@@ -1850,7 +1879,7 @@ class Build():
                     MakeStart = time.time()

                     for Ma in self.BuildModules:

                         if not Ma.IsBinaryModule:

-                            Bt = BuildTask.New(ModuleMakeUnit(Ma, self.Target))

+                            Bt = BuildTask.New(ModuleMakeUnit(Ma, Pa.BuildCommand,self.Target))

                         # Break build if any build thread has error

                         if BuildTask.HasError():

                             # we need a full version of makefile for platform

@@ -1980,7 +2009,7 @@ class Build():
                 Wa.CreateMakeFile(False)

 

                 # Add ffs build to makefile

-                CmdListDict = None

+                CmdListDict = {}

                 if GlobalData.gEnableGenfdsMultiThread and self.Fdf:

                     CmdListDict = self._GenFfsCmd(Wa.ArchList)

 

@@ -1988,6 +2017,7 @@ class Build():
                 ExitFlag = threading.Event()

                 ExitFlag.clear()

                 self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime)))

+                self.BuildModules = []

                 for Arch in Wa.ArchList:

                     PcdMaList    = []

                     AutoGenStart = time.time()

@@ -2005,6 +2035,8 @@ class Build():
                             if Inf in Pa.Platform.Modules:

                                 continue

                             ModuleList.append(Inf)

+                    Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict}

+                    Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp}

                     for Module in ModuleList:

                         # Get ModuleAutoGen object to generate C code file and makefile

                         Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)

@@ -2013,6 +2045,7 @@ class Build():
                             continue

                         if Ma.PcdIsDriver:

                             Ma.PlatformInfo = Pa

+                            Ma.Workspace = Wa

                             PcdMaList.append(Ma)

                         if Ma.CanSkipbyHash():

                             self.HashSkipModules.append(Ma)

@@ -2024,34 +2057,32 @@ class Build():
                                 EdkLogger.quiet("cache miss: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))

 

                         # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds'

-                        if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:

                             # for target which must generate AutoGen code and makefile

-                            if not self.SkipAutoGen or self.Target == 'genc':

-                                Ma.CreateCodeFile(True)

-                            if self.Target == "genc":

-                                continue

 

-                            if not self.SkipAutoGen or self.Target == 'genmake':

-                                if CmdListDict and self.Fdf and (Module.File, Arch) in CmdListDict:

-                                    Ma.CreateMakeFile(True, CmdListDict[Module.File, Arch])

-                                    del CmdListDict[Module.File, Arch]

-                                else:

-                                    Ma.CreateMakeFile(True)

-                            if self.Target == "genmake":

-                                continue

                         self.BuildModules.append(Ma)

                         # Initialize all modules in tracking to 'FAIL'

                         if Ma.Arch not in GlobalData.gModuleBuildTracking:

                             GlobalData.gModuleBuildTracking[Ma.Arch] = dict()

                         if Ma not in GlobalData.gModuleBuildTracking[Ma.Arch]:

                             GlobalData.gModuleBuildTracking[Ma.Arch][Ma] = 'FAIL'

+                    mqueue = mp.Queue()

+                    for m in Pa.GetAllModuleInfo:

+                        mqueue.put(m)

+                    data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch))

+                    Pa.DataPipe.dump(data_pipe_file)

+                    autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList)

                     self.Progress.Stop("done!")

                     self.AutoGenTime += int(round((time.time() - AutoGenStart)))

+                    if not autogen_rt:

+                        self.AutoGenMgr.TerminateWorkers()

+                        self.AutoGenMgr.join(0.1)

+                        raise FatalError(errorcode)

+                for Arch in Wa.ArchList:

                     MakeStart = time.time()

                     for Ma in self.BuildModules:

                         # Generate build task for the module

                         if not Ma.IsBinaryModule:

-                            Bt = BuildTask.New(ModuleMakeUnit(Ma, self.Target))

+                            Bt = BuildTask.New(ModuleMakeUnit(Ma, Pa.BuildCommand,self.Target))

                         # Break build if any build thread has error

                         if BuildTask.HasError():

                             # we need a full version of makefile for platform

@@ -2498,6 +2529,10 @@ def Main():
             EdkLogger.error(X.ToolName, FORMAT_INVALID, File=X.FileName, Line=X.LineNumber, ExtraData=X.Message, RaiseError=False)

         ReturnCode = FORMAT_INVALID

     except KeyboardInterrupt:

+        if MyBuild is not None:

+

+            # for multi-thread build exits safely

+            MyBuild.Relinquish()

         ReturnCode = ABORT_ERROR

         if Option is not None and Option.debug is not None:

             EdkLogger.quiet("(Python %s on %s) " % (platform.python_version(), sys.platform) + traceback.format_exc())

@@ -2554,6 +2589,10 @@ def Main():
     return ReturnCode

 

 if __name__ == '__main__':

+    try:

+        mp.set_start_method('spawn')

+    except:

+        pass

     r = Main()

     ## 0-127 is a safe return range, and 1 is a standard default error

     if r < 0 or r > 127: r = 1