BaseTools: Leverage compiler output to optimize binary cache

Redesign the binary cache and bases on the compiler to
output the dependency header files info for every module.
The binary cache will directly consume the dependency header
files info and doesn't parse the C source code by iteself.
Also redesign the dependency files list format for module
and try to share the common lib hash result as more as
possible in local process. Remove the unnecessary share data
access across multiprocessing.

Signed-off-by: Steven Shi <steven.shi@intel.com>

Cc: Liming Gao <liming.gao@intel.com>
Cc: Bob Feng <bob.c.feng@intel.com>
Reviewed-by: Bob Feng <bob.c.feng@intel.com>

1 files changed, 55 insertions, 22 deletions

diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
index 94ea61a487..40b448f5b2 100755
--- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
+++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
@@ -128,12 +128,27 @@ class AutoGenManager(threading.Thread):
         clearQ(taskq)

         clearQ(self.feedback_q)

         clearQ(logq)

+        # Copy the cache queue itmes to parent thread before clear

+        cacheq = self.autogen_workers[0].cache_q

+        try:

+            cache_num = 0

+            while True:

+                item = cacheq.get()

+                if item == "CacheDone":

+                    cache_num += 1

+                else:

+                    GlobalData.gModuleAllCacheStatus.add(item)

+                if cache_num  == len(self.autogen_workers):

+                    break

+        except:

+            print ("cache_q error")

+

     def TerminateWorkers(self):

         self.error_event.set()

     def kill(self):

         self.feedback_q.put(None)

 class AutoGenWorkerInProcess(mp.Process):

-    def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_lock,cache_lock,share_data,log_q,error_event):

+    def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_lock,cache_q,log_q,error_event):

         mp.Process.__init__(self)

         self.module_queue = module_queue

         self.data_pipe_file_path =data_pipe_file_path

@@ -141,8 +156,7 @@ class AutoGenWorkerInProcess(mp.Process):
         self.feedback_q = feedback_q

         self.PlatformMetaFileSet = {}

         self.file_lock = file_lock

-        self.cache_lock = cache_lock

-        self.share_data = share_data

+        self.cache_q = cache_q

         self.log_q = log_q

         self.error_event = error_event

     def GetPlatformMetaFile(self,filepath,root):

@@ -184,12 +198,19 @@ class AutoGenWorkerInProcess(mp.Process):
             GlobalData.gDisableIncludePathCheck = False

             GlobalData.gFdfParser = self.data_pipe.Get("FdfParser")

             GlobalData.gDatabasePath = self.data_pipe.Get("DatabasePath")

+

+            GlobalData.gUseHashCache = self.data_pipe.Get("UseHashCache")

             GlobalData.gBinCacheSource = self.data_pipe.Get("BinCacheSource")

             GlobalData.gBinCacheDest = self.data_pipe.Get("BinCacheDest")

-            GlobalData.gCacheIR = self.share_data

+            GlobalData.gPlatformHashFile = self.data_pipe.Get("PlatformHashFile")

+            GlobalData.gModulePreMakeCacheStatus = dict()

+            GlobalData.gModuleMakeCacheStatus = dict()

+            GlobalData.gHashChainStatus = dict()

+            GlobalData.gCMakeHashFile = dict()

+            GlobalData.gModuleHashFile = dict()

+            GlobalData.gFileHashDict = dict()

             GlobalData.gEnableGenfdsMultiThread = self.data_pipe.Get("EnableGenfdsMultiThread")

             GlobalData.file_lock = self.file_lock

-            GlobalData.cache_lock = self.cache_lock

             CommandTarget = self.data_pipe.Get("CommandTarget")

             pcd_from_build_option = []

             for pcd_tuple in self.data_pipe.Get("BuildOptPcd"):

@@ -205,10 +226,6 @@ class AutoGenWorkerInProcess(mp.Process):
             GlobalData.FfsCmd = FfsCmd

             PlatformMetaFile = self.GetPlatformMetaFile(self.data_pipe.Get("P_Info").get("ActivePlatform"),

                                              self.data_pipe.Get("P_Info").get("WorkspaceDir"))

-            libConstPcd = self.data_pipe.Get("LibConstPcd")

-            Refes = self.data_pipe.Get("REFS")

-            GlobalData.libConstPcd = libConstPcd

-            GlobalData.Refes = Refes

             while True:

                 if self.module_queue.empty():

                     break

@@ -230,27 +247,41 @@ class AutoGenWorkerInProcess(mp.Process):
                 toolchain = self.data_pipe.Get("P_Info").get("ToolChain")

                 Ma = ModuleAutoGen(self.Wa,module_metafile,target,toolchain,arch,PlatformMetaFile,self.data_pipe)

                 Ma.IsLibrary = IsLib

-                if IsLib:

-                    if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in libConstPcd:

-                        Ma.ConstPcd = libConstPcd[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]

-                    if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in Refes:

-                        Ma.ReferenceModules = Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]

-                if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]:

-                    Ma.GenModuleFilesHash(GlobalData.gCacheIR)

-                    Ma.GenPreMakefileHash(GlobalData.gCacheIR)

-                    if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR):

+                # SourceFileList calling sequence impact the makefile string sequence.

+                # Create cached SourceFileList here to unify its calling sequence for both

+                # CanSkipbyPreMakeCache and CreateCodeFile/CreateMakeFile.

+                RetVal = Ma.SourceFileList

+                if GlobalData.gUseHashCache and not GlobalData.gBinCacheDest and CommandTarget in [None, "", "all"]:

+                    try:

+                        CacheResult = Ma.CanSkipbyPreMakeCache()

+                    except:

+                        CacheResult = False

+                        traceback.print_exc(file=sys.stdout)

+                        self.feedback_q.put(taskname)

+

+                    if CacheResult:

+                        self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "PreMakeCache", True))

                         continue

+                    else:

+                        self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "PreMakeCache", False))

 

                 Ma.CreateCodeFile(False)

                 Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.Path, Ma.Arch),[]))

 

                 if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]:

-                    Ma.GenMakeHeaderFilesHash(GlobalData.gCacheIR)

-                    Ma.GenMakeHash(GlobalData.gCacheIR)

-                    if Ma.CanSkipbyMakeCache(GlobalData.gCacheIR):

+                    try:

+                        CacheResult = Ma.CanSkipbyMakeCache()

+                    except:

+                        CacheResult = False

+                        traceback.print_exc(file=sys.stdout)

+                        self.feedback_q.put(taskname)

+

+                    if CacheResult:

+                        self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "MakeCache", True))

                         continue

                     else:

-                        Ma.PrintFirstMakeCacheMissFile(GlobalData.gCacheIR)

+                        self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "MakeCache", False))

+

         except Empty:

             pass

         except:

@@ -258,6 +289,8 @@ class AutoGenWorkerInProcess(mp.Process):
             self.feedback_q.put(taskname)

         finally:

             self.feedback_q.put("Done")

+            self.cache_q.put("CacheDone")

+

     def printStatus(self):

         print("Processs ID: %d Run %d modules in AutoGen " % (os.getpid(),len(AutoGen.Cache())))

         print("Processs ID: %d Run %d modules in AutoGenInfo " % (os.getpid(),len(AutoGenInfo.GetCache())))