BaseTools: Improve the cache hit in the edk2 build cache

BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=1927

Current cache hash algorithm does not parse and generate
the makefile to get the accurate dependency files for a
module. It instead use the platform and package meta files
to get the module depenedency in a quick but over approximate
way. These meta files are monolithic and involve many redundant
dependency for the module, which cause the module build
cache miss easily.
This patch introduces one more cache checkpoint and a new
hash algorithm besides the current quick one. The new hash
algorithm leverages the module makefile to achieve more
accurate and precise dependency info for a module. When
the build cache miss with the first quick hash, the
Basetool will caculate new one after makefile is generated
and then check again.

Cc: Liming Gao <liming.gao@intel.com>
Cc: Bob Feng <bob.c.feng@intel.com>
Signed-off-by: Steven Shi <steven.shi@intel.com>
Reviewed-by: Bob Feng <bob.c.feng@intel.com>

1 files changed, 21 insertions, 0 deletions

diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
index e583828741..a84ed46f2e 100644..100755
--- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
+++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
@@ -182,6 +182,12 @@ class AutoGenWorkerInProcess(mp.Process):
             GlobalData.gDisableIncludePathCheck = False

             GlobalData.gFdfParser = self.data_pipe.Get("FdfParser")

             GlobalData.gDatabasePath = self.data_pipe.Get("DatabasePath")

+            GlobalData.gBinCacheSource = self.data_pipe.Get("BinCacheSource")

+            GlobalData.gBinCacheDest = self.data_pipe.Get("BinCacheDest")

+            GlobalData.gCacheIR = self.data_pipe.Get("CacheIR")

+            GlobalData.gEnableGenfdsMultiThread = self.data_pipe.Get("EnableGenfdsMultiThread")

+            GlobalData.file_lock = self.file_lock

+            CommandTarget = self.data_pipe.Get("CommandTarget")

             pcd_from_build_option = []

             for pcd_tuple in self.data_pipe.Get("BuildOptPcd"):

                 pcd_id = ".".join((pcd_tuple[0],pcd_tuple[1]))

@@ -193,10 +199,13 @@ class AutoGenWorkerInProcess(mp.Process):
             FfsCmd = self.data_pipe.Get("FfsCommand")

             if FfsCmd is None:

                 FfsCmd = {}

+            GlobalData.FfsCmd = FfsCmd

             PlatformMetaFile = self.GetPlatformMetaFile(self.data_pipe.Get("P_Info").get("ActivePlatform"),

                                              self.data_pipe.Get("P_Info").get("WorkspaceDir"))

             libConstPcd = self.data_pipe.Get("LibConstPcd")

             Refes = self.data_pipe.Get("REFS")

+            GlobalData.libConstPcd = libConstPcd

+            GlobalData.Refes = Refes

             while True:

                 if self.module_queue.empty():

                     break

@@ -223,8 +232,20 @@ class AutoGenWorkerInProcess(mp.Process):
                         Ma.ConstPcd = libConstPcd[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]

                     if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in Refes:

                         Ma.ReferenceModules = Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]

+                if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]:

+                    Ma.GenModuleFilesHash(GlobalData.gCacheIR)

+                    Ma.GenPreMakefileHash(GlobalData.gCacheIR)

+                    if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR):

+                       continue

+

                 Ma.CreateCodeFile(False)

                 Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.File, Ma.Arch),[]))

+

+                if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]:

+                    Ma.GenMakeHeaderFilesHash(GlobalData.gCacheIR)

+                    Ma.GenMakeHash(GlobalData.gCacheIR)

+                    if Ma.CanSkipbyMakeCache(GlobalData.gCacheIR):

+                        continue

         except Empty:

             pass

         except: