From fc8b8deac2d77524ff8cfe44acf95b5e1f59804e Mon Sep 17 00:00:00 2001 From: Steven Shi Date: Tue, 19 Nov 2019 16:17:00 +0800 Subject: BaseTools: Leverage compiler output to optimize binary cache Redesign the binary cache and bases on the compiler to output the dependency header files info for every module. The binary cache will directly consume the dependency header files info and doesn't parse the C source code by iteself. Also redesign the dependency files list format for module and try to share the common lib hash result as more as possible in local process. Remove the unnecessary share data access across multiprocessing. Signed-off-by: Steven Shi Cc: Liming Gao Cc: Bob Feng Reviewed-by: Bob Feng --- BaseTools/Source/Python/AutoGen/AutoGenWorker.py | 77 +++++++++++++++++------- 1 file changed, 55 insertions(+), 22 deletions(-) (limited to 'BaseTools/Source/Python/AutoGen/AutoGenWorker.py') diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py index 94ea61a487..40b448f5b2 100755 --- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py +++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py @@ -128,12 +128,27 @@ class AutoGenManager(threading.Thread): clearQ(taskq) clearQ(self.feedback_q) clearQ(logq) + # Copy the cache queue itmes to parent thread before clear + cacheq = self.autogen_workers[0].cache_q + try: + cache_num = 0 + while True: + item = cacheq.get() + if item == "CacheDone": + cache_num += 1 + else: + GlobalData.gModuleAllCacheStatus.add(item) + if cache_num == len(self.autogen_workers): + break + except: + print ("cache_q error") + def TerminateWorkers(self): self.error_event.set() def kill(self): self.feedback_q.put(None) class AutoGenWorkerInProcess(mp.Process): - def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_lock,cache_lock,share_data,log_q,error_event): + def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_lock,cache_q,log_q,error_event): mp.Process.__init__(self) self.module_queue = module_queue self.data_pipe_file_path =data_pipe_file_path @@ -141,8 +156,7 @@ class AutoGenWorkerInProcess(mp.Process): self.feedback_q = feedback_q self.PlatformMetaFileSet = {} self.file_lock = file_lock - self.cache_lock = cache_lock - self.share_data = share_data + self.cache_q = cache_q self.log_q = log_q self.error_event = error_event def GetPlatformMetaFile(self,filepath,root): @@ -184,12 +198,19 @@ class AutoGenWorkerInProcess(mp.Process): GlobalData.gDisableIncludePathCheck = False GlobalData.gFdfParser = self.data_pipe.Get("FdfParser") GlobalData.gDatabasePath = self.data_pipe.Get("DatabasePath") + + GlobalData.gUseHashCache = self.data_pipe.Get("UseHashCache") GlobalData.gBinCacheSource = self.data_pipe.Get("BinCacheSource") GlobalData.gBinCacheDest = self.data_pipe.Get("BinCacheDest") - GlobalData.gCacheIR = self.share_data + GlobalData.gPlatformHashFile = self.data_pipe.Get("PlatformHashFile") + GlobalData.gModulePreMakeCacheStatus = dict() + GlobalData.gModuleMakeCacheStatus = dict() + GlobalData.gHashChainStatus = dict() + GlobalData.gCMakeHashFile = dict() + GlobalData.gModuleHashFile = dict() + GlobalData.gFileHashDict = dict() GlobalData.gEnableGenfdsMultiThread = self.data_pipe.Get("EnableGenfdsMultiThread") GlobalData.file_lock = self.file_lock - GlobalData.cache_lock = self.cache_lock CommandTarget = self.data_pipe.Get("CommandTarget") pcd_from_build_option = [] for pcd_tuple in self.data_pipe.Get("BuildOptPcd"): @@ -205,10 +226,6 @@ class AutoGenWorkerInProcess(mp.Process): GlobalData.FfsCmd = FfsCmd PlatformMetaFile = self.GetPlatformMetaFile(self.data_pipe.Get("P_Info").get("ActivePlatform"), self.data_pipe.Get("P_Info").get("WorkspaceDir")) - libConstPcd = self.data_pipe.Get("LibConstPcd") - Refes = self.data_pipe.Get("REFS") - GlobalData.libConstPcd = libConstPcd - GlobalData.Refes = Refes while True: if self.module_queue.empty(): break @@ -230,27 +247,41 @@ class AutoGenWorkerInProcess(mp.Process): toolchain = self.data_pipe.Get("P_Info").get("ToolChain") Ma = ModuleAutoGen(self.Wa,module_metafile,target,toolchain,arch,PlatformMetaFile,self.data_pipe) Ma.IsLibrary = IsLib - if IsLib: - if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in libConstPcd: - Ma.ConstPcd = libConstPcd[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)] - if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in Refes: - Ma.ReferenceModules = Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)] - if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]: - Ma.GenModuleFilesHash(GlobalData.gCacheIR) - Ma.GenPreMakefileHash(GlobalData.gCacheIR) - if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR): + # SourceFileList calling sequence impact the makefile string sequence. + # Create cached SourceFileList here to unify its calling sequence for both + # CanSkipbyPreMakeCache and CreateCodeFile/CreateMakeFile. + RetVal = Ma.SourceFileList + if GlobalData.gUseHashCache and not GlobalData.gBinCacheDest and CommandTarget in [None, "", "all"]: + try: + CacheResult = Ma.CanSkipbyPreMakeCache() + except: + CacheResult = False + traceback.print_exc(file=sys.stdout) + self.feedback_q.put(taskname) + + if CacheResult: + self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "PreMakeCache", True)) continue + else: + self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "PreMakeCache", False)) Ma.CreateCodeFile(False) Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.Path, Ma.Arch),[])) if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]: - Ma.GenMakeHeaderFilesHash(GlobalData.gCacheIR) - Ma.GenMakeHash(GlobalData.gCacheIR) - if Ma.CanSkipbyMakeCache(GlobalData.gCacheIR): + try: + CacheResult = Ma.CanSkipbyMakeCache() + except: + CacheResult = False + traceback.print_exc(file=sys.stdout) + self.feedback_q.put(taskname) + + if CacheResult: + self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "MakeCache", True)) continue else: - Ma.PrintFirstMakeCacheMissFile(GlobalData.gCacheIR) + self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "MakeCache", False)) + except Empty: pass except: @@ -258,6 +289,8 @@ class AutoGenWorkerInProcess(mp.Process): self.feedback_q.put(taskname) finally: self.feedback_q.put("Done") + self.cache_q.put("CacheDone") + def printStatus(self): print("Processs ID: %d Run %d modules in AutoGen " % (os.getpid(),len(AutoGen.Cache()))) print("Processs ID: %d Run %d modules in AutoGenInfo " % (os.getpid(),len(AutoGenInfo.GetCache()))) -- cgit v1.2.3