diff options
Diffstat (limited to 'BaseTools/Source/Python/AutoGen/AutoGenWorker.py')
| -rwxr-xr-x | BaseTools/Source/Python/AutoGen/AutoGenWorker.py | 77 |
1 files changed, 55 insertions, 22 deletions
diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py index 94ea61a487..40b448f5b2 100755 --- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py +++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py @@ -128,12 +128,27 @@ class AutoGenManager(threading.Thread): clearQ(taskq)
clearQ(self.feedback_q)
clearQ(logq)
+ # Copy the cache queue itmes to parent thread before clear
+ cacheq = self.autogen_workers[0].cache_q
+ try:
+ cache_num = 0
+ while True:
+ item = cacheq.get()
+ if item == "CacheDone":
+ cache_num += 1
+ else:
+ GlobalData.gModuleAllCacheStatus.add(item)
+ if cache_num == len(self.autogen_workers):
+ break
+ except:
+ print ("cache_q error")
+
def TerminateWorkers(self):
self.error_event.set()
def kill(self):
self.feedback_q.put(None)
class AutoGenWorkerInProcess(mp.Process):
- def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_lock,cache_lock,share_data,log_q,error_event):
+ def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_lock,cache_q,log_q,error_event):
mp.Process.__init__(self)
self.module_queue = module_queue
self.data_pipe_file_path =data_pipe_file_path
@@ -141,8 +156,7 @@ class AutoGenWorkerInProcess(mp.Process): self.feedback_q = feedback_q
self.PlatformMetaFileSet = {}
self.file_lock = file_lock
- self.cache_lock = cache_lock
- self.share_data = share_data
+ self.cache_q = cache_q
self.log_q = log_q
self.error_event = error_event
def GetPlatformMetaFile(self,filepath,root):
@@ -184,12 +198,19 @@ class AutoGenWorkerInProcess(mp.Process): GlobalData.gDisableIncludePathCheck = False
GlobalData.gFdfParser = self.data_pipe.Get("FdfParser")
GlobalData.gDatabasePath = self.data_pipe.Get("DatabasePath")
+
+ GlobalData.gUseHashCache = self.data_pipe.Get("UseHashCache")
GlobalData.gBinCacheSource = self.data_pipe.Get("BinCacheSource")
GlobalData.gBinCacheDest = self.data_pipe.Get("BinCacheDest")
- GlobalData.gCacheIR = self.share_data
+ GlobalData.gPlatformHashFile = self.data_pipe.Get("PlatformHashFile")
+ GlobalData.gModulePreMakeCacheStatus = dict()
+ GlobalData.gModuleMakeCacheStatus = dict()
+ GlobalData.gHashChainStatus = dict()
+ GlobalData.gCMakeHashFile = dict()
+ GlobalData.gModuleHashFile = dict()
+ GlobalData.gFileHashDict = dict()
GlobalData.gEnableGenfdsMultiThread = self.data_pipe.Get("EnableGenfdsMultiThread")
GlobalData.file_lock = self.file_lock
- GlobalData.cache_lock = self.cache_lock
CommandTarget = self.data_pipe.Get("CommandTarget")
pcd_from_build_option = []
for pcd_tuple in self.data_pipe.Get("BuildOptPcd"):
@@ -205,10 +226,6 @@ class AutoGenWorkerInProcess(mp.Process): GlobalData.FfsCmd = FfsCmd
PlatformMetaFile = self.GetPlatformMetaFile(self.data_pipe.Get("P_Info").get("ActivePlatform"),
self.data_pipe.Get("P_Info").get("WorkspaceDir"))
- libConstPcd = self.data_pipe.Get("LibConstPcd")
- Refes = self.data_pipe.Get("REFS")
- GlobalData.libConstPcd = libConstPcd
- GlobalData.Refes = Refes
while True:
if self.module_queue.empty():
break
@@ -230,27 +247,41 @@ class AutoGenWorkerInProcess(mp.Process): toolchain = self.data_pipe.Get("P_Info").get("ToolChain")
Ma = ModuleAutoGen(self.Wa,module_metafile,target,toolchain,arch,PlatformMetaFile,self.data_pipe)
Ma.IsLibrary = IsLib
- if IsLib:
- if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in libConstPcd:
- Ma.ConstPcd = libConstPcd[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]
- if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in Refes:
- Ma.ReferenceModules = Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]
- if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]:
- Ma.GenModuleFilesHash(GlobalData.gCacheIR)
- Ma.GenPreMakefileHash(GlobalData.gCacheIR)
- if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR):
+ # SourceFileList calling sequence impact the makefile string sequence.
+ # Create cached SourceFileList here to unify its calling sequence for both
+ # CanSkipbyPreMakeCache and CreateCodeFile/CreateMakeFile.
+ RetVal = Ma.SourceFileList
+ if GlobalData.gUseHashCache and not GlobalData.gBinCacheDest and CommandTarget in [None, "", "all"]:
+ try:
+ CacheResult = Ma.CanSkipbyPreMakeCache()
+ except:
+ CacheResult = False
+ traceback.print_exc(file=sys.stdout)
+ self.feedback_q.put(taskname)
+
+ if CacheResult:
+ self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "PreMakeCache", True))
continue
+ else:
+ self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "PreMakeCache", False))
Ma.CreateCodeFile(False)
Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.Path, Ma.Arch),[]))
if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]:
- Ma.GenMakeHeaderFilesHash(GlobalData.gCacheIR)
- Ma.GenMakeHash(GlobalData.gCacheIR)
- if Ma.CanSkipbyMakeCache(GlobalData.gCacheIR):
+ try:
+ CacheResult = Ma.CanSkipbyMakeCache()
+ except:
+ CacheResult = False
+ traceback.print_exc(file=sys.stdout)
+ self.feedback_q.put(taskname)
+
+ if CacheResult:
+ self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "MakeCache", True))
continue
else:
- Ma.PrintFirstMakeCacheMissFile(GlobalData.gCacheIR)
+ self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "MakeCache", False))
+
except Empty:
pass
except:
@@ -258,6 +289,8 @@ class AutoGenWorkerInProcess(mp.Process): self.feedback_q.put(taskname)
finally:
self.feedback_q.put("Done")
+ self.cache_q.put("CacheDone")
+
def printStatus(self):
print("Processs ID: %d Run %d modules in AutoGen " % (os.getpid(),len(AutoGen.Cache())))
print("Processs ID: %d Run %d modules in AutoGenInfo " % (os.getpid(),len(AutoGenInfo.GetCache())))
|