summaryrefslogtreecommitdiffstats
path: root/BaseTools/Source/Python/build/build.py
diff options
context:
space:
mode:
Diffstat (limited to 'BaseTools/Source/Python/build/build.py')
-rwxr-xr-x[-rw-r--r--]BaseTools/Source/Python/build/build.py129
1 files changed, 101 insertions, 28 deletions
diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py
index 4bfa54666b..d7c817b95c 100644..100755
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -595,7 +595,7 @@ class BuildTask:
#
def AddDependency(self, Dependency):
for Dep in Dependency:
- if not Dep.BuildObject.IsBinaryModule and not Dep.BuildObject.CanSkipbyHash():
+ if not Dep.BuildObject.IsBinaryModule and not Dep.BuildObject.CanSkipbyCache(GlobalData.gCacheIR):
self.DependencyList.append(BuildTask.New(Dep)) # BuildTask list
## The thread wrapper of LaunchCommand function
@@ -811,7 +811,7 @@ class Build():
self.AutoGenMgr = None
EdkLogger.info("")
os.chdir(self.WorkspaceDir)
- self.share_data = Manager().dict()
+ GlobalData.gCacheIR = Manager().dict()
self.log_q = log_q
def StartAutoGen(self,mqueue, DataPipe,SkipAutoGen,PcdMaList,share_data):
try:
@@ -820,6 +820,13 @@ class Build():
feedback_q = mp.Queue()
file_lock = mp.Lock()
error_event = mp.Event()
+ GlobalData.file_lock = file_lock
+ FfsCmd = DataPipe.Get("FfsCommand")
+ if FfsCmd is None:
+ FfsCmd = {}
+ GlobalData.FfsCmd = FfsCmd
+ GlobalData.libConstPcd = DataPipe.Get("LibConstPcd")
+ GlobalData.Refes = DataPipe.Get("REFS")
auto_workers = [AutoGenWorkerInProcess(mqueue,DataPipe.dump_file,feedback_q,file_lock,share_data,self.log_q,error_event) for _ in range(self.ThreadNumber)]
self.AutoGenMgr = AutoGenManager(auto_workers,feedback_q,error_event)
self.AutoGenMgr.start()
@@ -827,14 +834,28 @@ class Build():
w.start()
if PcdMaList is not None:
for PcdMa in PcdMaList:
+ if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
+ PcdMa.GenModuleFilesHash(share_data)
+ PcdMa.GenPreMakefileHash(share_data)
+ if PcdMa.CanSkipbyPreMakefileCache(share_data):
+ continue
+
PcdMa.CreateCodeFile(False)
PcdMa.CreateMakeFile(False,GenFfsList = DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.File, PcdMa.Arch),[]))
+ if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
+ PcdMa.GenMakeHeaderFilesHash(share_data)
+ PcdMa.GenMakeHash(share_data)
+ if PcdMa.CanSkipbyMakeCache(share_data):
+ continue
+
self.AutoGenMgr.join()
rt = self.AutoGenMgr.Status
return rt, 0
- except Exception as e:
- return False,e.errcode
+ except FatalError as e:
+ return False, e.args[0]
+ except:
+ return False, UNKNOWN_ERROR
## Load configuration
#
@@ -1199,10 +1220,11 @@ class Build():
mqueue.put(m)
AutoGenObject.DataPipe.DataContainer = {"FfsCommand":FfsCommand}
+ AutoGenObject.DataPipe.DataContainer = {"CommandTarget": self.Target}
self.Progress.Start("Generating makefile and code")
data_pipe_file = os.path.join(AutoGenObject.BuildDir, "GlobalVar_%s_%s.bin" % (str(AutoGenObject.Guid),AutoGenObject.Arch))
AutoGenObject.DataPipe.dump(data_pipe_file)
- autogen_rt, errorcode = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList,self.share_data)
+ autogen_rt,errorcode = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList, GlobalData.gCacheIR)
self.Progress.Stop("done!")
if not autogen_rt:
self.AutoGenMgr.TerminateWorkers()
@@ -1799,6 +1821,15 @@ class Build():
CmdListDict = None
if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
CmdListDict = self._GenFfsCmd(Wa.ArchList)
+
+ # Add Platform and Package level hash in share_data for module hash calculation later
+ if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest:
+ GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash
+ for PkgName in GlobalData.gPackageHash.keys():
+ GlobalData.gCacheIR[(PkgName, 'PackageHash')] = GlobalData.gPackageHash[PkgName]
+ GlobalData.file_lock = mp.Lock()
+ GlobalData.FfsCmd = CmdListDict
+
self.Progress.Stop("done!")
MaList = []
ExitFlag = threading.Event()
@@ -1808,20 +1839,23 @@ class Build():
AutoGenStart = time.time()
GlobalData.gGlobalDefines['ARCH'] = Arch
Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)
+ GlobalData.libConstPcd = Pa.DataPipe.Get("LibConstPcd")
+ GlobalData.Refes = Pa.DataPipe.Get("REFS")
for Module in Pa.Platform.Modules:
if self.ModuleFile.Dir == Module.Dir and self.ModuleFile.Name == Module.Name:
Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
if Ma is None:
continue
MaList.append(Ma)
- if Ma.CanSkipbyHash():
- self.HashSkipModules.append(Ma)
- if GlobalData.gBinCacheSource:
- EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))
- continue
- else:
- if GlobalData.gBinCacheSource:
- EdkLogger.quiet("cache miss: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))
+
+ if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
+ Ma.GenModuleFilesHash(GlobalData.gCacheIR)
+ Ma.GenPreMakefileHash(GlobalData.gCacheIR)
+ if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR):
+ self.HashSkipModules.append(Ma)
+ EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))
+ continue
+
# Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds'
if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
# for target which must generate AutoGen code and makefile
@@ -1841,6 +1875,18 @@ class Build():
self.Progress.Stop("done!")
if self.Target == "genmake":
return True
+
+ if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
+ Ma.GenMakeHeaderFilesHash(GlobalData.gCacheIR)
+ Ma.GenMakeHash(GlobalData.gCacheIR)
+ if Ma.CanSkipbyMakeCache(GlobalData.gCacheIR):
+ self.HashSkipModules.append(Ma)
+ EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))
+ continue
+ else:
+ EdkLogger.quiet("cache miss: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))
+ Ma.PrintFirstMakeCacheMissFile(GlobalData.gCacheIR)
+
self.BuildModules.append(Ma)
# Initialize all modules in tracking to 'FAIL'
if Ma.Arch not in GlobalData.gModuleBuildTracking:
@@ -1985,11 +2031,18 @@ class Build():
if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
CmdListDict = self._GenFfsCmd(Wa.ArchList)
+ # Add Platform and Package level hash in share_data for module hash calculation later
+ if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest:
+ GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash
+ for PkgName in GlobalData.gPackageHash.keys():
+ GlobalData.gCacheIR[(PkgName, 'PackageHash')] = GlobalData.gPackageHash[PkgName]
+
# multi-thread exit flag
ExitFlag = threading.Event()
ExitFlag.clear()
self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime)))
self.BuildModules = []
+ TotalModules = []
for Arch in Wa.ArchList:
PcdMaList = []
AutoGenStart = time.time()
@@ -2009,6 +2062,7 @@ class Build():
ModuleList.append(Inf)
Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict}
Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp}
+ Pa.DataPipe.DataContainer = {"CommandTarget": self.Target}
for Module in ModuleList:
# Get ModuleAutoGen object to generate C code file and makefile
Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
@@ -2019,30 +2073,34 @@ class Build():
Ma.PlatformInfo = Pa
Ma.Workspace = Wa
PcdMaList.append(Ma)
- if Ma.CanSkipbyHash():
- self.HashSkipModules.append(Ma)
- if GlobalData.gBinCacheSource:
- EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))
- continue
- else:
- if GlobalData.gBinCacheSource:
- EdkLogger.quiet("cache miss: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))
-
- # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds'
- # for target which must generate AutoGen code and makefile
-
- self.BuildModules.append(Ma)
+ TotalModules.append(Ma)
# Initialize all modules in tracking to 'FAIL'
if Ma.Arch not in GlobalData.gModuleBuildTracking:
GlobalData.gModuleBuildTracking[Ma.Arch] = dict()
if Ma not in GlobalData.gModuleBuildTracking[Ma.Arch]:
GlobalData.gModuleBuildTracking[Ma.Arch][Ma] = 'FAIL'
+
mqueue = mp.Queue()
for m in Pa.GetAllModuleInfo:
mqueue.put(m)
data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch))
Pa.DataPipe.dump(data_pipe_file)
- autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList,self.share_data)
+ autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList, GlobalData.gCacheIR)
+
+ # Skip cache hit modules
+ if GlobalData.gBinCacheSource:
+ for Ma in TotalModules:
+ if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \
+ GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].PreMakeCacheHit:
+ self.HashSkipModules.append(Ma)
+ continue
+ if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \
+ GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].MakeCacheHit:
+ self.HashSkipModules.append(Ma)
+ continue
+ self.BuildModules.append(Ma)
+ else:
+ self.BuildModules.extend(TotalModules)
if not autogen_rt:
self.AutoGenMgr.TerminateWorkers()
@@ -2050,9 +2108,24 @@ class Build():
raise FatalError(errorcode)
self.AutoGenTime += int(round((time.time() - AutoGenStart)))
self.Progress.Stop("done!")
+
+ if GlobalData.gBinCacheSource:
+ EdkLogger.quiet("Total cache hit driver num: %s, cache miss driver num: %s" % (len(set(self.HashSkipModules)), len(set(self.BuildModules))))
+ CacheHitMa = set()
+ CacheNotHitMa = set()
+ for IR in GlobalData.gCacheIR.keys():
+ if 'PlatformHash' in IR or 'PackageHash' in IR:
+ continue
+ if GlobalData.gCacheIR[IR].PreMakeCacheHit or GlobalData.gCacheIR[IR].MakeCacheHit:
+ CacheHitMa.add(IR)
+ else:
+ # There might be binary module or module which has .inc files, not count for cache miss
+ CacheNotHitMa.add(IR)
+ EdkLogger.quiet("Total module num: %s, cache hit module num: %s" % (len(CacheHitMa)+len(CacheNotHitMa), len(CacheHitMa)))
+
for Arch in Wa.ArchList:
MakeStart = time.time()
- for Ma in self.BuildModules:
+ for Ma in set(self.BuildModules):
# Generate build task for the module
if not Ma.IsBinaryModule:
Bt = BuildTask.New(ModuleMakeUnit(Ma, Pa.BuildCommand,self.Target))
generated by cgit v1.2.3 (git 2.25.1) at 2024-05-23 07:29:45 +0000