From 2f818ed0fb57d98985d151781a2ce9b8683129ee Mon Sep 17 00:00:00 2001 From: BobCF Date: Fri, 9 Nov 2018 15:41:02 +0800 Subject: BaseTools: Replace the sqlite database with list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit https://bugzilla.tianocore.org/show_bug.cgi?id=1288 [V2] Optimize this patch so that it can be easy to review. This patch is just apply the change to original files while not create new similar files. [V1] This patch is one of build tool performance improvement series patches. This patch is going to use python list to store the parser data instead of using sqlite database. The replacement solution is as below: SQL insert: list.append() SQL select: list comprehension. for example: Select * from table where field = “something” -> [ item for item in table if item[3] == “something”] SQL update: python map function. for example: Update table set field1=newvalue where filed2 = “something”. -> map(lambda x: x[1] = newvalue, [item for item in table if item[2] == “something”]) SQL delete: list comprehension. With this change, We can save the time of interpreting SQL statement and the time of write database to file system Contributed-under: TianoCore Contribution Agreement 1.1 Signed-off-by: BobCF Cc: Liming Gao Cc: Jaben Carsey Reviewed-by: Liming Gao Reviewed-by: Jaben Carsey --- BaseTools/Source/Python/build/build.py | 27 +++++---------------------- 1 file changed, 5 insertions(+), 22 deletions(-) (limited to 'BaseTools/Source/Python/build/build.py') diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py index cf864d0ef5..cbbb291b2c 100644 --- a/BaseTools/Source/Python/build/build.py +++ b/BaseTools/Source/Python/build/build.py @@ -42,7 +42,7 @@ from Common.DataType import * from Common.BuildVersion import gBUILD_VERSION from AutoGen.AutoGen import * from Common.BuildToolError import * -from Workspace.WorkspaceDatabase import * +from Workspace.WorkspaceDatabase import WorkspaceDatabase from Common.MultipleWorkspace import MultipleWorkspace as mws from BuildReport import BuildReport @@ -826,10 +826,7 @@ class Build(): GlobalData.gConfDirectory = ConfDirectoryPath GlobalData.gDatabasePath = os.path.normpath(os.path.join(ConfDirectoryPath, GlobalData.gDatabasePath)) - if BuildOptions.DisableCache: - self.Db = WorkspaceDatabase(":memory:") - else: - self.Db = WorkspaceDatabase(GlobalData.gDatabasePath, self.Reparse) + self.Db = WorkspaceDatabase() self.BuildDatabase = self.Db.BuildObject self.Platform = None self.ToolChainFamily = None @@ -982,9 +979,6 @@ class Build(): if ErrorCode != 0: EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo) - # create metafile database - if not self.Db_Flag: - self.Db.InitDatabase() def InitPreBuild(self): self.LoadConfiguration() @@ -1003,7 +997,6 @@ class Build(): if 'PREBUILD' in GlobalData.gCommandLineDefines: self.Prebuild = GlobalData.gCommandLineDefines.get('PREBUILD') else: - self.Db.InitDatabase() self.Db_Flag = True Platform = self.Db.MapPlatform(str(self.PlatformFile)) self.Prebuild = str(Platform.Prebuild) @@ -2082,13 +2075,7 @@ class Build(): self.MakeTime += int(round((time.time() - MakeStart))) MakeContiue = time.time() - # - # Save temp tables to a TmpTableDict. - # - for Key in Wa.BuildDatabase._CACHE_: - if Wa.BuildDatabase._CACHE_[Key]._RawData and Wa.BuildDatabase._CACHE_[Key]._RawData._Table and Wa.BuildDatabase._CACHE_[Key]._RawData._Table.Table: - if TemporaryTablePattern.match(Wa.BuildDatabase._CACHE_[Key]._RawData._Table.Table): - TmpTableDict[Wa.BuildDatabase._CACHE_[Key]._RawData._Table.Table] = Wa.BuildDatabase._CACHE_[Key]._RawData._Table.Cur + # # # All modules have been put in build tasks queue. Tell task scheduler @@ -2230,7 +2217,6 @@ class Build(): self._BuildModule() if self.Target == 'cleanall': - self.Db.Close() RemoveDirectory(os.path.dirname(GlobalData.gDatabasePath), True) def CreateAsBuiltInf(self): @@ -2491,10 +2477,7 @@ def Main(): GlobalData.gCommandLineDefines['ARCH'] = ' '.join(MyBuild.ArchList) if not (MyBuild.LaunchPrebuildFlag and os.path.exists(MyBuild.PlatformBuildPath)): MyBuild.Launch() - # Drop temp tables to avoid database locked. - for TmpTableName in TmpTableDict: - SqlCommand = """drop table IF EXISTS %s""" % TmpTableName - TmpTableDict[TmpTableName].execute(SqlCommand) + #MyBuild.DumpBuildData() # # All job done, no error found and no exception raised @@ -2566,7 +2549,7 @@ def Main(): if MyBuild is not None: if not BuildError: MyBuild.BuildReport.GenerateReport(BuildDurationStr, LogBuildTime(MyBuild.AutoGenTime), LogBuildTime(MyBuild.MakeTime), LogBuildTime(MyBuild.GenFdsTime)) - MyBuild.Db.Close() + EdkLogger.SetLevel(EdkLogger.QUIET) EdkLogger.quiet("\n- %s -" % Conclusion) EdkLogger.quiet(time.strftime("Build end time: %H:%M:%S, %b.%d %Y", time.localtime())) -- cgit v1.2.3