BaseTools: Replace the sqlite database with list

https://bugzilla.tianocore.org/show_bug.cgi?id=1288

[V2]
Optimize this patch so that it can be easy to review.
This patch is just apply the change to original files while
not create new similar files.

[V1]
This patch is one of build tool performance improvement
series patches.

This patch is going to use python list to store the parser data
instead of using sqlite database.

The replacement solution is as below:

SQL insert: list.append()
SQL select: list comprehension. for example:
Select * from table where field = “something”
->
[ item for item in table if item[3] == “something”]

SQL update: python map function. for example:
Update table set field1=newvalue where filed2 = “something”.
-> map(lambda x: x[1] = newvalue,
   [item for item in table if item[2] == “something”])

SQL delete: list comprehension.

With this change, We can save the time of interpreting SQL statement
and the time of write database to file system

Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: BobCF <bob.c.feng@intel.com>
Cc: Liming Gao <liming.gao@intel.com>
Cc: Jaben Carsey <jaben.carsey@intel.com>
Reviewed-by: Liming Gao <liming.gao@intel.com>
Reviewed-by: Jaben Carsey <jaben.carsey@intel.com>

1 files changed, 5 insertions, 22 deletions

diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py
index cf864d0ef5..cbbb291b2c 100644
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -42,7 +42,7 @@ from Common.DataType import *
 from Common.BuildVersion import gBUILD_VERSION

 from AutoGen.AutoGen import *

 from Common.BuildToolError import *

-from Workspace.WorkspaceDatabase import *

+from Workspace.WorkspaceDatabase import WorkspaceDatabase

 from Common.MultipleWorkspace import MultipleWorkspace as mws

 

 from BuildReport import BuildReport

@@ -826,10 +826,7 @@ class Build():
         GlobalData.gConfDirectory = ConfDirectoryPath

         GlobalData.gDatabasePath = os.path.normpath(os.path.join(ConfDirectoryPath, GlobalData.gDatabasePath))

 

-        if BuildOptions.DisableCache:

-            self.Db         = WorkspaceDatabase(":memory:")

-        else:

-            self.Db = WorkspaceDatabase(GlobalData.gDatabasePath, self.Reparse)

+        self.Db = WorkspaceDatabase()

         self.BuildDatabase = self.Db.BuildObject

         self.Platform = None

         self.ToolChainFamily = None

@@ -982,9 +979,6 @@ class Build():
         if ErrorCode != 0:

             EdkLogger.error("build", ErrorCode, ExtraData=ErrorInfo)

 

-        # create metafile database

-        if not self.Db_Flag:

-            self.Db.InitDatabase()

 

     def InitPreBuild(self):

         self.LoadConfiguration()

@@ -1003,7 +997,6 @@ class Build():
         if 'PREBUILD' in GlobalData.gCommandLineDefines:

             self.Prebuild   = GlobalData.gCommandLineDefines.get('PREBUILD')

         else:

-            self.Db.InitDatabase()

             self.Db_Flag = True

             Platform = self.Db.MapPlatform(str(self.PlatformFile))

             self.Prebuild = str(Platform.Prebuild)

@@ -2082,13 +2075,7 @@ class Build():
                     self.MakeTime += int(round((time.time() - MakeStart)))

 

                 MakeContiue = time.time()

-                #

-                # Save temp tables to a TmpTableDict.

-                #

-                for Key in Wa.BuildDatabase._CACHE_:

-                    if Wa.BuildDatabase._CACHE_[Key]._RawData and Wa.BuildDatabase._CACHE_[Key]._RawData._Table and Wa.BuildDatabase._CACHE_[Key]._RawData._Table.Table:

-                        if TemporaryTablePattern.match(Wa.BuildDatabase._CACHE_[Key]._RawData._Table.Table):

-                            TmpTableDict[Wa.BuildDatabase._CACHE_[Key]._RawData._Table.Table] = Wa.BuildDatabase._CACHE_[Key]._RawData._Table.Cur

+

                 #

                 #

                 # All modules have been put in build tasks queue. Tell task scheduler

@@ -2230,7 +2217,6 @@ class Build():
             self._BuildModule()

 

         if self.Target == 'cleanall':

-            self.Db.Close()

             RemoveDirectory(os.path.dirname(GlobalData.gDatabasePath), True)

 

     def CreateAsBuiltInf(self):

@@ -2491,10 +2477,7 @@ def Main():
         GlobalData.gCommandLineDefines['ARCH'] = ' '.join(MyBuild.ArchList)

         if not (MyBuild.LaunchPrebuildFlag and os.path.exists(MyBuild.PlatformBuildPath)):

             MyBuild.Launch()

-        # Drop temp tables to avoid database locked.

-        for TmpTableName in TmpTableDict:

-            SqlCommand = """drop table IF EXISTS %s""" % TmpTableName

-            TmpTableDict[TmpTableName].execute(SqlCommand)

+

         #MyBuild.DumpBuildData()

         #

         # All job done, no error found and no exception raised

@@ -2566,7 +2549,7 @@ def Main():
     if MyBuild is not None:

         if not BuildError:

             MyBuild.BuildReport.GenerateReport(BuildDurationStr, LogBuildTime(MyBuild.AutoGenTime), LogBuildTime(MyBuild.MakeTime), LogBuildTime(MyBuild.GenFdsTime))

-        MyBuild.Db.Close()

+

     EdkLogger.SetLevel(EdkLogger.QUIET)

     EdkLogger.quiet("\n- %s -" % Conclusion)

     EdkLogger.quiet(time.strftime("Build end time: %H:%M:%S, %b.%d %Y", time.localtime()))