BaseTools: Replace the sqlite database with list
https://bugzilla.tianocore.org/show_bug.cgi?id=1288 [V2] Optimize this patch so that it can be easy to review. This patch is just apply the change to original files while not create new similar files. [V1] This patch is one of build tool performance improvement series patches. This patch is going to use python list to store the parser data instead of using sqlite database. The replacement solution is as below: SQL insert: list.append() SQL select: list comprehension. for example: Select * from table where field = “something” -> [ item for item in table if item[3] == “something”] SQL update: python map function. for example: Update table set field1=newvalue where filed2 = “something”. -> map(lambda x: x[1] = newvalue, [item for item in table if item[2] == “something”]) SQL delete: list comprehension. With this change, We can save the time of interpreting SQL statement and the time of write database to file system Contributed-under: TianoCore Contribution Agreement 1.1 Signed-off-by: BobCF <bob.c.feng@intel.com> Cc: Liming Gao <liming.gao@intel.com> Cc: Jaben Carsey <jaben.carsey@intel.com> Reviewed-by: Liming Gao <liming.gao@intel.com> Reviewed-by: Jaben Carsey <jaben.carsey@intel.com>
This commit is contained in:
BobCF
@ -16,7 +16,6 @@
|
||||
# Import Modules
|
||||
#
|
||||
from __future__ import absolute_import
|
||||
import sqlite3
|
||||
from Common.StringUtils import *
|
||||
from Common.DataType import *
|
||||
from Common.Misc import *
|
||||
@ -122,7 +121,7 @@ class WorkspaceDatabase(object):
|
||||
FilePath,
|
||||
FileType,
|
||||
Arch,
|
||||
MetaFileStorage(self.WorkspaceDb.Cur, FilePath, FileType)
|
||||
MetaFileStorage(self.WorkspaceDb, FilePath, FileType)
|
||||
)
|
||||
# alwasy do post-process, in case of macros change
|
||||
MetaFile.DoPostProcess()
|
||||
@ -152,133 +151,23 @@ class WorkspaceDatabase(object):
|
||||
# @param GlobalMacros Global macros used for replacement during file parsing
|
||||
# @prarm RenewDb=False Create new database file if it's already there
|
||||
#
|
||||
def __init__(self, DbPath, RenewDb=False):
|
||||
self._DbClosedFlag = False
|
||||
if not DbPath:
|
||||
DbPath = os.path.normpath(mws.join(GlobalData.gWorkspace, 'Conf', GlobalData.gDatabasePath))
|
||||
|
||||
# don't create necessary path for db in memory
|
||||
if DbPath != ':memory:':
|
||||
DbDir = os.path.split(DbPath)[0]
|
||||
if not os.path.exists(DbDir):
|
||||
os.makedirs(DbDir)
|
||||
|
||||
# remove db file in case inconsistency between db and file in file system
|
||||
if self._CheckWhetherDbNeedRenew(RenewDb, DbPath):
|
||||
os.remove(DbPath)
|
||||
|
||||
# create db with optimized parameters
|
||||
self.Conn = sqlite3.connect(DbPath, isolation_level='DEFERRED')
|
||||
self.Conn.execute("PRAGMA synchronous=OFF")
|
||||
self.Conn.execute("PRAGMA temp_store=MEMORY")
|
||||
self.Conn.execute("PRAGMA count_changes=OFF")
|
||||
self.Conn.execute("PRAGMA cache_size=8192")
|
||||
#self.Conn.execute("PRAGMA page_size=8192")
|
||||
|
||||
# to avoid non-ascii character conversion issue
|
||||
self.Conn.text_factory = str
|
||||
self.Cur = self.Conn.cursor()
|
||||
|
||||
def __init__(self):
|
||||
self.DB = dict()
|
||||
# create table for internal uses
|
||||
self.TblDataModel = TableDataModel(self.Cur)
|
||||
self.TblFile = TableFile(self.Cur)
|
||||
self.TblDataModel = DataClass.MODEL_LIST
|
||||
self.TblFile = []
|
||||
self.Platform = None
|
||||
|
||||
# conversion object for build or file format conversion purpose
|
||||
self.BuildObject = WorkspaceDatabase.BuildObjectFactory(self)
|
||||
self.TransformObject = WorkspaceDatabase.TransformObjectFactory(self)
|
||||
|
||||
## Check whether workspace database need to be renew.
|
||||
# The renew reason maybe:
|
||||
# 1) If user force to renew;
|
||||
# 2) If user do not force renew, and
|
||||
# a) If the time of last modified python source is newer than database file;
|
||||
# b) If the time of last modified frozen executable file is newer than database file;
|
||||
#
|
||||
# @param force User force renew database
|
||||
# @param DbPath The absolute path of workspace database file
|
||||
#
|
||||
# @return Bool value for whether need renew workspace databse
|
||||
#
|
||||
def _CheckWhetherDbNeedRenew (self, force, DbPath):
|
||||
# if database does not exist, we need do nothing
|
||||
if not os.path.exists(DbPath): return False
|
||||
def SetFileTimeStamp(self,FileId,TimeStamp):
|
||||
self.TblFile[FileId][6] = TimeStamp
|
||||
|
||||
# if user force to renew database, then not check whether database is out of date
|
||||
if force: return True
|
||||
def GetFileTimeStamp(self,FileId):
|
||||
return self.TblFile[FileId][6]
|
||||
|
||||
#
|
||||
# Check the time of last modified source file or build.exe
|
||||
# if is newer than time of database, then database need to be re-created.
|
||||
#
|
||||
timeOfToolModified = 0
|
||||
if hasattr(sys, "frozen"):
|
||||
exePath = os.path.abspath(sys.executable)
|
||||
timeOfToolModified = os.stat(exePath).st_mtime
|
||||
else:
|
||||
curPath = os.path.dirname(__file__) # curPath is the path of WorkspaceDatabase.py
|
||||
rootPath = os.path.split(curPath)[0] # rootPath is root path of python source, such as /BaseTools/Source/Python
|
||||
if rootPath == "" or rootPath is None:
|
||||
EdkLogger.verbose("\nFail to find the root path of build.exe or python sources, so can not \
|
||||
determine whether database file is out of date!\n")
|
||||
|
||||
# walk the root path of source or build's binary to get the time last modified.
|
||||
|
||||
for root, dirs, files in os.walk (rootPath):
|
||||
for dir in dirs:
|
||||
# bypass source control folder
|
||||
if dir.lower() in [".svn", "_svn", "cvs"]:
|
||||
dirs.remove(dir)
|
||||
|
||||
for file in files:
|
||||
ext = os.path.splitext(file)[1]
|
||||
if ext.lower() == ".py": # only check .py files
|
||||
fd = os.stat(os.path.join(root, file))
|
||||
if timeOfToolModified < fd.st_mtime:
|
||||
timeOfToolModified = fd.st_mtime
|
||||
if timeOfToolModified > os.stat(DbPath).st_mtime:
|
||||
EdkLogger.verbose("\nWorkspace database is out of data!")
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
## Initialize build database
|
||||
def InitDatabase(self):
|
||||
EdkLogger.verbose("\nInitialize build database started ...")
|
||||
|
||||
#
|
||||
# Create new tables
|
||||
#
|
||||
self.TblDataModel.Create(False)
|
||||
self.TblFile.Create(False)
|
||||
|
||||
#
|
||||
# Initialize table DataModel
|
||||
#
|
||||
self.TblDataModel.InitTable()
|
||||
EdkLogger.verbose("Initialize build database ... DONE!")
|
||||
|
||||
## Query a table
|
||||
#
|
||||
# @param Table: The instance of the table to be queried
|
||||
#
|
||||
def QueryTable(self, Table):
|
||||
Table.Query()
|
||||
|
||||
def __del__(self):
|
||||
self.Close()
|
||||
|
||||
## Close entire database
|
||||
#
|
||||
# Commit all first
|
||||
# Close the connection and cursor
|
||||
#
|
||||
def Close(self):
|
||||
if not self._DbClosedFlag:
|
||||
self.Conn.commit()
|
||||
self.Cur.close()
|
||||
self.Conn.close()
|
||||
self._DbClosedFlag = True
|
||||
|
||||
## Summarize all packages in the database
|
||||
def GetPackageList(self, Platform, Arch, TargetName, ToolChainTag):
|
||||
@ -307,7 +196,7 @@ determine whether database file is out of date!\n")
|
||||
## Summarize all platforms in the database
|
||||
def PlatformList(self):
|
||||
RetVal = []
|
||||
for PlatformFile in self.TblFile.GetFileList(MODEL_FILE_DSC):
|
||||
for PlatformFile in [item[3] for item in self.TblFile if item[5] == MODEL_FILE_DSC]:
|
||||
try:
|
||||
RetVal.append(self.BuildObject[PathClass(PlatformFile), TAB_COMMON])
|
||||
except:
|
||||
|
||||
Reference in New Issue
Block a user