sravan/system76-edk2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

BaseTools: Leverage compiler output to optimize binary cache

Browse Source 
Redesign the binary cache and bases on the compiler to
output the dependency header files info for every module.
The binary cache will directly consume the dependency header
files info and doesn't parse the C source code by iteself.
Also redesign the dependency files list format for module
and try to share the common lib hash result as more as
possible in local process. Remove the unnecessary share data
access across multiprocessing.

Signed-off-by: Steven Shi <steven.shi@intel.com>

Cc: Liming Gao <liming.gao@intel.com>
Cc: Bob Feng <bob.c.feng@intel.com>
Reviewed-by: Bob Feng <bob.c.feng@intel.com>
This commit is contained in:
Steven Shi
2019-11-19 16:17:00 +08:00
committed by mergify[bot]
parent 3bfbc91507
commit fc8b8deac2
6 changed files with 786 additions and 894 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
BaseTools/Source/Python/AutoGen/AutoGenWorker.py
View File

@ -128,12 +128,27 @@ class AutoGenManager(threading.Thread):
clearQ(taskq)
clearQ(self.feedback_q)
clearQ(logq)
# Copy the cache queue itmes to parent thread before clear
cacheq = self.autogen_workers[0].cache_q
try:
cache_num = 0
while True:
item = cacheq.get()
if item == "CacheDone":
cache_num += 1
else:
GlobalData.gModuleAllCacheStatus.add(item)
if cache_num == len(self.autogen_workers):
break
except:
print ("cache_q error")
def TerminateWorkers(self):
self.error_event.set()
def kill(self):
self.feedback_q.put(None)
class AutoGenWorkerInProcess(mp.Process):
def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_lock,cache_lock,share_data,log_q,error_event):
def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_lock,cache_q,log_q,error_event):
mp.Process.__init__(self)
self.module_queue = module_queue
self.data_pipe_file_path =data_pipe_file_path
@ -141,8 +156,7 @@ class AutoGenWorkerInProcess(mp.Process):
self.feedback_q = feedback_q
self.PlatformMetaFileSet = {}
self.file_lock = file_lock
self.cache_lock = cache_lock
self.share_data = share_data
self.cache_q = cache_q
self.log_q = log_q
self.error_event = error_event
def GetPlatformMetaFile(self,filepath,root):
@ -184,12 +198,19 @@ class AutoGenWorkerInProcess(mp.Process):
GlobalData.gDisableIncludePathCheck = False
GlobalData.gFdfParser = self.data_pipe.Get("FdfParser")
GlobalData.gDatabasePath = self.data_pipe.Get("DatabasePath")
GlobalData.gUseHashCache = self.data_pipe.Get("UseHashCache")
GlobalData.gBinCacheSource = self.data_pipe.Get("BinCacheSource")
GlobalData.gBinCacheDest = self.data_pipe.Get("BinCacheDest")
GlobalData.gCacheIR = self.share_data
GlobalData.gPlatformHashFile = self.data_pipe.Get("PlatformHashFile")
GlobalData.gModulePreMakeCacheStatus = dict()
GlobalData.gModuleMakeCacheStatus = dict()
GlobalData.gHashChainStatus = dict()
GlobalData.gCMakeHashFile = dict()
GlobalData.gModuleHashFile = dict()
GlobalData.gFileHashDict = dict()
GlobalData.gEnableGenfdsMultiThread = self.data_pipe.Get("EnableGenfdsMultiThread")
GlobalData.file_lock = self.file_lock
GlobalData.cache_lock = self.cache_lock
CommandTarget = self.data_pipe.Get("CommandTarget")
pcd_from_build_option = []
for pcd_tuple in self.data_pipe.Get("BuildOptPcd"):
@ -205,10 +226,6 @@ class AutoGenWorkerInProcess(mp.Process):
GlobalData.FfsCmd = FfsCmd
PlatformMetaFile = self.GetPlatformMetaFile(self.data_pipe.Get("P_Info").get("ActivePlatform"),
self.data_pipe.Get("P_Info").get("WorkspaceDir"))
libConstPcd = self.data_pipe.Get("LibConstPcd")
Refes = self.data_pipe.Get("REFS")
GlobalData.libConstPcd = libConstPcd
GlobalData.Refes = Refes
while True:
if self.module_queue.empty():
break
@ -230,27 +247,41 @@ class AutoGenWorkerInProcess(mp.Process):
toolchain = self.data_pipe.Get("P_Info").get("ToolChain")
Ma = ModuleAutoGen(self.Wa,module_metafile,target,toolchain,arch,PlatformMetaFile,self.data_pipe)
Ma.IsLibrary = IsLib
if IsLib:
if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in libConstPcd:
Ma.ConstPcd = libConstPcd[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]
if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in Refes:
Ma.ReferenceModules = Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]
if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]:
Ma.GenModuleFilesHash(GlobalData.gCacheIR)
Ma.GenPreMakefileHash(GlobalData.gCacheIR)
if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR):
# SourceFileList calling sequence impact the makefile string sequence.
# Create cached SourceFileList here to unify its calling sequence for both
# CanSkipbyPreMakeCache and CreateCodeFile/CreateMakeFile.
RetVal = Ma.SourceFileList
if GlobalData.gUseHashCache and not GlobalData.gBinCacheDest and CommandTarget in [None, "", "all"]:
try:
CacheResult = Ma.CanSkipbyPreMakeCache()
except:
CacheResult = False
traceback.print_exc(file=sys.stdout)
self.feedback_q.put(taskname)
if CacheResult:
self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "PreMakeCache", True))
continue
else:
self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "PreMakeCache", False))
Ma.CreateCodeFile(False)
Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.Path, Ma.Arch),[]))
if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]:
Ma.GenMakeHeaderFilesHash(GlobalData.gCacheIR)
Ma.GenMakeHash(GlobalData.gCacheIR)
if Ma.CanSkipbyMakeCache(GlobalData.gCacheIR):
try:
CacheResult = Ma.CanSkipbyMakeCache()
except:
CacheResult = False
traceback.print_exc(file=sys.stdout)
self.feedback_q.put(taskname)
if CacheResult:
self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "MakeCache", True))
continue
else:
Ma.PrintFirstMakeCacheMissFile(GlobalData.gCacheIR)
self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "MakeCache", False))
except Empty:
pass
except:
@ -258,6 +289,8 @@ class AutoGenWorkerInProcess(mp.Process):
self.feedback_q.put(taskname)
finally:
self.feedback_q.put("Done")
self.cache_q.put("CacheDone")
def printStatus(self):
print("Processs ID: %d Run %d modules in AutoGen " % (os.getpid(),len(AutoGen.Cache())))
print("Processs ID: %d Run %d modules in AutoGenInfo " % (os.getpid(),len(AutoGenInfo.GetCache())))

2
BaseTools/Source/Python/AutoGen/DataPipe.py
View File

@ -159,6 +159,8 @@ class MemoryDataPipe(DataPipe):
self.DataContainer = {"LogLevel": EdkLogger.GetLevel()}
self.DataContainer = {"UseHashCache":GlobalData.gUseHashCache}
self.DataContainer = {"BinCacheSource":GlobalData.gBinCacheSource}
self.DataContainer = {"BinCacheDest":GlobalData.gBinCacheDest}

1226
BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
View File

File diff suppressed because it is too large Load Diff

64
BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
View File

@ -23,6 +23,7 @@ from Common.StringUtils import NormPath
from Common.BuildToolError import *
from Common.DataType import *
from Common.Misc import *
import json
## Regular expression for splitting Dependency Expression string into tokens
gDepexTokenPattern = re.compile("(\(|\)|\w+| \S+\.inf)")
@ -127,7 +128,7 @@ class WorkspaceAutoGen(AutoGen):
self.CreateBuildOptionsFile()
self.CreatePcdTokenNumberFile()
self.CreateModuleHashInfo()
self.GeneratePlatformLevelHash()
#
# Merge Arch
@ -528,12 +529,12 @@ class WorkspaceAutoGen(AutoGen):
PcdTokenNumber += str(Pa.PcdTokenNumber[Pcd.TokenCName, Pcd.TokenSpaceGuidCName])
SaveFileOnChange(os.path.join(self.BuildDir, 'PcdTokenNumber'), PcdTokenNumber, False)
def CreateModuleHashInfo(self):
def GeneratePlatformLevelHash(self):
#
# Get set of workspace metafiles
#
AllWorkSpaceMetaFiles = self._GetMetaFiles(self.BuildTarget, self.ToolChain)
AllWorkSpaceMetaFileList = sorted(AllWorkSpaceMetaFiles, key=lambda x: str(x))
#
# Retrieve latest modified time of all metafiles
#
@ -544,16 +545,35 @@ class WorkspaceAutoGen(AutoGen):
self._SrcTimeStamp = SrcTimeStamp
if GlobalData.gUseHashCache:
FileList = []
m = hashlib.md5()
for files in AllWorkSpaceMetaFiles:
if files.endswith('.dec'):
for file in AllWorkSpaceMetaFileList:
if file.endswith('.dec'):
continue
f = open(files, 'rb')
f = open(file, 'rb')
Content = f.read()
f.close()
m.update(Content)
SaveFileOnChange(os.path.join(self.BuildDir, 'AutoGen.hash'), m.hexdigest(), False)
GlobalData.gPlatformHash = m.hexdigest()
FileList.append((str(file), hashlib.md5(Content).hexdigest()))
HashDir = path.join(self.BuildDir, "Hash_Platform")
HashFile = path.join(HashDir, 'Platform.hash.' + m.hexdigest())
SaveFileOnChange(HashFile, m.hexdigest(), False)
HashChainFile = path.join(HashDir, 'Platform.hashchain.' + m.hexdigest())
GlobalData.gPlatformHashFile = HashChainFile
try:
with open(HashChainFile, 'w') as f:
json.dump(FileList, f, indent=2)
except:
EdkLogger.quiet("[cache warning]: fail to save hashchain file:%s" % HashChainFile)
if GlobalData.gBinCacheDest:
# Copy platform hash files to cache destination
FileDir = path.join(GlobalData.gBinCacheDest, self.OutputDir, self.BuildTarget + "_" + self.ToolChain, "Hash_Platform")
CacheFileDir = FileDir
CreateDirectory(CacheFileDir)
CopyFileOnChange(HashFile, CacheFileDir)
CopyFileOnChange(HashChainFile, CacheFileDir)
#
# Write metafile list to build directory
@ -564,7 +584,7 @@ class WorkspaceAutoGen(AutoGen):
if not os.path.exists(self.BuildDir):
os.makedirs(self.BuildDir)
with open(os.path.join(self.BuildDir, 'AutoGen'), 'w+') as file:
for f in AllWorkSpaceMetaFiles:
for f in AllWorkSpaceMetaFileList:
print(f, file=file)
return True
@ -572,15 +592,16 @@ class WorkspaceAutoGen(AutoGen):
if Pkg.PackageName in GlobalData.gPackageHash:
return
PkgDir = os.path.join(self.BuildDir, Pkg.Arch, Pkg.PackageName)
PkgDir = os.path.join(self.BuildDir, Pkg.Arch, "Hash_Pkg", Pkg.PackageName)
CreateDirectory(PkgDir)
HashFile = os.path.join(PkgDir, Pkg.PackageName + '.hash')
FileList = []
m = hashlib.md5()
# Get .dec file's hash value
f = open(Pkg.MetaFile.Path, 'rb')
Content = f.read()
f.close()
m.update(Content)
FileList.append((str(Pkg.MetaFile.Path), hashlib.md5(Content).hexdigest()))
# Get include files hash value
if Pkg.Includes:
for inc in sorted(Pkg.Includes, key=lambda x: str(x)):
@ -591,9 +612,28 @@ class WorkspaceAutoGen(AutoGen):
Content = f.read()
f.close()
m.update(Content)
SaveFileOnChange(HashFile, m.hexdigest(), False)
FileList.append((str(File_Path), hashlib.md5(Content).hexdigest()))
GlobalData.gPackageHash[Pkg.PackageName] = m.hexdigest()
HashDir = PkgDir
HashFile = path.join(HashDir, Pkg.PackageName + '.hash.' + m.hexdigest())
SaveFileOnChange(HashFile, m.hexdigest(), False)
HashChainFile = path.join(HashDir, Pkg.PackageName + '.hashchain.' + m.hexdigest())
GlobalData.gPackageHashFile[(Pkg.PackageName, Pkg.Arch)] = HashChainFile
try:
with open(HashChainFile, 'w') as f:
json.dump(FileList, f, indent=2)
except:
EdkLogger.quiet("[cache warning]: fail to save hashchain file:%s" % HashChainFile)
if GlobalData.gBinCacheDest:
# Copy Pkg hash files to cache destination dir
FileDir = path.join(GlobalData.gBinCacheDest, self.OutputDir, self.BuildTarget + "_" + self.ToolChain, Pkg.Arch, "Hash_Pkg", Pkg.PackageName)
CacheFileDir = FileDir
CreateDirectory(CacheFileDir)
CopyFileOnChange(HashFile, CacheFileDir)
CopyFileOnChange(HashChainFile, CacheFileDir)
def _GetMetaFiles(self, Target, Toolchain):
AllWorkSpaceMetaFiles = set()
#