sravan/system76-edk2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

BaseTools: Leverage compiler output to optimize binary cache

Browse Source 
Redesign the binary cache and bases on the compiler to
output the dependency header files info for every module.
The binary cache will directly consume the dependency header
files info and doesn't parse the C source code by iteself.
Also redesign the dependency files list format for module
and try to share the common lib hash result as more as
possible in local process. Remove the unnecessary share data
access across multiprocessing.

Signed-off-by: Steven Shi <steven.shi@intel.com>

Cc: Liming Gao <liming.gao@intel.com>
Cc: Bob Feng <bob.c.feng@intel.com>
Reviewed-by: Bob Feng <bob.c.feng@intel.com>
This commit is contained in:
Steven Shi
2019-11-19 16:17:00 +08:00
committed by mergify[bot]
parent 3bfbc91507
commit fc8b8deac2
6 changed files with 786 additions and 894 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
BaseTools/Source/Python/AutoGen/AutoGenWorker.py
View File

@ -128,12 +128,27 @@ class AutoGenManager(threading.Thread):
clearQ(taskq) clearQ(taskq)
clearQ(self.feedback_q) clearQ(self.feedback_q)
clearQ(logq) clearQ(logq)
# Copy the cache queue itmes to parent thread before clear
cacheq = self.autogen_workers[0].cache_q
try:
cache_num = 0
while True:
item = cacheq.get()
if item == "CacheDone":
cache_num += 1
else:
GlobalData.gModuleAllCacheStatus.add(item)
if cache_num == len(self.autogen_workers):
break
except:
print ("cache_q error")
def TerminateWorkers(self): def TerminateWorkers(self):
self.error_event.set() self.error_event.set()
def kill(self): def kill(self):
self.feedback_q.put(None) self.feedback_q.put(None)
class AutoGenWorkerInProcess(mp.Process): class AutoGenWorkerInProcess(mp.Process):
def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_lock,cache_lock,share_data,log_q,error_event): def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_lock,cache_q,log_q,error_event):
mp.Process.__init__(self) mp.Process.__init__(self)
self.module_queue = module_queue self.module_queue = module_queue
self.data_pipe_file_path =data_pipe_file_path self.data_pipe_file_path =data_pipe_file_path
@ -141,8 +156,7 @@ class AutoGenWorkerInProcess(mp.Process):
self.feedback_q = feedback_q self.feedback_q = feedback_q
self.PlatformMetaFileSet = {} self.PlatformMetaFileSet = {}
self.file_lock = file_lock self.file_lock = file_lock
self.cache_lock = cache_lock self.cache_q = cache_q
self.share_data = share_data
self.log_q = log_q self.log_q = log_q
self.error_event = error_event self.error_event = error_event
def GetPlatformMetaFile(self,filepath,root): def GetPlatformMetaFile(self,filepath,root):
@ -184,12 +198,19 @@ class AutoGenWorkerInProcess(mp.Process):
GlobalData.gDisableIncludePathCheck = False GlobalData.gDisableIncludePathCheck = False
GlobalData.gFdfParser = self.data_pipe.Get("FdfParser") GlobalData.gFdfParser = self.data_pipe.Get("FdfParser")
GlobalData.gDatabasePath = self.data_pipe.Get("DatabasePath") GlobalData.gDatabasePath = self.data_pipe.Get("DatabasePath")
GlobalData.gUseHashCache = self.data_pipe.Get("UseHashCache")
GlobalData.gBinCacheSource = self.data_pipe.Get("BinCacheSource") GlobalData.gBinCacheSource = self.data_pipe.Get("BinCacheSource")
GlobalData.gBinCacheDest = self.data_pipe.Get("BinCacheDest") GlobalData.gBinCacheDest = self.data_pipe.Get("BinCacheDest")
GlobalData.gCacheIR = self.share_data GlobalData.gPlatformHashFile = self.data_pipe.Get("PlatformHashFile")
GlobalData.gModulePreMakeCacheStatus = dict()
GlobalData.gModuleMakeCacheStatus = dict()
GlobalData.gHashChainStatus = dict()
GlobalData.gCMakeHashFile = dict()
GlobalData.gModuleHashFile = dict()
GlobalData.gFileHashDict = dict()
GlobalData.gEnableGenfdsMultiThread = self.data_pipe.Get("EnableGenfdsMultiThread") GlobalData.gEnableGenfdsMultiThread = self.data_pipe.Get("EnableGenfdsMultiThread")
GlobalData.file_lock = self.file_lock GlobalData.file_lock = self.file_lock
GlobalData.cache_lock = self.cache_lock
CommandTarget = self.data_pipe.Get("CommandTarget") CommandTarget = self.data_pipe.Get("CommandTarget")
pcd_from_build_option = [] pcd_from_build_option = []
for pcd_tuple in self.data_pipe.Get("BuildOptPcd"): for pcd_tuple in self.data_pipe.Get("BuildOptPcd"):
@ -205,10 +226,6 @@ class AutoGenWorkerInProcess(mp.Process):
GlobalData.FfsCmd = FfsCmd GlobalData.FfsCmd = FfsCmd
PlatformMetaFile = self.GetPlatformMetaFile(self.data_pipe.Get("P_Info").get("ActivePlatform"), PlatformMetaFile = self.GetPlatformMetaFile(self.data_pipe.Get("P_Info").get("ActivePlatform"),
self.data_pipe.Get("P_Info").get("WorkspaceDir")) self.data_pipe.Get("P_Info").get("WorkspaceDir"))
libConstPcd = self.data_pipe.Get("LibConstPcd")
Refes = self.data_pipe.Get("REFS")
GlobalData.libConstPcd = libConstPcd
GlobalData.Refes = Refes
while True: while True:
if self.module_queue.empty(): if self.module_queue.empty():
break break
@ -230,27 +247,41 @@ class AutoGenWorkerInProcess(mp.Process):
toolchain = self.data_pipe.Get("P_Info").get("ToolChain") toolchain = self.data_pipe.Get("P_Info").get("ToolChain")
Ma = ModuleAutoGen(self.Wa,module_metafile,target,toolchain,arch,PlatformMetaFile,self.data_pipe) Ma = ModuleAutoGen(self.Wa,module_metafile,target,toolchain,arch,PlatformMetaFile,self.data_pipe)
Ma.IsLibrary = IsLib Ma.IsLibrary = IsLib
if IsLib: # SourceFileList calling sequence impact the makefile string sequence.
if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in libConstPcd: # Create cached SourceFileList here to unify its calling sequence for both
Ma.ConstPcd = libConstPcd[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)] # CanSkipbyPreMakeCache and CreateCodeFile/CreateMakeFile.
if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in Refes: RetVal = Ma.SourceFileList
Ma.ReferenceModules = Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)] if GlobalData.gUseHashCache and not GlobalData.gBinCacheDest and CommandTarget in [None, "", "all"]:
if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]: try:
Ma.GenModuleFilesHash(GlobalData.gCacheIR) CacheResult = Ma.CanSkipbyPreMakeCache()
Ma.GenPreMakefileHash(GlobalData.gCacheIR) except:
if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR): CacheResult = False
traceback.print_exc(file=sys.stdout)
self.feedback_q.put(taskname)
if CacheResult:
self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "PreMakeCache", True))
continue continue
else:
self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "PreMakeCache", False))
Ma.CreateCodeFile(False) Ma.CreateCodeFile(False)
Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.Path, Ma.Arch),[])) Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.Path, Ma.Arch),[]))
if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]: if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]:
Ma.GenMakeHeaderFilesHash(GlobalData.gCacheIR) try:
Ma.GenMakeHash(GlobalData.gCacheIR) CacheResult = Ma.CanSkipbyMakeCache()
if Ma.CanSkipbyMakeCache(GlobalData.gCacheIR): except:
CacheResult = False
traceback.print_exc(file=sys.stdout)
self.feedback_q.put(taskname)
if CacheResult:
self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "MakeCache", True))
continue continue
else: else:
Ma.PrintFirstMakeCacheMissFile(GlobalData.gCacheIR) self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "MakeCache", False))
except Empty: except Empty:
pass pass
except: except:
@ -258,6 +289,8 @@ class AutoGenWorkerInProcess(mp.Process):
self.feedback_q.put(taskname) self.feedback_q.put(taskname)
finally: finally:
self.feedback_q.put("Done") self.feedback_q.put("Done")
self.cache_q.put("CacheDone")
def printStatus(self): def printStatus(self):
print("Processs ID: %d Run %d modules in AutoGen " % (os.getpid(),len(AutoGen.Cache()))) print("Processs ID: %d Run %d modules in AutoGen " % (os.getpid(),len(AutoGen.Cache())))
print("Processs ID: %d Run %d modules in AutoGenInfo " % (os.getpid(),len(AutoGenInfo.GetCache()))) print("Processs ID: %d Run %d modules in AutoGenInfo " % (os.getpid(),len(AutoGenInfo.GetCache())))

2
BaseTools/Source/Python/AutoGen/DataPipe.py
View File

@ -159,6 +159,8 @@ class MemoryDataPipe(DataPipe):
self.DataContainer = {"LogLevel": EdkLogger.GetLevel()} self.DataContainer = {"LogLevel": EdkLogger.GetLevel()}
self.DataContainer = {"UseHashCache":GlobalData.gUseHashCache}
self.DataContainer = {"BinCacheSource":GlobalData.gBinCacheSource} self.DataContainer = {"BinCacheSource":GlobalData.gBinCacheSource}
self.DataContainer = {"BinCacheDest":GlobalData.gBinCacheDest} self.DataContainer = {"BinCacheDest":GlobalData.gBinCacheDest}

1226
BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
View File

File diff suppressed because it is too large Load Diff

64
BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
View File

@ -23,6 +23,7 @@ from Common.StringUtils import NormPath
from Common.BuildToolError import * from Common.BuildToolError import *
from Common.DataType import * from Common.DataType import *
from Common.Misc import * from Common.Misc import *
import json
## Regular expression for splitting Dependency Expression string into tokens ## Regular expression for splitting Dependency Expression string into tokens
gDepexTokenPattern = re.compile("(\(|\)|\w+| \S+\.inf)") gDepexTokenPattern = re.compile("(\(|\)|\w+| \S+\.inf)")
@ -127,7 +128,7 @@ class WorkspaceAutoGen(AutoGen):
self.CreateBuildOptionsFile() self.CreateBuildOptionsFile()
self.CreatePcdTokenNumberFile() self.CreatePcdTokenNumberFile()
self.CreateModuleHashInfo() self.GeneratePlatformLevelHash()
# #
# Merge Arch # Merge Arch
@ -528,12 +529,12 @@ class WorkspaceAutoGen(AutoGen):
PcdTokenNumber += str(Pa.PcdTokenNumber[Pcd.TokenCName, Pcd.TokenSpaceGuidCName]) PcdTokenNumber += str(Pa.PcdTokenNumber[Pcd.TokenCName, Pcd.TokenSpaceGuidCName])
SaveFileOnChange(os.path.join(self.BuildDir, 'PcdTokenNumber'), PcdTokenNumber, False) SaveFileOnChange(os.path.join(self.BuildDir, 'PcdTokenNumber'), PcdTokenNumber, False)
def CreateModuleHashInfo(self): def GeneratePlatformLevelHash(self):
# #
# Get set of workspace metafiles # Get set of workspace metafiles
# #
AllWorkSpaceMetaFiles = self._GetMetaFiles(self.BuildTarget, self.ToolChain) AllWorkSpaceMetaFiles = self._GetMetaFiles(self.BuildTarget, self.ToolChain)
AllWorkSpaceMetaFileList = sorted(AllWorkSpaceMetaFiles, key=lambda x: str(x))
# #
# Retrieve latest modified time of all metafiles # Retrieve latest modified time of all metafiles
# #
@ -544,16 +545,35 @@ class WorkspaceAutoGen(AutoGen):
self._SrcTimeStamp = SrcTimeStamp self._SrcTimeStamp = SrcTimeStamp
if GlobalData.gUseHashCache: if GlobalData.gUseHashCache:
FileList = []
m = hashlib.md5() m = hashlib.md5()
for files in AllWorkSpaceMetaFiles: for file in AllWorkSpaceMetaFileList:
if files.endswith('.dec'): if file.endswith('.dec'):
continue continue
f = open(files, 'rb') f = open(file, 'rb')
Content = f.read() Content = f.read()
f.close() f.close()
m.update(Content) m.update(Content)
SaveFileOnChange(os.path.join(self.BuildDir, 'AutoGen.hash'), m.hexdigest(), False) FileList.append((str(file), hashlib.md5(Content).hexdigest()))
GlobalData.gPlatformHash = m.hexdigest()
HashDir = path.join(self.BuildDir, "Hash_Platform")
HashFile = path.join(HashDir, 'Platform.hash.' + m.hexdigest())
SaveFileOnChange(HashFile, m.hexdigest(), False)
HashChainFile = path.join(HashDir, 'Platform.hashchain.' + m.hexdigest())
GlobalData.gPlatformHashFile = HashChainFile
try:
with open(HashChainFile, 'w') as f:
json.dump(FileList, f, indent=2)
except:
EdkLogger.quiet("[cache warning]: fail to save hashchain file:%s" % HashChainFile)
if GlobalData.gBinCacheDest:
# Copy platform hash files to cache destination
FileDir = path.join(GlobalData.gBinCacheDest, self.OutputDir, self.BuildTarget + "_" + self.ToolChain, "Hash_Platform")
CacheFileDir = FileDir
CreateDirectory(CacheFileDir)
CopyFileOnChange(HashFile, CacheFileDir)
CopyFileOnChange(HashChainFile, CacheFileDir)
# #
# Write metafile list to build directory # Write metafile list to build directory
@ -564,7 +584,7 @@ class WorkspaceAutoGen(AutoGen):
if not os.path.exists(self.BuildDir): if not os.path.exists(self.BuildDir):
os.makedirs(self.BuildDir) os.makedirs(self.BuildDir)
with open(os.path.join(self.BuildDir, 'AutoGen'), 'w+') as file: with open(os.path.join(self.BuildDir, 'AutoGen'), 'w+') as file:
for f in AllWorkSpaceMetaFiles: for f in AllWorkSpaceMetaFileList:
print(f, file=file) print(f, file=file)
return True return True
@ -572,15 +592,16 @@ class WorkspaceAutoGen(AutoGen):
if Pkg.PackageName in GlobalData.gPackageHash: if Pkg.PackageName in GlobalData.gPackageHash:
return return
PkgDir = os.path.join(self.BuildDir, Pkg.Arch, Pkg.PackageName) PkgDir = os.path.join(self.BuildDir, Pkg.Arch, "Hash_Pkg", Pkg.PackageName)
CreateDirectory(PkgDir) CreateDirectory(PkgDir)
HashFile = os.path.join(PkgDir, Pkg.PackageName + '.hash') FileList = []
m = hashlib.md5() m = hashlib.md5()
# Get .dec file's hash value # Get .dec file's hash value
f = open(Pkg.MetaFile.Path, 'rb') f = open(Pkg.MetaFile.Path, 'rb')
Content = f.read() Content = f.read()
f.close() f.close()
m.update(Content) m.update(Content)
FileList.append((str(Pkg.MetaFile.Path), hashlib.md5(Content).hexdigest()))
# Get include files hash value # Get include files hash value
if Pkg.Includes: if Pkg.Includes:
for inc in sorted(Pkg.Includes, key=lambda x: str(x)): for inc in sorted(Pkg.Includes, key=lambda x: str(x)):
@ -591,9 +612,28 @@ class WorkspaceAutoGen(AutoGen):
Content = f.read() Content = f.read()
f.close() f.close()
m.update(Content) m.update(Content)
SaveFileOnChange(HashFile, m.hexdigest(), False) FileList.append((str(File_Path), hashlib.md5(Content).hexdigest()))
GlobalData.gPackageHash[Pkg.PackageName] = m.hexdigest() GlobalData.gPackageHash[Pkg.PackageName] = m.hexdigest()
HashDir = PkgDir
HashFile = path.join(HashDir, Pkg.PackageName + '.hash.' + m.hexdigest())
SaveFileOnChange(HashFile, m.hexdigest(), False)
HashChainFile = path.join(HashDir, Pkg.PackageName + '.hashchain.' + m.hexdigest())
GlobalData.gPackageHashFile[(Pkg.PackageName, Pkg.Arch)] = HashChainFile
try:
with open(HashChainFile, 'w') as f:
json.dump(FileList, f, indent=2)
except:
EdkLogger.quiet("[cache warning]: fail to save hashchain file:%s" % HashChainFile)
if GlobalData.gBinCacheDest:
# Copy Pkg hash files to cache destination dir
FileDir = path.join(GlobalData.gBinCacheDest, self.OutputDir, self.BuildTarget + "_" + self.ToolChain, Pkg.Arch, "Hash_Pkg", Pkg.PackageName)
CacheFileDir = FileDir
CreateDirectory(CacheFileDir)
CopyFileOnChange(HashFile, CacheFileDir)
CopyFileOnChange(HashChainFile, CacheFileDir)
def _GetMetaFiles(self, Target, Toolchain): def _GetMetaFiles(self, Target, Toolchain):
AllWorkSpaceMetaFiles = set() AllWorkSpaceMetaFiles = set()
# #

35
BaseTools/Source/Python/Common/GlobalData.py
View File

@ -104,29 +104,20 @@ gUseHashCache = None
gBinCacheDest = None gBinCacheDest = None
gBinCacheSource = None gBinCacheSource = None
gPlatformHash = None gPlatformHash = None
gPackageHash = {} gPlatformHashFile = None
gModuleHash = {} gPackageHash = None
gPackageHashFile = None
gModuleHashFile = None
gCMakeHashFile = None
gHashChainStatus = None
gModulePreMakeCacheStatus = None
gModuleMakeCacheStatus = None
gFileHashDict = None
gModuleAllCacheStatus = None
gModuleCacheHit = None
gEnableGenfdsMultiThread = True gEnableGenfdsMultiThread = True
gSikpAutoGenCache = set() gSikpAutoGenCache = set()
# Dictionary for tracking Module build status as success or failure
# Top Dict: Key: Arch Type Value: Dictionary
# Second Dict: Key: AutoGen Obj Value: 'SUCCESS'\'FAIL'\'FAIL_METAFILE'
gModuleBuildTracking = dict()
# Dictionary of booleans that dictate whether a module or
# library can be skiped
# Top Dict: Key: Arch Type Value: Dictionary
# Second Dict: Key: Module\Library Name Value: True\False
gBuildHashSkipTracking = dict()
# Common dictionary to share module cache intermediate result and state
gCacheIR = None
# Common lock for the module cache intermediate data
cache_lock = None
# Common lock for the file access in multiple process AutoGens # Common lock for the file access in multiple process AutoGens
file_lock = None file_lock = None
# Common dictionary to share platform libraries' constant Pcd
libConstPcd = None
# Common dictionary to share platform libraries' reference info
Refes = None

276
BaseTools/Source/Python/build/build.py
View File

@ -620,7 +620,7 @@ class BuildTask:
# #
def AddDependency(self, Dependency): def AddDependency(self, Dependency):
for Dep in Dependency: for Dep in Dependency:
if not Dep.BuildObject.IsBinaryModule and not Dep.BuildObject.CanSkipbyCache(GlobalData.gCacheIR): if not Dep.BuildObject.IsBinaryModule and not Dep.BuildObject.CanSkipbyCache(GlobalData.gModuleCacheHit):
self.DependencyList.append(BuildTask.New(Dep)) # BuildTask list self.DependencyList.append(BuildTask.New(Dep)) # BuildTask list
## The thread wrapper of LaunchCommand function ## The thread wrapper of LaunchCommand function
@ -633,10 +633,13 @@ class BuildTask:
self.BuildItem.BuildObject.BuildTime = LaunchCommand(Command, WorkingDir,self.BuildItem.BuildObject) self.BuildItem.BuildObject.BuildTime = LaunchCommand(Command, WorkingDir,self.BuildItem.BuildObject)
self.CompleteFlag = True self.CompleteFlag = True
# Run hash operation post dependency, to account for libs # Run hash operation post dependency to account for libs
if GlobalData.gUseHashCache and self.BuildItem.BuildObject.IsLibrary: # Run if --hash or --binary-destination
HashFile = path.join(self.BuildItem.BuildObject.BuildDir, self.BuildItem.BuildObject.Name + ".hash") if GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:
SaveFileOnChange(HashFile, self.BuildItem.BuildObject.GenModuleHash(), True) self.BuildItem.BuildObject.GenModuleHash()
if GlobalData.gBinCacheDest:
self.BuildItem.BuildObject.GenCMakeHash()
except: except:
# #
# TRICK: hide the output of threads left running, so that the user can # TRICK: hide the output of threads left running, so that the user can
@ -653,14 +656,6 @@ class BuildTask:
BuildTask._ErrorMessage = "%s broken\n %s [%s]" % \ BuildTask._ErrorMessage = "%s broken\n %s [%s]" % \
(threading.currentThread().getName(), Command, WorkingDir) (threading.currentThread().getName(), Command, WorkingDir)
# Set the value used by hash invalidation flow in GlobalData.gModuleBuildTracking to 'SUCCESS'
# If Module or Lib is being tracked, it did not fail header check test, and built successfully
if (self.BuildItem.BuildObject in GlobalData.gModuleBuildTracking and
GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject] != 'FAIL_METAFILE' and
not BuildTask._ErrorFlag.isSet()
):
GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject] = 'SUCCESS'
# indicate there's a thread is available for another build task # indicate there's a thread is available for another build task
BuildTask._RunningQueueLock.acquire() BuildTask._RunningQueueLock.acquire()
BuildTask._RunningQueue.pop(self.BuildItem) BuildTask._RunningQueue.pop(self.BuildItem)
@ -835,11 +830,20 @@ class Build():
self.AutoGenMgr = None self.AutoGenMgr = None
EdkLogger.info("") EdkLogger.info("")
os.chdir(self.WorkspaceDir) os.chdir(self.WorkspaceDir)
GlobalData.gCacheIR = Manager().dict()
self.log_q = log_q self.log_q = log_q
GlobalData.file_lock = mp.Lock() GlobalData.file_lock = mp.Lock()
GlobalData.cache_lock = mp.Lock() # Init cache data for local only
def StartAutoGen(self,mqueue, DataPipe,SkipAutoGen,PcdMaList,share_data): GlobalData.gPackageHashFile = dict()
GlobalData.gModulePreMakeCacheStatus = dict()
GlobalData.gModuleMakeCacheStatus = dict()
GlobalData.gHashChainStatus = dict()
GlobalData.gCMakeHashFile = dict()
GlobalData.gModuleHashFile = dict()
GlobalData.gFileHashDict = dict()
GlobalData.gModuleAllCacheStatus = set()
GlobalData.gModuleCacheHit = set()
def StartAutoGen(self,mqueue, DataPipe,SkipAutoGen,PcdMaList,cqueue):
try: try:
if SkipAutoGen: if SkipAutoGen:
return True,0 return True,0
@ -849,29 +853,27 @@ class Build():
if FfsCmd is None: if FfsCmd is None:
FfsCmd = {} FfsCmd = {}
GlobalData.FfsCmd = FfsCmd GlobalData.FfsCmd = FfsCmd
GlobalData.libConstPcd = DataPipe.Get("LibConstPcd") auto_workers = [AutoGenWorkerInProcess(mqueue,DataPipe.dump_file,feedback_q,GlobalData.file_lock,cqueue,self.log_q,error_event) for _ in range(self.ThreadNumber)]
GlobalData.Refes = DataPipe.Get("REFS")
auto_workers = [AutoGenWorkerInProcess(mqueue,DataPipe.dump_file,feedback_q,GlobalData.file_lock,GlobalData.cache_lock,share_data,self.log_q,error_event) for _ in range(self.ThreadNumber)]
self.AutoGenMgr = AutoGenManager(auto_workers,feedback_q,error_event) self.AutoGenMgr = AutoGenManager(auto_workers,feedback_q,error_event)
self.AutoGenMgr.start() self.AutoGenMgr.start()
for w in auto_workers: for w in auto_workers:
w.start() w.start()
if PcdMaList is not None: if PcdMaList is not None:
for PcdMa in PcdMaList: for PcdMa in PcdMaList:
if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]: # SourceFileList calling sequence impact the makefile string sequence.
PcdMa.GenModuleFilesHash(share_data) # Create cached SourceFileList here to unify its calling sequence for both
PcdMa.GenPreMakefileHash(share_data) # CanSkipbyPreMakeCache and CreateCodeFile/CreateMakeFile.
if PcdMa.CanSkipbyPreMakefileCache(share_data): RetVal = PcdMa.SourceFileList
continue # Force cache miss for PCD driver
if GlobalData.gUseHashCache and not GlobalData.gBinCacheDest and self.Target in [None, "", "all"]:
cqueue.put((PcdMa.MetaFile.Path, PcdMa.Arch, "PreMakeCache", False))
PcdMa.CreateCodeFile(False) PcdMa.CreateCodeFile(False)
PcdMa.CreateMakeFile(False,GenFfsList = DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.Path, PcdMa.Arch),[])) PcdMa.CreateMakeFile(False,GenFfsList = DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.Path, PcdMa.Arch),[]))
# Force cache miss for PCD driver
if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]: if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
PcdMa.GenMakeHeaderFilesHash(share_data) cqueue.put((PcdMa.MetaFile.Path, PcdMa.Arch, "MakeCache", False))
PcdMa.GenMakeHash(share_data)
if PcdMa.CanSkipbyMakeCache(share_data):
continue
self.AutoGenMgr.join() self.AutoGenMgr.join()
rt = self.AutoGenMgr.Status rt = self.AutoGenMgr.Status
@ -1182,38 +1184,6 @@ class Build():
EdkLogger.error("Postbuild", POSTBUILD_ERROR, 'Postbuild process is not success!') EdkLogger.error("Postbuild", POSTBUILD_ERROR, 'Postbuild process is not success!')
EdkLogger.info("\n- Postbuild Done -\n") EdkLogger.info("\n- Postbuild Done -\n")
## Error handling for hash feature
#
# On BuildTask error, iterate through the Module Build tracking
# dictionary to determine wheather a module failed to build. Invalidate
# the hash associated with that module by removing it from storage.
#
#
def invalidateHash(self):
# Only for hashing feature
if not GlobalData.gUseHashCache:
return
# GlobalData.gModuleBuildTracking contains only modules or libs that cannot be skipped by hash
for Ma in GlobalData.gModuleBuildTracking:
# Skip invalidating for Successful Module/Lib builds
if GlobalData.gModuleBuildTracking[Ma] == 'SUCCESS':
continue
# The module failed to build, failed to start building, or failed the header check test from this point on
# Remove .hash from build
ModuleHashFile = os.path.join(Ma.BuildDir, Ma.Name + ".hash")
if os.path.exists(ModuleHashFile):
os.remove(ModuleHashFile)
# Remove .hash file from cache
if GlobalData.gBinCacheDest:
FileDir = os.path.join(GlobalData.gBinCacheDest, Ma.PlatformInfo.OutputDir, Ma.BuildTarget + "_" + Ma.ToolChain, Ma.Arch, Ma.SourceDir, Ma.MetaFile.BaseName)
HashFile = os.path.join(FileDir, Ma.Name + '.hash')
if os.path.exists(HashFile):
os.remove(HashFile)
## Build a module or platform ## Build a module or platform
# #
# Create autogen code and makefile for a module or platform, and the launch # Create autogen code and makefile for a module or platform, and the launch
@ -1251,7 +1221,8 @@ class Build():
self.Progress.Start("Generating makefile and code") self.Progress.Start("Generating makefile and code")
data_pipe_file = os.path.join(AutoGenObject.BuildDir, "GlobalVar_%s_%s.bin" % (str(AutoGenObject.Guid),AutoGenObject.Arch)) data_pipe_file = os.path.join(AutoGenObject.BuildDir, "GlobalVar_%s_%s.bin" % (str(AutoGenObject.Guid),AutoGenObject.Arch))
AutoGenObject.DataPipe.dump(data_pipe_file) AutoGenObject.DataPipe.dump(data_pipe_file)
autogen_rt,errorcode = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList, GlobalData.gCacheIR) cqueue = mp.Queue()
autogen_rt,errorcode = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList, cqueue)
AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt") AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
with open(AutoGenIdFile,"w") as fw: with open(AutoGenIdFile,"w") as fw:
fw.write("Arch=%s\n" % "|".join((AutoGenObject.Workspace.ArchList))) fw.write("Arch=%s\n" % "|".join((AutoGenObject.Workspace.ArchList)))
@ -1292,7 +1263,11 @@ class Build():
LaunchCommand(BuildCommand, AutoGenObject.MakeFileDir) LaunchCommand(BuildCommand, AutoGenObject.MakeFileDir)
self.CreateAsBuiltInf() self.CreateAsBuiltInf()
if GlobalData.gBinCacheDest: if GlobalData.gBinCacheDest:
self.UpdateBuildCache() self.GenDestCache()
elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:
# Only for --hash
# Update PreMakeCacheChain files
self.GenLocalPreMakeCache()
self.BuildModules = [] self.BuildModules = []
return True return True
@ -1326,7 +1301,11 @@ class Build():
LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir,ModAutoGen) LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir,ModAutoGen)
self.CreateAsBuiltInf() self.CreateAsBuiltInf()
if GlobalData.gBinCacheDest: if GlobalData.gBinCacheDest:
self.UpdateBuildCache() self.GenDestCache()
elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:
# Only for --hash
# Update PreMakeCacheChain files
self.GenLocalPreMakeCache()
self.BuildModules = [] self.BuildModules = []
return True return True
@ -1422,7 +1401,11 @@ class Build():
AutoGenObject.BuildTime = LaunchCommand(BuildCommand, AutoGenObject.MakeFileDir) AutoGenObject.BuildTime = LaunchCommand(BuildCommand, AutoGenObject.MakeFileDir)
self.CreateAsBuiltInf() self.CreateAsBuiltInf()
if GlobalData.gBinCacheDest: if GlobalData.gBinCacheDest:
self.UpdateBuildCache() self.GenDestCache()
elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:
# Only for --hash
# Update PreMakeCacheChain files
self.GenLocalPreMakeCache()
self.BuildModules = [] self.BuildModules = []
return True return True
@ -1870,13 +1853,7 @@ class Build():
if GlobalData.gEnableGenfdsMultiThread and self.Fdf: if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
CmdListDict = self._GenFfsCmd(Wa.ArchList) CmdListDict = self._GenFfsCmd(Wa.ArchList)
# Add Platform and Package level hash in share_data for module hash calculation later
if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest:
GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash
for PkgName in GlobalData.gPackageHash.keys():
GlobalData.gCacheIR[(PkgName, 'PackageHash')] = GlobalData.gPackageHash[PkgName]
GlobalData.file_lock = mp.Lock() GlobalData.file_lock = mp.Lock()
GlobalData.cache_lock = mp.Lock()
GlobalData.FfsCmd = CmdListDict GlobalData.FfsCmd = CmdListDict
self.Progress.Stop("done!") self.Progress.Stop("done!")
@ -1888,8 +1865,6 @@ class Build():
AutoGenStart = time.time() AutoGenStart = time.time()
GlobalData.gGlobalDefines['ARCH'] = Arch GlobalData.gGlobalDefines['ARCH'] = Arch
Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch) Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)
GlobalData.libConstPcd = Pa.DataPipe.Get("LibConstPcd")
GlobalData.Refes = Pa.DataPipe.Get("REFS")
for Module in Pa.Platform.Modules: for Module in Pa.Platform.Modules:
if self.ModuleFile.Dir == Module.Dir and self.ModuleFile.Name == Module.Name: if self.ModuleFile.Dir == Module.Dir and self.ModuleFile.Name == Module.Name:
Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe) Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
@ -1900,13 +1875,11 @@ class Build():
Ma.Workspace = Wa Ma.Workspace = Wa
MaList.append(Ma) MaList.append(Ma)
if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]: if GlobalData.gUseHashCache and not GlobalData.gBinCacheDest and self.Target in [None, "", "all"]:
Ma.GenModuleFilesHash(GlobalData.gCacheIR) if Ma.CanSkipbyPreMakeCache():
Ma.GenPreMakefileHash(GlobalData.gCacheIR)
if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR):
self.HashSkipModules.append(Ma)
EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))
continue continue
else:
self.PreMakeCacheMiss.add(Ma)
# Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds' # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds'
if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']: if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
@ -1929,19 +1902,12 @@ class Build():
return True return True
if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]: if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
Ma.GenMakeHeaderFilesHash(GlobalData.gCacheIR) if Ma.CanSkipbyMakeCache():
Ma.GenMakeHash(GlobalData.gCacheIR)
if Ma.CanSkipbyMakeCache(GlobalData.gCacheIR):
self.HashSkipModules.append(Ma)
EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))
continue continue
else: else:
EdkLogger.quiet("cache miss: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch)) self.MakeCacheMiss.add(Ma)
Ma.PrintFirstMakeCacheMissFile(GlobalData.gCacheIR)
self.BuildModules.append(Ma) self.BuildModules.append(Ma)
# Initialize all modules in tracking to 'FAIL'
GlobalData.gModuleBuildTracking[Ma] = 'FAIL'
self.AutoGenTime += int(round((time.time() - AutoGenStart))) self.AutoGenTime += int(round((time.time() - AutoGenStart)))
MakeStart = time.time() MakeStart = time.time()
for Ma in self.BuildModules: for Ma in self.BuildModules:
@ -1952,7 +1918,6 @@ class Build():
# we need a full version of makefile for platform # we need a full version of makefile for platform
ExitFlag.set() ExitFlag.set()
BuildTask.WaitForComplete() BuildTask.WaitForComplete()
self.invalidateHash()
Pa.CreateMakeFile(False) Pa.CreateMakeFile(False)
EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule) EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
# Start task scheduler # Start task scheduler
@ -1962,7 +1927,6 @@ class Build():
# in case there's an interruption. we need a full version of makefile for platform # in case there's an interruption. we need a full version of makefile for platform
Pa.CreateMakeFile(False) Pa.CreateMakeFile(False)
if BuildTask.HasError(): if BuildTask.HasError():
self.invalidateHash()
EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule) EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
self.MakeTime += int(round((time.time() - MakeStart))) self.MakeTime += int(round((time.time() - MakeStart)))
@ -1971,11 +1935,14 @@ class Build():
BuildTask.WaitForComplete() BuildTask.WaitForComplete()
self.CreateAsBuiltInf() self.CreateAsBuiltInf()
if GlobalData.gBinCacheDest: if GlobalData.gBinCacheDest:
self.UpdateBuildCache() self.GenDestCache()
elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:
# Only for --hash
# Update PreMakeCacheChain files
self.GenLocalPreMakeCache()
self.BuildModules = [] self.BuildModules = []
self.MakeTime += int(round((time.time() - MakeContiue))) self.MakeTime += int(round((time.time() - MakeContiue)))
if BuildTask.HasError(): if BuildTask.HasError():
self.invalidateHash()
EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule) EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
self.BuildReport.AddPlatformReport(Wa, MaList) self.BuildReport.AddPlatformReport(Wa, MaList)
@ -2028,7 +1995,6 @@ class Build():
# Save MAP buffer into MAP file. # Save MAP buffer into MAP file.
# #
self._SaveMapFile (MapBuffer, Wa) self._SaveMapFile (MapBuffer, Wa)
self.invalidateHash()
def _GenFfsCmd(self,ArchList): def _GenFfsCmd(self,ArchList):
# convert dictionary of Cmd:(Inf,Arch) # convert dictionary of Cmd:(Inf,Arch)
@ -2134,20 +2100,13 @@ class Build():
self.BuildReport.AddPlatformReport(Wa) self.BuildReport.AddPlatformReport(Wa)
Wa.CreateMakeFile(False) Wa.CreateMakeFile(False)
# Add ffs build to makefile # Add ffs build to makefile
CmdListDict = {} CmdListDict = {}
if GlobalData.gEnableGenfdsMultiThread and self.Fdf: if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
CmdListDict = self._GenFfsCmd(Wa.ArchList) CmdListDict = self._GenFfsCmd(Wa.ArchList)
# Add Platform and Package level hash in share_data for module hash calculation later
if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest:
GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash
for PkgName in GlobalData.gPackageHash.keys():
GlobalData.gCacheIR[(PkgName, 'PackageHash')] = GlobalData.gPackageHash[PkgName]
self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime))) self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime)))
BuildModules = [] BuildModules = []
TotalModules = []
for Arch in Wa.ArchList: for Arch in Wa.ArchList:
PcdMaList = [] PcdMaList = []
AutoGenStart = time.time() AutoGenStart = time.time()
@ -2158,7 +2117,7 @@ class Build():
ModuleList = [] ModuleList = []
for Inf in Pa.Platform.Modules: for Inf in Pa.Platform.Modules:
ModuleList.append(Inf) ModuleList.append(Inf)
# Add the INF only list in FDF # Add the INF only list in FDF
if GlobalData.gFdfParser is not None: if GlobalData.gFdfParser is not None:
for InfName in GlobalData.gFdfParser.Profile.InfList: for InfName in GlobalData.gFdfParser.Profile.InfList:
Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch) Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch)
@ -2172,58 +2131,73 @@ class Build():
Pa.DataPipe.DataContainer = {"LibraryBuildDirectoryList":Pa.LibraryBuildDirectoryList} Pa.DataPipe.DataContainer = {"LibraryBuildDirectoryList":Pa.LibraryBuildDirectoryList}
Pa.DataPipe.DataContainer = {"ModuleBuildDirectoryList":Pa.ModuleBuildDirectoryList} Pa.DataPipe.DataContainer = {"ModuleBuildDirectoryList":Pa.ModuleBuildDirectoryList}
Pa.DataPipe.DataContainer = {"FdsCommandDict": Wa.GenFdsCommandDict} Pa.DataPipe.DataContainer = {"FdsCommandDict": Wa.GenFdsCommandDict}
# Prepare the cache share data for multiprocessing
Pa.DataPipe.DataContainer = {"gPlatformHashFile":GlobalData.gPlatformHashFile}
ModuleCodaFile = {} ModuleCodaFile = {}
for ma in Pa.ModuleAutoGenList: for ma in Pa.ModuleAutoGenList:
ModuleCodaFile[(ma.MetaFile.File,ma.MetaFile.Root,ma.Arch,ma.MetaFile.Path)] = [item.Target for item in ma.CodaTargetList] ModuleCodaFile[(ma.MetaFile.File,ma.MetaFile.Root,ma.Arch,ma.MetaFile.Path)] = [item.Target for item in ma.CodaTargetList]
Pa.DataPipe.DataContainer = {"ModuleCodaFile":ModuleCodaFile} Pa.DataPipe.DataContainer = {"ModuleCodaFile":ModuleCodaFile}
# ModuleList contains all driver modules only
for Module in ModuleList: for Module in ModuleList:
# Get ModuleAutoGen object to generate C code file and makefile # Get ModuleAutoGen object to generate C code file and makefile
Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe) Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
if Ma is None: if Ma is None:
continue continue
if Ma.PcdIsDriver: if Ma.PcdIsDriver:
Ma.PlatformInfo = Pa Ma.PlatformInfo = Pa
Ma.Workspace = Wa Ma.Workspace = Wa
PcdMaList.append(Ma) PcdMaList.append(Ma)
TotalModules.append(Ma) self.AllDrivers.add(Ma)
# Initialize all modules in tracking to 'FAIL' self.AllModules.add(Ma)
GlobalData.gModuleBuildTracking[Ma] = 'FAIL'
mqueue = mp.Queue() mqueue = mp.Queue()
cqueue = mp.Queue()
for m in Pa.GetAllModuleInfo: for m in Pa.GetAllModuleInfo:
mqueue.put(m) mqueue.put(m)
module_file,module_root,module_path,module_basename,\
module_originalpath,module_arch,IsLib = m
Ma = ModuleAutoGen(Wa, PathClass(module_path, Wa), BuildTarget,\
ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
self.AllModules.add(Ma)
data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch)) data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch))
Pa.DataPipe.dump(data_pipe_file) Pa.DataPipe.dump(data_pipe_file)
autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList,GlobalData.gCacheIR) autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList, cqueue)
# Skip cache hit modules
if GlobalData.gBinCacheSource:
for Ma in TotalModules:
if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \
GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].PreMakeCacheHit:
self.HashSkipModules.append(Ma)
continue
if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \
GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].MakeCacheHit:
self.HashSkipModules.append(Ma)
continue
BuildModules.append(Ma)
else:
BuildModules.extend(TotalModules)
if not autogen_rt: if not autogen_rt:
self.AutoGenMgr.TerminateWorkers() self.AutoGenMgr.TerminateWorkers()
self.AutoGenMgr.join(1) self.AutoGenMgr.join(1)
raise FatalError(errorcode) raise FatalError(errorcode)
if GlobalData.gUseHashCache:
for item in GlobalData.gModuleAllCacheStatus:
(MetaFilePath, Arch, CacheStr, Status) = item
Ma = ModuleAutoGen(Wa, PathClass(MetaFilePath, Wa), BuildTarget,\
ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
if CacheStr == "PreMakeCache" and Status == False:
self.PreMakeCacheMiss.add(Ma)
if CacheStr == "PreMakeCache" and Status == True:
self.PreMakeCacheHit.add(Ma)
GlobalData.gModuleCacheHit.add(Ma)
if CacheStr == "MakeCache" and Status == False:
self.MakeCacheMiss.add(Ma)
if CacheStr == "MakeCache" and Status == True:
self.MakeCacheHit.add(Ma)
GlobalData.gModuleCacheHit.add(Ma)
self.AutoGenTime += int(round((time.time() - AutoGenStart))) self.AutoGenTime += int(round((time.time() - AutoGenStart)))
AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt") AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
with open(AutoGenIdFile,"w") as fw: with open(AutoGenIdFile,"w") as fw:
fw.write("Arch=%s\n" % "|".join((Wa.ArchList))) fw.write("Arch=%s\n" % "|".join((Wa.ArchList)))
fw.write("BuildDir=%s\n" % Wa.BuildDir) fw.write("BuildDir=%s\n" % Wa.BuildDir)
fw.write("PlatformGuid=%s\n" % str(Wa.AutoGenObjectList[0].Guid)) fw.write("PlatformGuid=%s\n" % str(Wa.AutoGenObjectList[0].Guid))
if GlobalData.gBinCacheSource:
BuildModules.extend(self.MakeCacheMiss)
elif GlobalData.gUseHashCache and not GlobalData.gBinCacheDest:
BuildModules.extend(self.PreMakeCacheMiss)
else:
BuildModules.extend(self.AllDrivers)
self.Progress.Stop("done!") self.Progress.Stop("done!")
return Wa, BuildModules return Wa, BuildModules
@ -2253,18 +2227,9 @@ class Build():
GlobalData.gAutoGenPhase = False GlobalData.gAutoGenPhase = False
if GlobalData.gBinCacheSource: if GlobalData.gBinCacheSource:
EdkLogger.quiet("Total cache hit driver num: %s, cache miss driver num: %s" % (len(set(self.HashSkipModules)), len(set(self.BuildModules)))) EdkLogger.quiet("[cache Summary]: Total module num: %s" % len(self.AllModules))
CacheHitMa = set() EdkLogger.quiet("[cache Summary]: PreMakecache miss num: %s " % len(self.PreMakeCacheMiss))
CacheNotHitMa = set() EdkLogger.quiet("[cache Summary]: Makecache miss num: %s " % len(self.MakeCacheMiss))
for IR in GlobalData.gCacheIR.keys():
if 'PlatformHash' in IR or 'PackageHash' in IR:
continue
if GlobalData.gCacheIR[IR].PreMakeCacheHit or GlobalData.gCacheIR[IR].MakeCacheHit:
CacheHitMa.add(IR)
else:
# There might be binary module or module which has .inc files, not count for cache miss
CacheNotHitMa.add(IR)
EdkLogger.quiet("Total module num: %s, cache hit module num: %s" % (len(CacheHitMa)+len(CacheNotHitMa), len(CacheHitMa)))
for Arch in Wa.ArchList: for Arch in Wa.ArchList:
MakeStart = time.time() MakeStart = time.time()
@ -2277,7 +2242,6 @@ class Build():
# we need a full version of makefile for platform # we need a full version of makefile for platform
ExitFlag.set() ExitFlag.set()
BuildTask.WaitForComplete() BuildTask.WaitForComplete()
self.invalidateHash()
Pa.CreateMakeFile(False) Pa.CreateMakeFile(False)
EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule) EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
# Start task scheduler # Start task scheduler
@ -2287,7 +2251,6 @@ class Build():
# in case there's an interruption. we need a full version of makefile for platform # in case there's an interruption. we need a full version of makefile for platform
if BuildTask.HasError(): if BuildTask.HasError():
self.invalidateHash()
EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule) EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
self.MakeTime += int(round((time.time() - MakeStart))) self.MakeTime += int(round((time.time() - MakeStart)))
@ -2301,7 +2264,11 @@ class Build():
BuildTask.WaitForComplete() BuildTask.WaitForComplete()
self.CreateAsBuiltInf() self.CreateAsBuiltInf()
if GlobalData.gBinCacheDest: if GlobalData.gBinCacheDest:
self.UpdateBuildCache() self.GenDestCache()
elif GlobalData.gUseHashCache and not GlobalData.gBinCacheSource:
# Only for --hash
# Update PreMakeCacheChain files
self.GenLocalPreMakeCache()
# #
# Get Module List # Get Module List
# #
@ -2313,7 +2280,6 @@ class Build():
# has been signaled. # has been signaled.
# #
if BuildTask.HasError(): if BuildTask.HasError():
self.invalidateHash()
EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule) EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
# Create MAP file when Load Fix Address is enabled. # Create MAP file when Load Fix Address is enabled.
@ -2350,7 +2316,6 @@ class Build():
# #
self._SaveMapFile(MapBuffer, Wa) self._SaveMapFile(MapBuffer, Wa)
self.CreateGuidedSectionToolsFile(Wa) self.CreateGuidedSectionToolsFile(Wa)
self.invalidateHash()
## Generate GuidedSectionTools.txt in the FV directories. ## Generate GuidedSectionTools.txt in the FV directories.
# #
def CreateGuidedSectionToolsFile(self,Wa): def CreateGuidedSectionToolsFile(self,Wa):
@ -2406,6 +2371,12 @@ class Build():
## Launch the module or platform build ## Launch the module or platform build
# #
def Launch(self): def Launch(self):
self.AllDrivers = set()
self.AllModules = set()
self.PreMakeCacheMiss = set()
self.PreMakeCacheHit = set()
self.MakeCacheMiss = set()
self.MakeCacheHit = set()
if not self.ModuleFile: if not self.ModuleFile:
if not self.SpawnMode or self.Target not in ["", "all"]: if not self.SpawnMode or self.Target not in ["", "all"]:
self.SpawnMode = False self.SpawnMode = False
@ -2423,23 +2394,16 @@ class Build():
for Module in self.BuildModules: for Module in self.BuildModules:
Module.CreateAsBuiltInf() Module.CreateAsBuiltInf()
def UpdateBuildCache(self): def GenDestCache(self):
all_lib_set = set() for Module in self.AllModules:
all_mod_set = set() Module.GenPreMakefileHashList()
for Module in self.BuildModules: Module.GenMakefileHashList()
Module.CopyModuleToCache() Module.CopyModuleToCache()
all_mod_set.add(Module)
for Module in self.HashSkipModules: def GenLocalPreMakeCache(self):
Module.CopyModuleToCache() for Module in self.PreMakeCacheMiss:
all_mod_set.add(Module) Module.GenPreMakefileHashList()
for Module in all_mod_set:
for lib in Module.LibraryAutoGenList:
all_lib_set.add(lib)
for lib in all_lib_set:
lib.CopyModuleToCache()
all_lib_set.clear()
all_mod_set.clear()
self.HashSkipModules = []
## Do some clean-up works when error occurred ## Do some clean-up works when error occurred
def Relinquish(self): def Relinquish(self):
OldLogLevel = EdkLogger.GetLevel() OldLogLevel = EdkLogger.GetLevel()