diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py old mode 100644 new mode 100755 index e583828741..a84ed46f2e --- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py +++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py @@ -182,6 +182,12 @@ class AutoGenWorkerInProcess(mp.Process): GlobalData.gDisableIncludePathCheck = False GlobalData.gFdfParser = self.data_pipe.Get("FdfParser") GlobalData.gDatabasePath = self.data_pipe.Get("DatabasePath") + GlobalData.gBinCacheSource = self.data_pipe.Get("BinCacheSource") + GlobalData.gBinCacheDest = self.data_pipe.Get("BinCacheDest") + GlobalData.gCacheIR = self.data_pipe.Get("CacheIR") + GlobalData.gEnableGenfdsMultiThread = self.data_pipe.Get("EnableGenfdsMultiThread") + GlobalData.file_lock = self.file_lock + CommandTarget = self.data_pipe.Get("CommandTarget") pcd_from_build_option = [] for pcd_tuple in self.data_pipe.Get("BuildOptPcd"): pcd_id = ".".join((pcd_tuple[0],pcd_tuple[1])) @@ -193,10 +199,13 @@ class AutoGenWorkerInProcess(mp.Process): FfsCmd = self.data_pipe.Get("FfsCommand") if FfsCmd is None: FfsCmd = {} + GlobalData.FfsCmd = FfsCmd PlatformMetaFile = self.GetPlatformMetaFile(self.data_pipe.Get("P_Info").get("ActivePlatform"), self.data_pipe.Get("P_Info").get("WorkspaceDir")) libConstPcd = self.data_pipe.Get("LibConstPcd") Refes = self.data_pipe.Get("REFS") + GlobalData.libConstPcd = libConstPcd + GlobalData.Refes = Refes while True: if self.module_queue.empty(): break @@ -223,8 +232,20 @@ class AutoGenWorkerInProcess(mp.Process): Ma.ConstPcd = libConstPcd[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)] if (Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path) in Refes: Ma.ReferenceModules = Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)] + if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]: + Ma.GenModuleFilesHash(GlobalData.gCacheIR) + Ma.GenPreMakefileHash(GlobalData.gCacheIR) + if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR): + continue + Ma.CreateCodeFile(False) Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.File, Ma.Arch),[])) + + if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]: + Ma.GenMakeHeaderFilesHash(GlobalData.gCacheIR) + Ma.GenMakeHash(GlobalData.gCacheIR) + if Ma.CanSkipbyMakeCache(GlobalData.gCacheIR): + continue except Empty: pass except: diff --git a/BaseTools/Source/Python/AutoGen/CacheIR.py b/BaseTools/Source/Python/AutoGen/CacheIR.py new file mode 100755 index 0000000000..2d9ffe3f0b --- /dev/null +++ b/BaseTools/Source/Python/AutoGen/CacheIR.py @@ -0,0 +1,28 @@ +## @file +# Build cache intermediate result and state +# +# Copyright (c) 2019, Intel Corporation. All rights reserved.
+# SPDX-License-Identifier: BSD-2-Clause-Patent +# + +class ModuleBuildCacheIR(): + def __init__(self, Path, Arch): + self.ModulePath = Path + self.ModuleArch = Arch + self.ModuleFilesHashDigest = None + self.ModuleFilesHashHexDigest = None + self.ModuleFilesChain = [] + self.PreMakefileHashHexDigest = None + self.CreateCodeFileDone = False + self.CreateMakeFileDone = False + self.MakefilePath = None + self.AutoGenFileList = None + self.DependencyHeaderFileSet = None + self.MakeHeaderFilesHashChain = None + self.MakeHeaderFilesHashDigest = None + self.MakeHeaderFilesHashChain = [] + self.MakeHashDigest = None + self.MakeHashHexDigest = None + self.MakeHashChain = [] + self.PreMakeCacheHit = False + self.MakeCacheHit = False diff --git a/BaseTools/Source/Python/AutoGen/DataPipe.py b/BaseTools/Source/Python/AutoGen/DataPipe.py old mode 100644 new mode 100755 index 2052084bdb..87a1a125c8 --- a/BaseTools/Source/Python/AutoGen/DataPipe.py +++ b/BaseTools/Source/Python/AutoGen/DataPipe.py @@ -158,3 +158,11 @@ class MemoryDataPipe(DataPipe): self.DataContainer = {"FdfParser": True if GlobalData.gFdfParser else False} self.DataContainer = {"LogLevel": EdkLogger.GetLevel()} + + self.DataContainer = {"BinCacheSource":GlobalData.gBinCacheSource} + + self.DataContainer = {"BinCacheDest":GlobalData.gBinCacheDest} + + self.DataContainer = {"CacheIR":GlobalData.gCacheIR} + + self.DataContainer = {"EnableGenfdsMultiThread":GlobalData.gEnableGenfdsMultiThread} \ No newline at end of file diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py b/BaseTools/Source/Python/AutoGen/GenMake.py old mode 100644 new mode 100755 index 499ef82aea..0d581c9415 --- a/BaseTools/Source/Python/AutoGen/GenMake.py +++ b/BaseTools/Source/Python/AutoGen/GenMake.py @@ -906,6 +906,11 @@ cleanlib: self._AutoGenObject.IncludePathList + self._AutoGenObject.BuildOptionIncPathList ) + self.DependencyHeaderFileSet = set() + if FileDependencyDict: + for Dependency in FileDependencyDict.values(): + self.DependencyHeaderFileSet.update(set(Dependency)) + # Get a set of unique package includes from MetaFile parentMetaFileIncludes = set() for aInclude in self._AutoGenObject.PackageIncludePathList: @@ -1115,7 +1120,7 @@ cleanlib: ## For creating makefile targets for dependent libraries def ProcessDependentLibrary(self): for LibraryAutoGen in self._AutoGenObject.LibraryAutoGenList: - if not LibraryAutoGen.IsBinaryModule and not LibraryAutoGen.CanSkipbyHash(): + if not LibraryAutoGen.IsBinaryModule: self.LibraryBuildDirectoryList.append(self.PlaceMacro(LibraryAutoGen.BuildDir, self.Macros)) ## Return a list containing source file's dependencies @@ -1129,114 +1134,9 @@ cleanlib: def GetFileDependency(self, FileList, ForceInculeList, SearchPathList): Dependency = {} for F in FileList: - Dependency[F] = self.GetDependencyList(F, ForceInculeList, SearchPathList) + Dependency[F] = GetDependencyList(self._AutoGenObject, self.FileCache, F, ForceInculeList, SearchPathList) return Dependency - ## Find dependencies for one source file - # - # By searching recursively "#include" directive in file, find out all the - # files needed by given source file. The dependencies will be only searched - # in given search path list. - # - # @param File The source file - # @param ForceInculeList The list of files which will be included forcely - # @param SearchPathList The list of search path - # - # @retval list The list of files the given source file depends on - # - def GetDependencyList(self, File, ForceList, SearchPathList): - EdkLogger.debug(EdkLogger.DEBUG_1, "Try to get dependency files for %s" % File) - FileStack = [File] + ForceList - DependencySet = set() - - if self._AutoGenObject.Arch not in gDependencyDatabase: - gDependencyDatabase[self._AutoGenObject.Arch] = {} - DepDb = gDependencyDatabase[self._AutoGenObject.Arch] - - while len(FileStack) > 0: - F = FileStack.pop() - - FullPathDependList = [] - if F in self.FileCache: - for CacheFile in self.FileCache[F]: - FullPathDependList.append(CacheFile) - if CacheFile not in DependencySet: - FileStack.append(CacheFile) - DependencySet.update(FullPathDependList) - continue - - CurrentFileDependencyList = [] - if F in DepDb: - CurrentFileDependencyList = DepDb[F] - else: - try: - Fd = open(F.Path, 'rb') - FileContent = Fd.read() - Fd.close() - except BaseException as X: - EdkLogger.error("build", FILE_OPEN_FAILURE, ExtraData=F.Path + "\n\t" + str(X)) - if len(FileContent) == 0: - continue - try: - if FileContent[0] == 0xff or FileContent[0] == 0xfe: - FileContent = FileContent.decode('utf-16') - else: - FileContent = FileContent.decode() - except: - # The file is not txt file. for example .mcb file - continue - IncludedFileList = gIncludePattern.findall(FileContent) - - for Inc in IncludedFileList: - Inc = Inc.strip() - # if there's macro used to reference header file, expand it - HeaderList = gMacroPattern.findall(Inc) - if len(HeaderList) == 1 and len(HeaderList[0]) == 2: - HeaderType = HeaderList[0][0] - HeaderKey = HeaderList[0][1] - if HeaderType in gIncludeMacroConversion: - Inc = gIncludeMacroConversion[HeaderType] % {"HeaderKey" : HeaderKey} - else: - # not known macro used in #include, always build the file by - # returning a empty dependency - self.FileCache[File] = [] - return [] - Inc = os.path.normpath(Inc) - CurrentFileDependencyList.append(Inc) - DepDb[F] = CurrentFileDependencyList - - CurrentFilePath = F.Dir - PathList = [CurrentFilePath] + SearchPathList - for Inc in CurrentFileDependencyList: - for SearchPath in PathList: - FilePath = os.path.join(SearchPath, Inc) - if FilePath in gIsFileMap: - if not gIsFileMap[FilePath]: - continue - # If isfile is called too many times, the performance is slow down. - elif not os.path.isfile(FilePath): - gIsFileMap[FilePath] = False - continue - else: - gIsFileMap[FilePath] = True - FilePath = PathClass(FilePath) - FullPathDependList.append(FilePath) - if FilePath not in DependencySet: - FileStack.append(FilePath) - break - else: - EdkLogger.debug(EdkLogger.DEBUG_9, "%s included by %s was not found "\ - "in any given path:\n\t%s" % (Inc, F, "\n\t".join(SearchPathList))) - - self.FileCache[F] = FullPathDependList - DependencySet.update(FullPathDependList) - - DependencySet.update(ForceList) - if File in DependencySet: - DependencySet.remove(File) - DependencyList = list(DependencySet) # remove duplicate ones - - return DependencyList ## CustomMakefile class # @@ -1618,7 +1518,7 @@ cleanlib: def GetLibraryBuildDirectoryList(self): DirList = [] for LibraryAutoGen in self._AutoGenObject.LibraryAutoGenList: - if not LibraryAutoGen.IsBinaryModule and not LibraryAutoGen.CanSkipbyHash(): + if not LibraryAutoGen.IsBinaryModule: DirList.append(os.path.join(self._AutoGenObject.BuildDir, LibraryAutoGen.BuildDir)) return DirList @@ -1754,11 +1654,116 @@ class TopLevelMakefile(BuildFile): def GetLibraryBuildDirectoryList(self): DirList = [] for LibraryAutoGen in self._AutoGenObject.LibraryAutoGenList: - if not LibraryAutoGen.IsBinaryModule and not LibraryAutoGen.CanSkipbyHash(): + if not LibraryAutoGen.IsBinaryModule: DirList.append(os.path.join(self._AutoGenObject.BuildDir, LibraryAutoGen.BuildDir)) return DirList +## Find dependencies for one source file +# +# By searching recursively "#include" directive in file, find out all the +# files needed by given source file. The dependencies will be only searched +# in given search path list. +# +# @param File The source file +# @param ForceInculeList The list of files which will be included forcely +# @param SearchPathList The list of search path +# +# @retval list The list of files the given source file depends on +# +def GetDependencyList(AutoGenObject, FileCache, File, ForceList, SearchPathList): + EdkLogger.debug(EdkLogger.DEBUG_1, "Try to get dependency files for %s" % File) + FileStack = [File] + ForceList + DependencySet = set() + + if AutoGenObject.Arch not in gDependencyDatabase: + gDependencyDatabase[AutoGenObject.Arch] = {} + DepDb = gDependencyDatabase[AutoGenObject.Arch] + + while len(FileStack) > 0: + F = FileStack.pop() + + FullPathDependList = [] + if F in FileCache: + for CacheFile in FileCache[F]: + FullPathDependList.append(CacheFile) + if CacheFile not in DependencySet: + FileStack.append(CacheFile) + DependencySet.update(FullPathDependList) + continue + + CurrentFileDependencyList = [] + if F in DepDb: + CurrentFileDependencyList = DepDb[F] + else: + try: + Fd = open(F.Path, 'rb') + FileContent = Fd.read() + Fd.close() + except BaseException as X: + EdkLogger.error("build", FILE_OPEN_FAILURE, ExtraData=F.Path + "\n\t" + str(X)) + if len(FileContent) == 0: + continue + try: + if FileContent[0] == 0xff or FileContent[0] == 0xfe: + FileContent = FileContent.decode('utf-16') + else: + FileContent = FileContent.decode() + except: + # The file is not txt file. for example .mcb file + continue + IncludedFileList = gIncludePattern.findall(FileContent) + + for Inc in IncludedFileList: + Inc = Inc.strip() + # if there's macro used to reference header file, expand it + HeaderList = gMacroPattern.findall(Inc) + if len(HeaderList) == 1 and len(HeaderList[0]) == 2: + HeaderType = HeaderList[0][0] + HeaderKey = HeaderList[0][1] + if HeaderType in gIncludeMacroConversion: + Inc = gIncludeMacroConversion[HeaderType] % {"HeaderKey" : HeaderKey} + else: + # not known macro used in #include, always build the file by + # returning a empty dependency + FileCache[File] = [] + return [] + Inc = os.path.normpath(Inc) + CurrentFileDependencyList.append(Inc) + DepDb[F] = CurrentFileDependencyList + + CurrentFilePath = F.Dir + PathList = [CurrentFilePath] + SearchPathList + for Inc in CurrentFileDependencyList: + for SearchPath in PathList: + FilePath = os.path.join(SearchPath, Inc) + if FilePath in gIsFileMap: + if not gIsFileMap[FilePath]: + continue + # If isfile is called too many times, the performance is slow down. + elif not os.path.isfile(FilePath): + gIsFileMap[FilePath] = False + continue + else: + gIsFileMap[FilePath] = True + FilePath = PathClass(FilePath) + FullPathDependList.append(FilePath) + if FilePath not in DependencySet: + FileStack.append(FilePath) + break + else: + EdkLogger.debug(EdkLogger.DEBUG_9, "%s included by %s was not found "\ + "in any given path:\n\t%s" % (Inc, F, "\n\t".join(SearchPathList))) + + FileCache[F] = FullPathDependList + DependencySet.update(FullPathDependList) + + DependencySet.update(ForceList) + if File in DependencySet: + DependencySet.remove(File) + DependencyList = list(DependencySet) # remove duplicate ones + + return DependencyList + # This acts like the main() function for the script, unless it is 'import'ed into another script. if __name__ == '__main__': - pass - + pass \ No newline at end of file diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py old mode 100644 new mode 100755 index 076ce0e39c..383078c376 --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py @@ -26,6 +26,8 @@ from Workspace.MetaFileCommentParser import UsageList from .GenPcdDb import CreatePcdDatabaseCode from Common.caching import cached_class_function from AutoGen.ModuleAutoGenHelper import PlatformInfo,WorkSpaceInfo +from AutoGen.CacheIR import ModuleBuildCacheIR +import json ## Mapping Makefile type gMakeTypeMap = {TAB_COMPILER_MSFT:"nmake", "GCC":"gmake"} @@ -252,6 +254,8 @@ class ModuleAutoGen(AutoGen): self.AutoGenDepSet = set() self.ReferenceModules = [] self.ConstPcd = {} + self.Makefile = None + self.FileDependCache = {} def __init_platform_info__(self): pinfo = self.DataPipe.Get("P_Info") @@ -1608,12 +1612,37 @@ class ModuleAutoGen(AutoGen): self.IsAsBuiltInfCreated = True + def CacheCopyFile(self, OriginDir, CopyDir, File): + sub_dir = os.path.relpath(File, CopyDir) + destination_file = os.path.join(OriginDir, sub_dir) + destination_dir = os.path.dirname(destination_file) + CreateDirectory(destination_dir) + try: + CopyFileOnChange(File, destination_dir) + except: + EdkLogger.quiet("[cache warning]: fail to copy file:%s to folder:%s" % (File, destination_dir)) + return + def CopyModuleToCache(self): - FileDir = path.join(GlobalData.gBinCacheDest, self.PlatformInfo.Name, self.BuildTarget + "_" + self.ToolChain, self.Arch, self.SourceDir, self.MetaFile.BaseName) + self.GenPreMakefileHash(GlobalData.gCacheIR) + if not (self.MetaFile.Path, self.Arch) in GlobalData.gCacheIR or \ + not GlobalData.gCacheIR[(self.MetaFile.Path, self.Arch)].PreMakefileHashHexDigest: + EdkLogger.quiet("[cache warning]: Cannot generate PreMakefileHash for module: %s[%s]" % (self.MetaFile.Path, self.Arch)) + return False + + self.GenMakeHash(GlobalData.gCacheIR) + if not (self.MetaFile.Path, self.Arch) in GlobalData.gCacheIR or \ + not GlobalData.gCacheIR[(self.MetaFile.Path, self.Arch)].MakeHashChain or \ + not GlobalData.gCacheIR[(self.MetaFile.Path, self.Arch)].MakeHashHexDigest: + EdkLogger.quiet("[cache warning]: Cannot generate MakeHashChain for module: %s[%s]" % (self.MetaFile.Path, self.Arch)) + return False + + MakeHashStr = str(GlobalData.gCacheIR[(self.MetaFile.Path, self.Arch)].MakeHashHexDigest) + FileDir = path.join(GlobalData.gBinCacheDest, self.PlatformInfo.OutputDir, self.BuildTarget + "_" + self.ToolChain, self.Arch, self.SourceDir, self.MetaFile.BaseName, MakeHashStr) + FfsDir = path.join(GlobalData.gBinCacheDest, self.PlatformInfo.OutputDir, self.BuildTarget + "_" + self.ToolChain, TAB_FV_DIRECTORY, "Ffs", self.Guid + self.Name, MakeHashStr) + CreateDirectory (FileDir) - HashFile = path.join(self.BuildDir, self.Name + '.hash') - if os.path.exists(HashFile): - CopyFileOnChange(HashFile, FileDir) + self.SaveHashChainFileToCache(GlobalData.gCacheIR) ModuleFile = path.join(self.OutputDir, self.Name + '.inf') if os.path.exists(ModuleFile): CopyFileOnChange(ModuleFile, FileDir) @@ -1631,38 +1660,73 @@ class ModuleAutoGen(AutoGen): CreateDirectory(destination_dir) CopyFileOnChange(File, destination_dir) - def AttemptModuleCacheCopy(self): - # If library or Module is binary do not skip by hash - if self.IsBinaryModule: + def SaveHashChainFileToCache(self, gDict): + if not GlobalData.gBinCacheDest: return False - # .inc is contains binary information so do not skip by hash as well - for f_ext in self.SourceFileList: - if '.inc' in str(f_ext): - return False - FileDir = path.join(GlobalData.gBinCacheSource, self.PlatformInfo.Name, self.BuildTarget + "_" + self.ToolChain, self.Arch, self.SourceDir, self.MetaFile.BaseName) - HashFile = path.join(FileDir, self.Name + '.hash') - if os.path.exists(HashFile): - f = open(HashFile, 'r') - CacheHash = f.read() - f.close() - self.GenModuleHash() - if GlobalData.gModuleHash[self.Arch][self.Name]: - if CacheHash == GlobalData.gModuleHash[self.Arch][self.Name]: - for root, dir, files in os.walk(FileDir): - for f in files: - if self.Name + '.hash' in f: - CopyFileOnChange(HashFile, self.BuildDir) - else: - File = path.join(root, f) - sub_dir = os.path.relpath(File, FileDir) - destination_file = os.path.join(self.OutputDir, sub_dir) - destination_dir = os.path.dirname(destination_file) - CreateDirectory(destination_dir) - CopyFileOnChange(File, destination_dir) - if self.Name == "PcdPeim" or self.Name == "PcdDxe": - CreatePcdDatabaseCode(self, TemplateString(), TemplateString()) - return True - return False + + self.GenPreMakefileHash(gDict) + if not (self.MetaFile.Path, self.Arch) in gDict or \ + not gDict[(self.MetaFile.Path, self.Arch)].PreMakefileHashHexDigest: + EdkLogger.quiet("[cache warning]: Cannot generate PreMakefileHash for module: %s[%s]" % (self.MetaFile.Path, self.Arch)) + return False + + self.GenMakeHash(gDict) + if not (self.MetaFile.Path, self.Arch) in gDict or \ + not gDict[(self.MetaFile.Path, self.Arch)].MakeHashChain or \ + not gDict[(self.MetaFile.Path, self.Arch)].MakeHashHexDigest: + EdkLogger.quiet("[cache warning]: Cannot generate MakeHashChain for module: %s[%s]" % (self.MetaFile.Path, self.Arch)) + return False + + # save the hash chain list as cache file + MakeHashStr = str(GlobalData.gCacheIR[(self.MetaFile.Path, self.Arch)].MakeHashHexDigest) + CacheDestDir = path.join(GlobalData.gBinCacheDest, self.PlatformInfo.OutputDir, self.BuildTarget + "_" + self.ToolChain, self.Arch, self.SourceDir, self.MetaFile.BaseName) + CacheHashDestDir = path.join(CacheDestDir, MakeHashStr) + ModuleHashPair = path.join(CacheDestDir, self.Name + ".ModuleHashPair") + MakeHashChain = path.join(CacheHashDestDir, self.Name + ".MakeHashChain") + ModuleFilesChain = path.join(CacheHashDestDir, self.Name + ".ModuleFilesChain") + + # save the HashChainDict as json file + CreateDirectory (CacheDestDir) + CreateDirectory (CacheHashDestDir) + try: + ModuleHashPairList = [] # tuple list: [tuple(PreMakefileHash, MakeHash)] + if os.path.exists(ModuleHashPair): + f = open(ModuleHashPair, 'r') + ModuleHashPairList = json.load(f) + f.close() + PreMakeHash = gDict[(self.MetaFile.Path, self.Arch)].PreMakefileHashHexDigest + MakeHash = gDict[(self.MetaFile.Path, self.Arch)].MakeHashHexDigest + ModuleHashPairList.append((PreMakeHash, MakeHash)) + ModuleHashPairList = list(set(map(tuple, ModuleHashPairList))) + with open(ModuleHashPair, 'w') as f: + json.dump(ModuleHashPairList, f, indent=2) + except: + EdkLogger.quiet("[cache warning]: fail to save ModuleHashPair file in cache: %s" % ModuleHashPair) + return False + + try: + with open(MakeHashChain, 'w') as f: + json.dump(gDict[(self.MetaFile.Path, self.Arch)].MakeHashChain, f, indent=2) + except: + EdkLogger.quiet("[cache warning]: fail to save MakeHashChain file in cache: %s" % MakeHashChain) + return False + + try: + with open(ModuleFilesChain, 'w') as f: + json.dump(gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesChain, f, indent=2) + except: + EdkLogger.quiet("[cache warning]: fail to save ModuleFilesChain file in cache: %s" % ModuleFilesChain) + return False + + # save the autogenfile and makefile for debug usage + CacheDebugDir = path.join(CacheHashDestDir, "CacheDebug") + CreateDirectory (CacheDebugDir) + CopyFileOnChange(gDict[(self.MetaFile.Path, self.Arch)].MakefilePath, CacheDebugDir) + if gDict[(self.MetaFile.Path, self.Arch)].AutoGenFileList: + for File in gDict[(self.MetaFile.Path, self.Arch)].AutoGenFileList: + CopyFileOnChange(str(File), CacheDebugDir) + + return True ## Create makefile for the module and its dependent libraries # @@ -1671,6 +1735,11 @@ class ModuleAutoGen(AutoGen): # @cached_class_function def CreateMakeFile(self, CreateLibraryMakeFile=True, GenFfsList = []): + gDict = GlobalData.gCacheIR + if (self.MetaFile.Path, self.Arch) in gDict and \ + gDict[(self.MetaFile.Path, self.Arch)].CreateMakeFileDone: + return + # nest this function inside it's only caller. def CreateTimeStamp(): FileSet = {self.MetaFile.Path} @@ -1701,8 +1770,8 @@ class ModuleAutoGen(AutoGen): for LibraryAutoGen in self.LibraryAutoGenList: LibraryAutoGen.CreateMakeFile() - # Don't enable if hash feature enabled, CanSkip uses timestamps to determine build skipping - if not GlobalData.gUseHashCache and self.CanSkip(): + # CanSkip uses timestamps to determine build skipping + if self.CanSkip(): return if len(self.CustomMakefile) == 0: @@ -1718,6 +1787,24 @@ class ModuleAutoGen(AutoGen): CreateTimeStamp() + MakefileType = Makefile._FileType + MakefileName = Makefile._FILE_NAME_[MakefileType] + MakefilePath = os.path.join(self.MakeFileDir, MakefileName) + + MewIR = ModuleBuildCacheIR(self.MetaFile.Path, self.Arch) + MewIR.MakefilePath = MakefilePath + MewIR.DependencyHeaderFileSet = Makefile.DependencyHeaderFileSet + MewIR.CreateMakeFileDone = True + with GlobalData.file_lock: + try: + IR = gDict[(self.MetaFile.Path, self.Arch)] + IR.MakefilePath = MakefilePath + IR.DependencyHeaderFileSet = Makefile.DependencyHeaderFileSet + IR.CreateMakeFileDone = True + gDict[(self.MetaFile.Path, self.Arch)] = IR + except: + gDict[(self.MetaFile.Path, self.Arch)] = MewIR + def CopyBinaryFiles(self): for File in self.Module.Binaries: SrcPath = File.Path @@ -1729,6 +1816,11 @@ class ModuleAutoGen(AutoGen): # dependent libraries will be created # def CreateCodeFile(self, CreateLibraryCodeFile=True): + gDict = GlobalData.gCacheIR + if (self.MetaFile.Path, self.Arch) in gDict and \ + gDict[(self.MetaFile.Path, self.Arch)].CreateCodeFileDone: + return + if self.IsCodeFileCreated: return @@ -1744,8 +1836,9 @@ class ModuleAutoGen(AutoGen): if not self.IsLibrary and CreateLibraryCodeFile: for LibraryAutoGen in self.LibraryAutoGenList: LibraryAutoGen.CreateCodeFile() - # Don't enable if hash feature enabled, CanSkip uses timestamps to determine build skipping - if not GlobalData.gUseHashCache and self.CanSkip(): + + # CanSkip uses timestamps to determine build skipping + if self.CanSkip(): return AutoGenList = [] @@ -1785,6 +1878,16 @@ class ModuleAutoGen(AutoGen): (" ".join(AutoGenList), " ".join(IgoredAutoGenList), self.Name, self.Arch)) self.IsCodeFileCreated = True + MewIR = ModuleBuildCacheIR(self.MetaFile.Path, self.Arch) + MewIR.CreateCodeFileDone = True + with GlobalData.file_lock: + try: + IR = gDict[(self.MetaFile.Path, self.Arch)] + IR.CreateCodeFileDone = True + gDict[(self.MetaFile.Path, self.Arch)] = IR + except: + gDict[(self.MetaFile.Path, self.Arch)] = MewIR + return AutoGenList ## Summarize the ModuleAutoGen objects of all libraries used by this module @@ -1854,17 +1957,275 @@ class ModuleAutoGen(AutoGen): return GlobalData.gModuleHash[self.Arch][self.Name].encode('utf-8') - ## Decide whether we can skip the ModuleAutoGen process - def CanSkipbyHash(self): - # Hashing feature is off - if not GlobalData.gUseHashCache: + def GenModuleFilesHash(self, gDict): + # Early exit if module or library has been hashed and is in memory + if (self.MetaFile.Path, self.Arch) in gDict: + if gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesChain: + return gDict[(self.MetaFile.Path, self.Arch)] + + DependencyFileSet = set() + # Add Module Meta file + DependencyFileSet.add(self.MetaFile) + + # Add Module's source files + if self.SourceFileList: + for File in set(self.SourceFileList): + DependencyFileSet.add(File) + + # Add modules's include header files + # Search dependency file list for each source file + SourceFileList = [] + OutPutFileList = [] + for Target in self.IntroTargetList: + SourceFileList.extend(Target.Inputs) + OutPutFileList.extend(Target.Outputs) + if OutPutFileList: + for Item in OutPutFileList: + if Item in SourceFileList: + SourceFileList.remove(Item) + SearchList = [] + for file_path in self.IncludePathList + self.BuildOptionIncPathList: + # skip the folders in platform BuildDir which are not been generated yet + if file_path.startswith(os.path.abspath(self.PlatformInfo.BuildDir)+os.sep): + continue + SearchList.append(file_path) + FileDependencyDict = {} + ForceIncludedFile = [] + for F in SourceFileList: + # skip the files which are not been generated yet, because + # the SourceFileList usually contains intermediate build files, e.g. AutoGen.c + if not os.path.exists(F.Path): + continue + FileDependencyDict[F] = GenMake.GetDependencyList(self, self.FileDependCache, F, ForceIncludedFile, SearchList) + + if FileDependencyDict: + for Dependency in FileDependencyDict.values(): + DependencyFileSet.update(set(Dependency)) + + # Caculate all above dependency files hash + # Initialze hash object + FileList = [] + m = hashlib.md5() + for File in sorted(DependencyFileSet, key=lambda x: str(x)): + if not os.path.exists(str(File)): + EdkLogger.quiet("[cache warning]: header file %s is missing for module: %s[%s]" % (File, self.MetaFile.Path, self.Arch)) + continue + f = open(str(File), 'rb') + Content = f.read() + f.close() + m.update(Content) + FileList.append((str(File), hashlib.md5(Content).hexdigest())) + + + MewIR = ModuleBuildCacheIR(self.MetaFile.Path, self.Arch) + MewIR.ModuleFilesHashDigest = m.digest() + MewIR.ModuleFilesHashHexDigest = m.hexdigest() + MewIR.ModuleFilesChain = FileList + with GlobalData.file_lock: + try: + IR = gDict[(self.MetaFile.Path, self.Arch)] + IR.ModuleFilesHashDigest = m.digest() + IR.ModuleFilesHashHexDigest = m.hexdigest() + IR.ModuleFilesChain = FileList + gDict[(self.MetaFile.Path, self.Arch)] = IR + except: + gDict[(self.MetaFile.Path, self.Arch)] = MewIR + + return gDict[(self.MetaFile.Path, self.Arch)] + + def GenPreMakefileHash(self, gDict): + # Early exit if module or library has been hashed and is in memory + if (self.MetaFile.Path, self.Arch) in gDict and \ + gDict[(self.MetaFile.Path, self.Arch)].PreMakefileHashHexDigest: + return gDict[(self.MetaFile.Path, self.Arch)] + + # skip binary module + if self.IsBinaryModule: + return + + if not (self.MetaFile.Path, self.Arch) in gDict or \ + not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest: + self.GenModuleFilesHash(gDict) + + if not (self.MetaFile.Path, self.Arch) in gDict or \ + not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest: + EdkLogger.quiet("[cache warning]: Cannot generate ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path, self.Arch)) + return + + # Initialze hash object + m = hashlib.md5() + + # Add Platform level hash + if ('PlatformHash') in gDict: + m.update(gDict[('PlatformHash')].encode('utf-8')) + else: + EdkLogger.quiet("[cache warning]: PlatformHash is missing") + + # Add Package level hash + if self.DependentPackageList: + for Pkg in sorted(self.DependentPackageList, key=lambda x: x.PackageName): + if (Pkg.PackageName, 'PackageHash') in gDict: + m.update(gDict[(Pkg.PackageName, 'PackageHash')].encode('utf-8')) + else: + EdkLogger.quiet("[cache warning]: %s PackageHash needed by %s[%s] is missing" %(Pkg.PackageName, self.MetaFile.Name, self.Arch)) + + # Add Library hash + if self.LibraryAutoGenList: + for Lib in sorted(self.LibraryAutoGenList, key=lambda x: x.Name): + if not (Lib.MetaFile.Path, Lib.Arch) in gDict or \ + not gDict[(Lib.MetaFile.Path, Lib.Arch)].ModuleFilesHashDigest: + Lib.GenPreMakefileHash(gDict) + m.update(gDict[(Lib.MetaFile.Path, Lib.Arch)].ModuleFilesHashDigest) + + # Add Module self + m.update(gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest) + + with GlobalData.file_lock: + IR = gDict[(self.MetaFile.Path, self.Arch)] + IR.PreMakefileHashHexDigest = m.hexdigest() + gDict[(self.MetaFile.Path, self.Arch)] = IR + + return gDict[(self.MetaFile.Path, self.Arch)] + + def GenMakeHeaderFilesHash(self, gDict): + # Early exit if module or library has been hashed and is in memory + if (self.MetaFile.Path, self.Arch) in gDict and \ + gDict[(self.MetaFile.Path, self.Arch)].MakeHeaderFilesHashDigest: + return gDict[(self.MetaFile.Path, self.Arch)] + + # skip binary module + if self.IsBinaryModule: + return + + if not (self.MetaFile.Path, self.Arch) in gDict or \ + not gDict[(self.MetaFile.Path, self.Arch)].CreateCodeFileDone: + if self.IsLibrary: + if (self.MetaFile.File,self.MetaFile.Root,self.Arch,self.MetaFile.Path) in GlobalData.libConstPcd: + self.ConstPcd = GlobalData.libConstPcd[(self.MetaFile.File,self.MetaFile.Root,self.Arch,self.MetaFile.Path)] + if (self.MetaFile.File,self.MetaFile.Root,self.Arch,self.MetaFile.Path) in GlobalData.Refes: + self.ReferenceModules = GlobalData.Refes[(self.MetaFile.File,self.MetaFile.Root,self.Arch,self.MetaFile.Path)] + self.CreateCodeFile() + if not (self.MetaFile.Path, self.Arch) in gDict or \ + not gDict[(self.MetaFile.Path, self.Arch)].CreateMakeFileDone: + self.CreateMakeFile(GenFfsList=GlobalData.FfsCmd.get((self.MetaFile.File, self.Arch),[])) + + if not (self.MetaFile.Path, self.Arch) in gDict or \ + not gDict[(self.MetaFile.Path, self.Arch)].CreateCodeFileDone or \ + not gDict[(self.MetaFile.Path, self.Arch)].CreateMakeFileDone: + EdkLogger.quiet("[cache warning]: Cannot create CodeFile or Makefile for module %s[%s]" %(self.MetaFile.Path, self.Arch)) + return + + DependencyFileSet = set() + # Add Makefile + if gDict[(self.MetaFile.Path, self.Arch)].MakefilePath: + DependencyFileSet.add(gDict[(self.MetaFile.Path, self.Arch)].MakefilePath) + else: + EdkLogger.quiet("[cache warning]: makefile is missing for module %s[%s]" %(self.MetaFile.Path, self.Arch)) + + # Add header files + if gDict[(self.MetaFile.Path, self.Arch)].DependencyHeaderFileSet: + for File in gDict[(self.MetaFile.Path, self.Arch)].DependencyHeaderFileSet: + DependencyFileSet.add(File) + else: + EdkLogger.quiet("[cache warning]: No dependency header found for module %s[%s]" %(self.MetaFile.Path, self.Arch)) + + # Add AutoGen files + if self.AutoGenFileList: + for File in set(self.AutoGenFileList): + DependencyFileSet.add(File) + + # Caculate all above dependency files hash + # Initialze hash object + FileList = [] + m = hashlib.md5() + for File in sorted(DependencyFileSet, key=lambda x: str(x)): + if not os.path.exists(str(File)): + EdkLogger.quiet("[cache warning]: header file: %s doesn't exist for module: %s[%s]" % (File, self.MetaFile.Path, self.Arch)) + continue + f = open(str(File), 'rb') + Content = f.read() + f.close() + m.update(Content) + FileList.append((str(File), hashlib.md5(Content).hexdigest())) + + with GlobalData.file_lock: + IR = gDict[(self.MetaFile.Path, self.Arch)] + IR.AutoGenFileList = self.AutoGenFileList.keys() + IR.MakeHeaderFilesHashChain = FileList + IR.MakeHeaderFilesHashDigest = m.digest() + gDict[(self.MetaFile.Path, self.Arch)] = IR + + return gDict[(self.MetaFile.Path, self.Arch)] + + def GenMakeHash(self, gDict): + # Early exit if module or library has been hashed and is in memory + if (self.MetaFile.Path, self.Arch) in gDict and \ + gDict[(self.MetaFile.Path, self.Arch)].MakeHashChain: + return gDict[(self.MetaFile.Path, self.Arch)] + + # skip binary module + if self.IsBinaryModule: + return + + if not (self.MetaFile.Path, self.Arch) in gDict or \ + not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest: + self.GenModuleFilesHash(gDict) + if not gDict[(self.MetaFile.Path, self.Arch)].MakeHeaderFilesHashDigest: + self.GenMakeHeaderFilesHash(gDict) + + if not (self.MetaFile.Path, self.Arch) in gDict or \ + not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest or \ + not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesChain or \ + not gDict[(self.MetaFile.Path, self.Arch)].MakeHeaderFilesHashDigest or \ + not gDict[(self.MetaFile.Path, self.Arch)].MakeHeaderFilesHashChain: + EdkLogger.quiet("[cache warning]: Cannot generate ModuleFilesHash or MakeHeaderFilesHash for module %s[%s]" %(self.MetaFile.Path, self.Arch)) + return + + # Initialze hash object + m = hashlib.md5() + MakeHashChain = [] + + # Add hash of makefile and dependency header files + m.update(gDict[(self.MetaFile.Path, self.Arch)].MakeHeaderFilesHashDigest) + New = list(set(gDict[(self.MetaFile.Path, self.Arch)].MakeHeaderFilesHashChain) - set(MakeHashChain)) + New.sort(key=lambda x: str(x)) + MakeHashChain += New + + # Add Library hash + if self.LibraryAutoGenList: + for Lib in sorted(self.LibraryAutoGenList, key=lambda x: x.Name): + if not (Lib.MetaFile.Path, Lib.Arch) in gDict or \ + not gDict[(Lib.MetaFile.Path, Lib.Arch)].MakeHashChain: + Lib.GenMakeHash(gDict) + if not gDict[(Lib.MetaFile.Path, Lib.Arch)].MakeHashDigest: + print("Cannot generate MakeHash for lib module:", Lib.MetaFile.Path, Lib.Arch) + continue + m.update(gDict[(Lib.MetaFile.Path, Lib.Arch)].MakeHashDigest) + New = list(set(gDict[(Lib.MetaFile.Path, Lib.Arch)].MakeHashChain) - set(MakeHashChain)) + New.sort(key=lambda x: str(x)) + MakeHashChain += New + + # Add Module self + m.update(gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest) + New = list(set(gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesChain) - set(MakeHashChain)) + New.sort(key=lambda x: str(x)) + MakeHashChain += New + + with GlobalData.file_lock: + IR = gDict[(self.MetaFile.Path, self.Arch)] + IR.MakeHashDigest = m.digest() + IR.MakeHashHexDigest = m.hexdigest() + IR.MakeHashChain = MakeHashChain + gDict[(self.MetaFile.Path, self.Arch)] = IR + + return gDict[(self.MetaFile.Path, self.Arch)] + + ## Decide whether we can skip the left autogen and make process + def CanSkipbyPreMakefileCache(self, gDict): + if not GlobalData.gBinCacheSource: return False - # Initialize a dictionary for each arch type - if self.Arch not in GlobalData.gBuildHashSkipTracking: - GlobalData.gBuildHashSkipTracking[self.Arch] = dict() - - # If library or Module is binary do not skip by hash + # If Module is binary, do not skip by cache if self.IsBinaryModule: return False @@ -1873,27 +2234,191 @@ class ModuleAutoGen(AutoGen): if '.inc' in str(f_ext): return False - # Use Cache, if exists and if Module has a copy in cache - if GlobalData.gBinCacheSource and self.AttemptModuleCacheCopy(): - return True + # Get the module hash values from stored cache and currrent build + # then check whether cache hit based on the hash values + # if cache hit, restore all the files from cache + FileDir = path.join(GlobalData.gBinCacheSource, self.PlatformInfo.OutputDir, self.BuildTarget + "_" + self.ToolChain, self.Arch, self.SourceDir, self.MetaFile.BaseName) + FfsDir = path.join(GlobalData.gBinCacheSource, self.PlatformInfo.OutputDir, self.BuildTarget + "_" + self.ToolChain, TAB_FV_DIRECTORY, "Ffs", self.Guid + self.Name) - # Early exit for libraries that haven't yet finished building - HashFile = path.join(self.BuildDir, self.Name + ".hash") - if self.IsLibrary and not os.path.exists(HashFile): + ModuleHashPairList = [] # tuple list: [tuple(PreMakefileHash, MakeHash)] + ModuleHashPair = path.join(FileDir, self.Name + ".ModuleHashPair") + if not os.path.exists(ModuleHashPair): + EdkLogger.quiet("[cache warning]: Cannot find ModuleHashPair file: %s" % ModuleHashPair) return False - # Return a Boolean based on if can skip by hash, either from memory or from IO. - if self.Name not in GlobalData.gBuildHashSkipTracking[self.Arch]: - # If hashes are the same, SaveFileOnChange() will return False. - GlobalData.gBuildHashSkipTracking[self.Arch][self.Name] = not SaveFileOnChange(HashFile, self.GenModuleHash(), True) - return GlobalData.gBuildHashSkipTracking[self.Arch][self.Name] - else: - return GlobalData.gBuildHashSkipTracking[self.Arch][self.Name] + try: + f = open(ModuleHashPair, 'r') + ModuleHashPairList = json.load(f) + f.close() + except: + EdkLogger.quiet("[cache warning]: fail to load ModuleHashPair file: %s" % ModuleHashPair) + return False + + self.GenPreMakefileHash(gDict) + if not (self.MetaFile.Path, self.Arch) in gDict or \ + not gDict[(self.MetaFile.Path, self.Arch)].PreMakefileHashHexDigest: + EdkLogger.quiet("[cache warning]: PreMakefileHashHexDigest is missing for module %s[%s]" %(self.MetaFile.Path, self.Arch)) + return False + + MakeHashStr = None + CurrentPreMakeHash = gDict[(self.MetaFile.Path, self.Arch)].PreMakefileHashHexDigest + for idx, (PreMakefileHash, MakeHash) in enumerate (ModuleHashPairList): + if PreMakefileHash == CurrentPreMakeHash: + MakeHashStr = str(MakeHash) + + if not MakeHashStr: + return False + + TargetHashDir = path.join(FileDir, MakeHashStr) + TargetFfsHashDir = path.join(FfsDir, MakeHashStr) + + if not os.path.exists(TargetHashDir): + EdkLogger.quiet("[cache warning]: Cache folder is missing: %s" % TargetHashDir) + return False + + for root, dir, files in os.walk(TargetHashDir): + for f in files: + File = path.join(root, f) + self.CacheCopyFile(self.OutputDir, TargetHashDir, File) + if os.path.exists(TargetFfsHashDir): + for root, dir, files in os.walk(TargetFfsHashDir): + for f in files: + File = path.join(root, f) + self.CacheCopyFile(self.FfsOutputDir, TargetFfsHashDir, File) + + if self.Name == "PcdPeim" or self.Name == "PcdDxe": + CreatePcdDatabaseCode(self, TemplateString(), TemplateString()) + + with GlobalData.file_lock: + IR = gDict[(self.MetaFile.Path, self.Arch)] + IR.PreMakeCacheHit = True + gDict[(self.MetaFile.Path, self.Arch)] = IR + print("[cache hit]: checkpoint_PreMakefile:", self.MetaFile.Path, self.Arch) + #EdkLogger.quiet("cache hit: %s[%s]" % (self.MetaFile.Path, self.Arch)) + return True + + ## Decide whether we can skip the make process + def CanSkipbyMakeCache(self, gDict): + if not GlobalData.gBinCacheSource: + return False + + # If Module is binary, do not skip by cache + if self.IsBinaryModule: + print("[cache miss]: checkpoint_Makefile: binary module:", self.MetaFile.Path, self.Arch) + return False + + # .inc is contains binary information so do not skip by hash as well + for f_ext in self.SourceFileList: + if '.inc' in str(f_ext): + with GlobalData.file_lock: + IR = gDict[(self.MetaFile.Path, self.Arch)] + IR.MakeCacheHit = False + gDict[(self.MetaFile.Path, self.Arch)] = IR + print("[cache miss]: checkpoint_Makefile: .inc module:", self.MetaFile.Path, self.Arch) + return False + + # Get the module hash values from stored cache and currrent build + # then check whether cache hit based on the hash values + # if cache hit, restore all the files from cache + FileDir = path.join(GlobalData.gBinCacheSource, self.PlatformInfo.OutputDir, self.BuildTarget + "_" + self.ToolChain, self.Arch, self.SourceDir, self.MetaFile.BaseName) + FfsDir = path.join(GlobalData.gBinCacheSource, self.PlatformInfo.OutputDir, self.BuildTarget + "_" + self.ToolChain, TAB_FV_DIRECTORY, "Ffs", self.Guid + self.Name) + + ModuleHashPairList = [] # tuple list: [tuple(PreMakefileHash, MakeHash)] + ModuleHashPair = path.join(FileDir, self.Name + ".ModuleHashPair") + if not os.path.exists(ModuleHashPair): + EdkLogger.quiet("[cache warning]: Cannot find ModuleHashPair file: %s" % ModuleHashPair) + return False + + try: + f = open(ModuleHashPair, 'r') + ModuleHashPairList = json.load(f) + f.close() + except: + EdkLogger.quiet("[cache warning]: fail to load ModuleHashPair file: %s" % ModuleHashPair) + return False + + self.GenMakeHash(gDict) + if not (self.MetaFile.Path, self.Arch) in gDict or \ + not gDict[(self.MetaFile.Path, self.Arch)].MakeHashHexDigest: + EdkLogger.quiet("[cache warning]: MakeHashHexDigest is missing for module %s[%s]" %(self.MetaFile.Path, self.Arch)) + return False + + MakeHashStr = None + CurrentMakeHash = gDict[(self.MetaFile.Path, self.Arch)].MakeHashHexDigest + for idx, (PreMakefileHash, MakeHash) in enumerate (ModuleHashPairList): + if MakeHash == CurrentMakeHash: + MakeHashStr = str(MakeHash) + + if not MakeHashStr: + print("[cache miss]: checkpoint_Makefile:", self.MetaFile.Path, self.Arch) + return False + + TargetHashDir = path.join(FileDir, MakeHashStr) + TargetFfsHashDir = path.join(FfsDir, MakeHashStr) + if not os.path.exists(TargetHashDir): + EdkLogger.quiet("[cache warning]: Cache folder is missing: %s" % TargetHashDir) + return False + + for root, dir, files in os.walk(TargetHashDir): + for f in files: + File = path.join(root, f) + self.CacheCopyFile(self.OutputDir, TargetHashDir, File) + + if os.path.exists(TargetFfsHashDir): + for root, dir, files in os.walk(TargetFfsHashDir): + for f in files: + File = path.join(root, f) + self.CacheCopyFile(self.FfsOutputDir, TargetFfsHashDir, File) + + if self.Name == "PcdPeim" or self.Name == "PcdDxe": + CreatePcdDatabaseCode(self, TemplateString(), TemplateString()) + with GlobalData.file_lock: + IR = gDict[(self.MetaFile.Path, self.Arch)] + IR.MakeCacheHit = True + gDict[(self.MetaFile.Path, self.Arch)] = IR + print("[cache hit]: checkpoint_Makefile:", self.MetaFile.Path, self.Arch) + return True + + ## Decide whether we can skip the ModuleAutoGen process + def CanSkipbyCache(self, gDict): + # Hashing feature is off + if not GlobalData.gBinCacheSource: + return False + + if self in GlobalData.gBuildHashSkipTracking: + return GlobalData.gBuildHashSkipTracking[self] + + # If library or Module is binary do not skip by hash + if self.IsBinaryModule: + GlobalData.gBuildHashSkipTracking[self] = False + return False + + # .inc is contains binary information so do not skip by hash as well + for f_ext in self.SourceFileList: + if '.inc' in str(f_ext): + GlobalData.gBuildHashSkipTracking[self] = False + return False + + if not (self.MetaFile.Path, self.Arch) in gDict: + return False + + if gDict[(self.MetaFile.Path, self.Arch)].PreMakeCacheHit: + GlobalData.gBuildHashSkipTracking[self] = True + return True + + if gDict[(self.MetaFile.Path, self.Arch)].MakeCacheHit: + GlobalData.gBuildHashSkipTracking[self] = True + return True + + return False ## Decide whether we can skip the ModuleAutoGen process # If any source file is newer than the module than we cannot skip # def CanSkip(self): + # Don't skip if cache feature enabled + if GlobalData.gUseHashCache or GlobalData.gBinCacheDest or GlobalData.gBinCacheSource: + return False if self.MakeFileDir in GlobalData.gSikpAutoGenCache: return True if not os.path.exists(self.TimeStampPath): diff --git a/BaseTools/Source/Python/Common/GlobalData.py b/BaseTools/Source/Python/Common/GlobalData.py old mode 100644 new mode 100755 index bd45a43728..9ea835314a --- a/BaseTools/Source/Python/Common/GlobalData.py +++ b/BaseTools/Source/Python/Common/GlobalData.py @@ -119,3 +119,12 @@ gModuleBuildTracking = dict() # Top Dict: Key: Arch Type Value: Dictionary # Second Dict: Key: Module\Library Name Value: True\False gBuildHashSkipTracking = dict() + +# Common dictionary to share module cache intermediate result and state +gCacheIR = None +# Common lock for the file access in multiple process AutoGens +file_lock = None +# Common dictionary to share platform libraries' constant Pcd +libConstPcd = None +# Common dictionary to share platform libraries' reference info +Refes = None \ No newline at end of file diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py old mode 100644 new mode 100755 index 4bfa54666b..d7c817b95c --- a/BaseTools/Source/Python/build/build.py +++ b/BaseTools/Source/Python/build/build.py @@ -595,7 +595,7 @@ class BuildTask: # def AddDependency(self, Dependency): for Dep in Dependency: - if not Dep.BuildObject.IsBinaryModule and not Dep.BuildObject.CanSkipbyHash(): + if not Dep.BuildObject.IsBinaryModule and not Dep.BuildObject.CanSkipbyCache(GlobalData.gCacheIR): self.DependencyList.append(BuildTask.New(Dep)) # BuildTask list ## The thread wrapper of LaunchCommand function @@ -811,7 +811,7 @@ class Build(): self.AutoGenMgr = None EdkLogger.info("") os.chdir(self.WorkspaceDir) - self.share_data = Manager().dict() + GlobalData.gCacheIR = Manager().dict() self.log_q = log_q def StartAutoGen(self,mqueue, DataPipe,SkipAutoGen,PcdMaList,share_data): try: @@ -820,6 +820,13 @@ class Build(): feedback_q = mp.Queue() file_lock = mp.Lock() error_event = mp.Event() + GlobalData.file_lock = file_lock + FfsCmd = DataPipe.Get("FfsCommand") + if FfsCmd is None: + FfsCmd = {} + GlobalData.FfsCmd = FfsCmd + GlobalData.libConstPcd = DataPipe.Get("LibConstPcd") + GlobalData.Refes = DataPipe.Get("REFS") auto_workers = [AutoGenWorkerInProcess(mqueue,DataPipe.dump_file,feedback_q,file_lock,share_data,self.log_q,error_event) for _ in range(self.ThreadNumber)] self.AutoGenMgr = AutoGenManager(auto_workers,feedback_q,error_event) self.AutoGenMgr.start() @@ -827,14 +834,28 @@ class Build(): w.start() if PcdMaList is not None: for PcdMa in PcdMaList: + if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]: + PcdMa.GenModuleFilesHash(share_data) + PcdMa.GenPreMakefileHash(share_data) + if PcdMa.CanSkipbyPreMakefileCache(share_data): + continue + PcdMa.CreateCodeFile(False) PcdMa.CreateMakeFile(False,GenFfsList = DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.File, PcdMa.Arch),[])) + if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]: + PcdMa.GenMakeHeaderFilesHash(share_data) + PcdMa.GenMakeHash(share_data) + if PcdMa.CanSkipbyMakeCache(share_data): + continue + self.AutoGenMgr.join() rt = self.AutoGenMgr.Status return rt, 0 - except Exception as e: - return False,e.errcode + except FatalError as e: + return False, e.args[0] + except: + return False, UNKNOWN_ERROR ## Load configuration # @@ -1199,10 +1220,11 @@ class Build(): mqueue.put(m) AutoGenObject.DataPipe.DataContainer = {"FfsCommand":FfsCommand} + AutoGenObject.DataPipe.DataContainer = {"CommandTarget": self.Target} self.Progress.Start("Generating makefile and code") data_pipe_file = os.path.join(AutoGenObject.BuildDir, "GlobalVar_%s_%s.bin" % (str(AutoGenObject.Guid),AutoGenObject.Arch)) AutoGenObject.DataPipe.dump(data_pipe_file) - autogen_rt, errorcode = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList,self.share_data) + autogen_rt,errorcode = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList, GlobalData.gCacheIR) self.Progress.Stop("done!") if not autogen_rt: self.AutoGenMgr.TerminateWorkers() @@ -1799,6 +1821,15 @@ class Build(): CmdListDict = None if GlobalData.gEnableGenfdsMultiThread and self.Fdf: CmdListDict = self._GenFfsCmd(Wa.ArchList) + + # Add Platform and Package level hash in share_data for module hash calculation later + if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest: + GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash + for PkgName in GlobalData.gPackageHash.keys(): + GlobalData.gCacheIR[(PkgName, 'PackageHash')] = GlobalData.gPackageHash[PkgName] + GlobalData.file_lock = mp.Lock() + GlobalData.FfsCmd = CmdListDict + self.Progress.Stop("done!") MaList = [] ExitFlag = threading.Event() @@ -1808,20 +1839,23 @@ class Build(): AutoGenStart = time.time() GlobalData.gGlobalDefines['ARCH'] = Arch Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch) + GlobalData.libConstPcd = Pa.DataPipe.Get("LibConstPcd") + GlobalData.Refes = Pa.DataPipe.Get("REFS") for Module in Pa.Platform.Modules: if self.ModuleFile.Dir == Module.Dir and self.ModuleFile.Name == Module.Name: Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe) if Ma is None: continue MaList.append(Ma) - if Ma.CanSkipbyHash(): - self.HashSkipModules.append(Ma) - if GlobalData.gBinCacheSource: - EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch)) - continue - else: - if GlobalData.gBinCacheSource: - EdkLogger.quiet("cache miss: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch)) + + if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]: + Ma.GenModuleFilesHash(GlobalData.gCacheIR) + Ma.GenPreMakefileHash(GlobalData.gCacheIR) + if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR): + self.HashSkipModules.append(Ma) + EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch)) + continue + # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds' if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']: # for target which must generate AutoGen code and makefile @@ -1841,6 +1875,18 @@ class Build(): self.Progress.Stop("done!") if self.Target == "genmake": return True + + if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]: + Ma.GenMakeHeaderFilesHash(GlobalData.gCacheIR) + Ma.GenMakeHash(GlobalData.gCacheIR) + if Ma.CanSkipbyMakeCache(GlobalData.gCacheIR): + self.HashSkipModules.append(Ma) + EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch)) + continue + else: + EdkLogger.quiet("cache miss: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch)) + Ma.PrintFirstMakeCacheMissFile(GlobalData.gCacheIR) + self.BuildModules.append(Ma) # Initialize all modules in tracking to 'FAIL' if Ma.Arch not in GlobalData.gModuleBuildTracking: @@ -1985,11 +2031,18 @@ class Build(): if GlobalData.gEnableGenfdsMultiThread and self.Fdf: CmdListDict = self._GenFfsCmd(Wa.ArchList) + # Add Platform and Package level hash in share_data for module hash calculation later + if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest: + GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash + for PkgName in GlobalData.gPackageHash.keys(): + GlobalData.gCacheIR[(PkgName, 'PackageHash')] = GlobalData.gPackageHash[PkgName] + # multi-thread exit flag ExitFlag = threading.Event() ExitFlag.clear() self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime))) self.BuildModules = [] + TotalModules = [] for Arch in Wa.ArchList: PcdMaList = [] AutoGenStart = time.time() @@ -2009,6 +2062,7 @@ class Build(): ModuleList.append(Inf) Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict} Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp} + Pa.DataPipe.DataContainer = {"CommandTarget": self.Target} for Module in ModuleList: # Get ModuleAutoGen object to generate C code file and makefile Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe) @@ -2019,30 +2073,34 @@ class Build(): Ma.PlatformInfo = Pa Ma.Workspace = Wa PcdMaList.append(Ma) - if Ma.CanSkipbyHash(): - self.HashSkipModules.append(Ma) - if GlobalData.gBinCacheSource: - EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch)) - continue - else: - if GlobalData.gBinCacheSource: - EdkLogger.quiet("cache miss: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch)) - - # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds' - # for target which must generate AutoGen code and makefile - - self.BuildModules.append(Ma) + TotalModules.append(Ma) # Initialize all modules in tracking to 'FAIL' if Ma.Arch not in GlobalData.gModuleBuildTracking: GlobalData.gModuleBuildTracking[Ma.Arch] = dict() if Ma not in GlobalData.gModuleBuildTracking[Ma.Arch]: GlobalData.gModuleBuildTracking[Ma.Arch][Ma] = 'FAIL' + mqueue = mp.Queue() for m in Pa.GetAllModuleInfo: mqueue.put(m) data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch)) Pa.DataPipe.dump(data_pipe_file) - autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList,self.share_data) + autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList, GlobalData.gCacheIR) + + # Skip cache hit modules + if GlobalData.gBinCacheSource: + for Ma in TotalModules: + if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \ + GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].PreMakeCacheHit: + self.HashSkipModules.append(Ma) + continue + if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \ + GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].MakeCacheHit: + self.HashSkipModules.append(Ma) + continue + self.BuildModules.append(Ma) + else: + self.BuildModules.extend(TotalModules) if not autogen_rt: self.AutoGenMgr.TerminateWorkers() @@ -2050,9 +2108,24 @@ class Build(): raise FatalError(errorcode) self.AutoGenTime += int(round((time.time() - AutoGenStart))) self.Progress.Stop("done!") + + if GlobalData.gBinCacheSource: + EdkLogger.quiet("Total cache hit driver num: %s, cache miss driver num: %s" % (len(set(self.HashSkipModules)), len(set(self.BuildModules)))) + CacheHitMa = set() + CacheNotHitMa = set() + for IR in GlobalData.gCacheIR.keys(): + if 'PlatformHash' in IR or 'PackageHash' in IR: + continue + if GlobalData.gCacheIR[IR].PreMakeCacheHit or GlobalData.gCacheIR[IR].MakeCacheHit: + CacheHitMa.add(IR) + else: + # There might be binary module or module which has .inc files, not count for cache miss + CacheNotHitMa.add(IR) + EdkLogger.quiet("Total module num: %s, cache hit module num: %s" % (len(CacheHitMa)+len(CacheNotHitMa), len(CacheHitMa))) + for Arch in Wa.ArchList: MakeStart = time.time() - for Ma in self.BuildModules: + for Ma in set(self.BuildModules): # Generate build task for the module if not Ma.IsBinaryModule: Bt = BuildTask.New(ModuleMakeUnit(Ma, Pa.BuildCommand,self.Target))