sravan/system76-edk2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

BaseTools: Leverage compiler output to optimize binary cache

Browse Source 
Redesign the binary cache and bases on the compiler to
output the dependency header files info for every module.
The binary cache will directly consume the dependency header
files info and doesn't parse the C source code by iteself.
Also redesign the dependency files list format for module
and try to share the common lib hash result as more as
possible in local process. Remove the unnecessary share data
access across multiprocessing.

Signed-off-by: Steven Shi <steven.shi@intel.com>

Cc: Liming Gao <liming.gao@intel.com>
Cc: Bob Feng <bob.c.feng@intel.com>
Reviewed-by: Bob Feng <bob.c.feng@intel.com>
This commit is contained in:
Steven Shi
2019-11-19 16:17:00 +08:00
committed by mergify[bot]
parent 3bfbc91507
commit fc8b8deac2
6 changed files with 786 additions and 894 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
View File

@ -23,6 +23,7 @@ from Common.StringUtils import NormPath
from Common.BuildToolError import *
from Common.DataType import *
from Common.Misc import *
import json
## Regular expression for splitting Dependency Expression string into tokens
gDepexTokenPattern = re.compile("(\(|\)|\w+| \S+\.inf)")
@ -127,7 +128,7 @@ class WorkspaceAutoGen(AutoGen):
self.CreateBuildOptionsFile()
self.CreatePcdTokenNumberFile()
self.CreateModuleHashInfo()
self.GeneratePlatformLevelHash()
#
# Merge Arch
@ -528,12 +529,12 @@ class WorkspaceAutoGen(AutoGen):
PcdTokenNumber += str(Pa.PcdTokenNumber[Pcd.TokenCName, Pcd.TokenSpaceGuidCName])
SaveFileOnChange(os.path.join(self.BuildDir, 'PcdTokenNumber'), PcdTokenNumber, False)
def CreateModuleHashInfo(self):
def GeneratePlatformLevelHash(self):
#
# Get set of workspace metafiles
#
AllWorkSpaceMetaFiles = self._GetMetaFiles(self.BuildTarget, self.ToolChain)
AllWorkSpaceMetaFileList = sorted(AllWorkSpaceMetaFiles, key=lambda x: str(x))
#
# Retrieve latest modified time of all metafiles
#
@ -544,16 +545,35 @@ class WorkspaceAutoGen(AutoGen):
self._SrcTimeStamp = SrcTimeStamp
if GlobalData.gUseHashCache:
FileList = []
m = hashlib.md5()
for files in AllWorkSpaceMetaFiles:
if files.endswith('.dec'):
for file in AllWorkSpaceMetaFileList:
if file.endswith('.dec'):
continue
f = open(files, 'rb')
f = open(file, 'rb')
Content = f.read()
f.close()
m.update(Content)
SaveFileOnChange(os.path.join(self.BuildDir, 'AutoGen.hash'), m.hexdigest(), False)
GlobalData.gPlatformHash = m.hexdigest()
FileList.append((str(file), hashlib.md5(Content).hexdigest()))
HashDir = path.join(self.BuildDir, "Hash_Platform")
HashFile = path.join(HashDir, 'Platform.hash.' + m.hexdigest())
SaveFileOnChange(HashFile, m.hexdigest(), False)
HashChainFile = path.join(HashDir, 'Platform.hashchain.' + m.hexdigest())
GlobalData.gPlatformHashFile = HashChainFile
try:
with open(HashChainFile, 'w') as f:
json.dump(FileList, f, indent=2)
except:
EdkLogger.quiet("[cache warning]: fail to save hashchain file:%s" % HashChainFile)
if GlobalData.gBinCacheDest:
# Copy platform hash files to cache destination
FileDir = path.join(GlobalData.gBinCacheDest, self.OutputDir, self.BuildTarget + "_" + self.ToolChain, "Hash_Platform")
CacheFileDir = FileDir
CreateDirectory(CacheFileDir)
CopyFileOnChange(HashFile, CacheFileDir)
CopyFileOnChange(HashChainFile, CacheFileDir)
#
# Write metafile list to build directory
@ -564,7 +584,7 @@ class WorkspaceAutoGen(AutoGen):
if not os.path.exists(self.BuildDir):
os.makedirs(self.BuildDir)
with open(os.path.join(self.BuildDir, 'AutoGen'), 'w+') as file:
for f in AllWorkSpaceMetaFiles:
for f in AllWorkSpaceMetaFileList:
print(f, file=file)
return True
@ -572,15 +592,16 @@ class WorkspaceAutoGen(AutoGen):
if Pkg.PackageName in GlobalData.gPackageHash:
return
PkgDir = os.path.join(self.BuildDir, Pkg.Arch, Pkg.PackageName)
PkgDir = os.path.join(self.BuildDir, Pkg.Arch, "Hash_Pkg", Pkg.PackageName)
CreateDirectory(PkgDir)
HashFile = os.path.join(PkgDir, Pkg.PackageName + '.hash')
FileList = []
m = hashlib.md5()
# Get .dec file's hash value
f = open(Pkg.MetaFile.Path, 'rb')
Content = f.read()
f.close()
m.update(Content)
FileList.append((str(Pkg.MetaFile.Path), hashlib.md5(Content).hexdigest()))
# Get include files hash value
if Pkg.Includes:
for inc in sorted(Pkg.Includes, key=lambda x: str(x)):
@ -591,9 +612,28 @@ class WorkspaceAutoGen(AutoGen):
Content = f.read()
f.close()
m.update(Content)
SaveFileOnChange(HashFile, m.hexdigest(), False)
FileList.append((str(File_Path), hashlib.md5(Content).hexdigest()))
GlobalData.gPackageHash[Pkg.PackageName] = m.hexdigest()
HashDir = PkgDir
HashFile = path.join(HashDir, Pkg.PackageName + '.hash.' + m.hexdigest())
SaveFileOnChange(HashFile, m.hexdigest(), False)
HashChainFile = path.join(HashDir, Pkg.PackageName + '.hashchain.' + m.hexdigest())
GlobalData.gPackageHashFile[(Pkg.PackageName, Pkg.Arch)] = HashChainFile
try:
with open(HashChainFile, 'w') as f:
json.dump(FileList, f, indent=2)
except:
EdkLogger.quiet("[cache warning]: fail to save hashchain file:%s" % HashChainFile)
if GlobalData.gBinCacheDest:
# Copy Pkg hash files to cache destination dir
FileDir = path.join(GlobalData.gBinCacheDest, self.OutputDir, self.BuildTarget + "_" + self.ToolChain, Pkg.Arch, "Hash_Pkg", Pkg.PackageName)
CacheFileDir = FileDir
CreateDirectory(CacheFileDir)
CopyFileOnChange(HashFile, CacheFileDir)
CopyFileOnChange(HashChainFile, CacheFileDir)
def _GetMetaFiles(self, Target, Toolchain):
AllWorkSpaceMetaFiles = set()
#