sravan/system76-edk2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

BaseTools: Fixed issue for IgnoreAutoGen

Browse Source 
https://bugzilla.tianocore.org/show_bug.cgi?id=2080

This patch is to improve build -u option to re-use
GlobalVar_<platformguid>_<arch>.bin file which is
introduced by multiple-process-autogen feature.

Cc: Liming Gao <liming.gao@intel.com>
Cc: Steven Shi <steven.shi@intel.com>
Signed-off-by: Bob Feng <bob.c.feng@intel.com>
Acked-by: Liming Gao <liming.gao@intel.com>
This commit is contained in:
Feng, Bob C
2019-09-05 17:04:58 +08:00
parent 53e2eaba7a
commit 373298ca0d
6 changed files with 253 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
View File

@ -236,6 +236,10 @@ class AutoGenInfo(object):
#
class WorkSpaceInfo(AutoGenInfo):
def __init__(self,Workspace, MetaFile, Target, ToolChain, Arch):
if not hasattr(self, "_Init"):
self.do_init(Workspace, MetaFile, Target, ToolChain, Arch)
self._Init = True
def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch):
self._SrcTimeStamp = 0
self.Db = BuildDB
self.BuildDatabase = self.Db.BuildObject
@ -244,10 +248,35 @@ class WorkSpaceInfo(AutoGenInfo):
self.WorkspaceDir = Workspace
self.ActivePlatform = MetaFile
self.ArchList = Arch
self.AutoGenObjectList = []
@property
def BuildDir(self):
return self.AutoGenObjectList[0].BuildDir
@property
def Name(self):
return self.AutoGenObjectList[0].Platform.PlatformName
@property
def FlashDefinition(self):
return self.AutoGenObjectList[0].Platform.FlashDefinition
@property
def GenFdsCommandDict(self):
FdsCommandDict = self.AutoGenObjectList[0].DataPipe.Get("FdsCommandDict")
if FdsCommandDict:
return FdsCommandDict
return {}
@cached_property
def FvDir(self):
return os.path.join(self.BuildDir, TAB_FV_DIRECTORY)
class PlatformInfo(AutoGenInfo):
def __init__(self, Workspace, MetaFile, Target, ToolChain, Arch,DataPipe):
if not hasattr(self, "_Init"):
self.do_init(Workspace, MetaFile, Target, ToolChain, Arch,DataPipe)
self._Init = True
def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch,DataPipe):
self.Wa = Workspace
self.WorkspaceDir = self.Wa.WorkspaceDir
self.MetaFile = MetaFile

2
BaseTools/Source/Python/AutoGen/PlatformAutoGen.py
View File

@ -188,7 +188,9 @@ class PlatformAutoGen(AutoGen):
Ma.CreateMakeFile(CreateModuleMakeFile, FfsCommand[key])
else:
Ma.CreateMakeFile(CreateModuleMakeFile)
self.CreateLibModuelDirs()
def CreateLibModuelDirs(self):
# no need to create makefile for the platform more than once
if self.IsMakeFileCreated:
return

1
BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
View File

@ -128,7 +128,6 @@ class WorkspaceAutoGen(AutoGen):
self.CreateBuildOptionsFile()
self.CreatePcdTokenNumberFile()
self.CreateModuleHashInfo()
GlobalData.gAutoGenPhase = False
#
# Merge Arch

181
BaseTools/Source/Python/build/build.py
View File

@ -54,6 +54,9 @@ import Common.GlobalData as GlobalData
from GenFds.GenFds import GenFds, GenFdsApi
import multiprocessing as mp
from multiprocessing import Manager
from AutoGen.DataPipe import MemoryDataPipe
from AutoGen.ModuleAutoGenHelper import WorkSpaceInfo, PlatformInfo
from GenFds.FdfParser import FdfParser
## standard targets of build command
@ -1218,14 +1221,24 @@ class Build():
mqueue.put(m)
AutoGenObject.DataPipe.DataContainer = {"CommandTarget": self.Target}
AutoGenObject.DataPipe.DataContainer = {"Workspace_timestamp": AutoGenObject.Workspace._SrcTimeStamp}
AutoGenObject.CreateLibModuelDirs()
AutoGenObject.DataPipe.DataContainer = {"LibraryBuildDirectoryList":AutoGenObject.LibraryBuildDirectoryList}
AutoGenObject.DataPipe.DataContainer = {"ModuleBuildDirectoryList":AutoGenObject.ModuleBuildDirectoryList}
AutoGenObject.DataPipe.DataContainer = {"FdsCommandDict": AutoGenObject.Workspace.GenFdsCommandDict}
self.Progress.Start("Generating makefile and code")
data_pipe_file = os.path.join(AutoGenObject.BuildDir, "GlobalVar_%s_%s.bin" % (str(AutoGenObject.Guid),AutoGenObject.Arch))
AutoGenObject.DataPipe.dump(data_pipe_file)
autogen_rt,errorcode = self.StartAutoGen(mqueue, AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList, GlobalData.gCacheIR)
AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
with open(AutoGenIdFile,"w") as fw:
fw.write("Arch=%s\n" % "|".join((AutoGenObject.Workspace.ArchList)))
fw.write("BuildDir=%s\n" % AutoGenObject.Workspace.BuildDir)
fw.write("PlatformGuid=%s\n" % str(AutoGenObject.Guid))
self.Progress.Stop("done!")
if not autogen_rt:
self.AutoGenMgr.TerminateWorkers()
self.AutoGenMgr.join(0.1)
self.AutoGenMgr.join(1)
raise FatalError(errorcode)
AutoGenObject.CreateCodeFile(False)
AutoGenObject.CreateMakeFile(False)
@ -1778,6 +1791,7 @@ class Build():
# Save MAP buffer into MAP file.
#
self._SaveMapFile (MapBuffer, Wa)
self.CreateGuidedSectionToolsFile(Wa)
## Build active module for different build targets, different tool chains and different archs
#
@ -1987,20 +2001,79 @@ class Build():
tmpInf, tmpArch = GenFfsDict[Cmd]
CmdSetDict[tmpInf, tmpArch].add(Cmd)
return CmdSetDict
def VerifyAutoGenFiles(self):
AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
try:
with open(AutoGenIdFile) as fd:
lines = fd.readlines()
except:
return None
for line in lines:
if "Arch" in line:
ArchList = line.strip().split("=")[1].split("|")
if "BuildDir" in line:
BuildDir = line.split("=")[1].strip()
if "PlatformGuid" in line:
PlatformGuid = line.split("=")[1].strip()
GlobalVarList = []
for arch in ArchList:
global_var = os.path.join(BuildDir, "GlobalVar_%s_%s.bin" % (str(PlatformGuid),arch))
if not os.path.exists(global_var):
return None
GlobalVarList.append(global_var)
for global_var in GlobalVarList:
data_pipe = MemoryDataPipe()
data_pipe.load(global_var)
target = data_pipe.Get("P_Info").get("Target")
toolchain = data_pipe.Get("P_Info").get("ToolChain")
archlist = data_pipe.Get("P_Info").get("ArchList")
Arch = data_pipe.Get("P_Info").get("Arch")
active_p = data_pipe.Get("P_Info").get("ActivePlatform")
workspacedir = data_pipe.Get("P_Info").get("WorkspaceDir")
PackagesPath = os.getenv("PACKAGES_PATH")
mws.setWs(workspacedir, PackagesPath)
LibraryBuildDirectoryList = data_pipe.Get("LibraryBuildDirectoryList")
ModuleBuildDirectoryList = data_pipe.Get("ModuleBuildDirectoryList")
for m_build_dir in LibraryBuildDirectoryList:
if not os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[GenMake.gMakeType])):
return None
for m_build_dir in ModuleBuildDirectoryList:
if not os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[GenMake.gMakeType])):
return None
Wa = WorkSpaceInfo(
workspacedir,active_p,target,toolchain,archlist
)
Pa = PlatformInfo(Wa, active_p, target, toolchain, Arch,data_pipe)
Wa.AutoGenObjectList.append(Pa)
return Wa
def SetupMakeSetting(self,Wa):
BuildModules = []
for Pa in Wa.AutoGenObjectList:
for m in Pa._MbList:
ma = ModuleAutoGen(Wa,m.MetaFile, Pa.BuildTarget, Wa.ToolChain, Pa.Arch, Pa.MetaFile,Pa.DataPipe)
BuildModules.append(ma)
fdf_file = Wa.FlashDefinition
if fdf_file:
Fdf = FdfParser(fdf_file.Path)
Fdf.ParseFile()
GlobalData.gFdfParser = Fdf
if Fdf.CurrentFdName and Fdf.CurrentFdName in Fdf.Profile.FdDict:
FdDict = Fdf.Profile.FdDict[Fdf.CurrentFdName]
for FdRegion in FdDict.RegionList:
if str(FdRegion.RegionType) is 'FILE' and self.Platform.VpdToolGuid in str(FdRegion.RegionDataList):
if int(FdRegion.Offset) % 8 != 0:
EdkLogger.error("build", FORMAT_INVALID, 'The VPD Base Address %s must be 8-byte aligned.' % (FdRegion.Offset))
Wa.FdfProfile = Fdf.Profile
self.Fdf = Fdf
else:
self.Fdf = None
return BuildModules
## Build a platform in multi-thread mode
#
def _MultiThreadBuildPlatform(self):
SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False)
for BuildTarget in self.BuildTargetList:
GlobalData.gGlobalDefines['TARGET'] = BuildTarget
index = 0
for ToolChain in self.ToolChainList:
def PerformAutoGen(self,BuildTarget,ToolChain):
WorkspaceAutoGenTime = time.time()
GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain
GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain
GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]
index += 1
Wa = WorkspaceAutoGen(
self.WorkspaceDir,
self.PlatformFile,
@ -2034,11 +2107,8 @@ class Build():
for PkgName in GlobalData.gPackageHash.keys():
GlobalData.gCacheIR[(PkgName, 'PackageHash')] = GlobalData.gPackageHash[PkgName]
# multi-thread exit flag
ExitFlag = threading.Event()
ExitFlag.clear()
self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime)))
self.BuildModules = []
BuildModules = []
TotalModules = []
for Arch in Wa.ArchList:
PcdMaList = []
@ -2060,6 +2130,14 @@ class Build():
Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict}
Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp}
Pa.DataPipe.DataContainer = {"CommandTarget": self.Target}
Pa.CreateLibModuelDirs()
Pa.DataPipe.DataContainer = {"LibraryBuildDirectoryList":Pa.LibraryBuildDirectoryList}
Pa.DataPipe.DataContainer = {"ModuleBuildDirectoryList":Pa.ModuleBuildDirectoryList}
Pa.DataPipe.DataContainer = {"FdsCommandDict": Wa.GenFdsCommandDict}
ModuleCodaFile = {}
for ma in Pa.ModuleAutoGenList:
ModuleCodaFile[(ma.MetaFile.File,ma.MetaFile.Root,ma.Arch,ma.MetaFile.Path)] = [item.Target for item in ma.CodaTargetList]
Pa.DataPipe.DataContainer = {"ModuleCodaFile":ModuleCodaFile}
for Module in ModuleList:
# Get ModuleAutoGen object to generate C code file and makefile
Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
@ -2074,12 +2152,14 @@ class Build():
# Initialize all modules in tracking to 'FAIL'
GlobalData.gModuleBuildTracking[Ma] = 'FAIL'
mqueue = mp.Queue()
for m in Pa.GetAllModuleInfo:
mqueue.put(m)
data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch))
Pa.DataPipe.dump(data_pipe_file)
autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList, GlobalData.gCacheIR)
autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList,GlobalData.gCacheIR)
# Skip cache hit modules
if GlobalData.gBinCacheSource:
@ -2092,16 +2172,46 @@ class Build():
GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].MakeCacheHit:
self.HashSkipModules.append(Ma)
continue
self.BuildModules.append(Ma)
BuildModules.append(Ma)
else:
self.BuildModules.extend(TotalModules)
BuildModules.extend(TotalModules)
if not autogen_rt:
self.AutoGenMgr.TerminateWorkers()
self.AutoGenMgr.join(0.1)
self.AutoGenMgr.join(1)
raise FatalError(errorcode)
self.AutoGenTime += int(round((time.time() - AutoGenStart)))
AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
with open(AutoGenIdFile,"w") as fw:
fw.write("Arch=%s\n" % "|".join((Wa.ArchList)))
fw.write("BuildDir=%s\n" % Wa.BuildDir)
fw.write("PlatformGuid=%s\n" % str(Wa.AutoGenObjectList[0].Guid))
self.Progress.Stop("done!")
return Wa, BuildModules
def _MultiThreadBuildPlatform(self):
SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False)
for BuildTarget in self.BuildTargetList:
GlobalData.gGlobalDefines['TARGET'] = BuildTarget
index = 0
for ToolChain in self.ToolChainList:
GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain
GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain
GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]
index += 1
ExitFlag = threading.Event()
ExitFlag.clear()
if self.SkipAutoGen:
Wa = self.VerifyAutoGenFiles()
if Wa is None:
Wa, self.BuildModules = self.PerformAutoGen(BuildTarget,ToolChain)
else:
GlobalData.gAutoGenPhase = True
self.BuildModules = self.SetupMakeSetting(Wa)
else:
Wa, self.BuildModules = self.PerformAutoGen(BuildTarget,ToolChain)
Pa = Wa.AutoGenObjectList[0]
GlobalData.gAutoGenPhase = False
if GlobalData.gBinCacheSource:
EdkLogger.quiet("Total cache hit driver num: %s, cache miss driver num: %s" % (len(set(self.HashSkipModules)), len(set(self.BuildModules))))
@ -2136,14 +2246,13 @@ class Build():
BuildTask.StartScheduler(self.ThreadNumber, ExitFlag)
# in case there's an interruption. we need a full version of makefile for platform
Pa.CreateMakeFile(False)
if BuildTask.HasError():
self.invalidateHash()
EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
self.MakeTime += int(round((time.time() - MakeStart)))
MakeContiue = time.time()
#
#
# All modules have been put in build tasks queue. Tell task scheduler
@ -2175,13 +2284,8 @@ class Build():
#
# Get Module List
#
ModuleList = {}
for Pa in Wa.AutoGenObjectList:
for Ma in Pa.ModuleAutoGenList:
if Ma is None:
continue
if not Ma.IsLibrary:
ModuleList[Ma.Guid.upper()] = Ma
ModuleList = {ma.Guid.upper():ma for ma in self.BuildModules}
#
# Rebase module to the preferred memory address before GenFds
#
@ -2206,29 +2310,13 @@ class Build():
# Save MAP buffer into MAP file.
#
self._SaveMapFile(MapBuffer, Wa)
self.CreateGuidedSectionToolsFile(Wa)
self.invalidateHash()
## Generate GuidedSectionTools.txt in the FV directories.
#
def CreateGuidedSectionToolsFile(self):
def CreateGuidedSectionToolsFile(self,Wa):
for BuildTarget in self.BuildTargetList:
for ToolChain in self.ToolChainList:
Wa = WorkspaceAutoGen(
self.WorkspaceDir,
self.PlatformFile,
BuildTarget,
ToolChain,
self.ArchList,
self.BuildDatabase,
self.TargetTxt,
self.ToolDef,
self.Fdf,
self.FdList,
self.FvList,
self.CapList,
self.SkuId,
self.UniFlag
)
FvDir = Wa.FvDir
if not os.path.exists(FvDir):
continue
@ -2285,7 +2373,6 @@ class Build():
self._BuildPlatform()
else:
self._MultiThreadBuildPlatform()
self.CreateGuidedSectionToolsFile()
else:
self.SpawnMode = False
self._BuildModule()