sravan/system76-edk2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

BaseTools: Enhance Basetool for incremental build

Browse Source 
BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=2311

Include dependency file in Makefile to enhance
incremental build

Signed-off-by: Bob Feng <bob.c.feng@intel.com>

Cc: Liming Gao <liming.gao@intel.com>
Cc: Steven Shi <steven.shi@intel.com>
Reviewed-by: Liming Gao <liming.gao@intel.com>
This commit is contained in:
Bob Feng
2019-11-20 10:58:30 +08:00
committed by mergify[bot]
parent cb277815d5
commit 0c3e8e9947
4 changed files with 380 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

284
BaseTools/Source/Python/AutoGen/IncludesAutoGen.py Normal file
View File

@ -0,0 +1,284 @@
## @file
# Build cache intermediate result and state
#
# Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>
# SPDX-License-Identifier: BSD-2-Clause-Patent
#
from Common.caching import cached_property
import Common.EdkLogger as EdkLogger
import Common.LongFilePathOs as os
from Common.BuildToolError import *
from Common.Misc import SaveFileOnChange, PathClass
from Common.Misc import TemplateString
import sys
gIsFileMap = {}
if sys.platform == "win32":
_INCLUDE_DEPS_TEMPLATE = TemplateString('''
${BEGIN}
!IF EXIST(${deps_file})
!INCLUDE ${deps_file}
!ENDIF
${END}
''')
else:
_INCLUDE_DEPS_TEMPLATE = TemplateString('''
${BEGIN}
-include ${deps_file}
${END}
''')
DEP_FILE_TAIL = "# Updated \n"
class IncludesAutoGen():
""" This class is to manage the dependent files witch are used in Makefile to support incremental build.
1. C files:
1. MSVS.
cl.exe has a build option /showIncludes to display include files on stdout. Build tool captures
that messages and generate dependency files, .deps files.
2. CLANG and GCC
-MMD -MF build option are used to generate dependency files by compiler. Build tool updates the
.deps files.
2. ASL files:
1. Trim find out all the included files with asl specific include format and generate .trim.deps file.
2. ASL PP use c preprocessor to find out all included files with #include format and generate a .deps file
3. build tool updates the .deps file
3. ASM files (.asm, .s or .nasm):
1. Trim find out all the included files with asl specific include format and generate .trim.deps file.
2. ASM PP use c preprocessor to find out all included files with #include format and generate a deps file
3. build tool updates the .deps file
"""
def __init__(self, makefile_folder, ModuleAuto):
self.d_folder = makefile_folder
self.makefile_folder = makefile_folder
self.module_autogen = ModuleAuto
self.ToolChainFamily = ModuleAuto.ToolChainFamily
self.workspace = ModuleAuto.WorkspaceDir
def CreateModuleDeps(self):
SaveFileOnChange(os.path.join(self.makefile_folder,"deps.txt"),"\n".join(self.DepsCollection),False)
def CreateDepsInclude(self):
deps_file = {'deps_file':self.deps_files}
try:
deps_include_str = _INCLUDE_DEPS_TEMPLATE.Replace(deps_file)
except Exception as e:
print(e)
SaveFileOnChange(os.path.join(self.makefile_folder,"dependency"),deps_include_str,False)
@cached_property
def deps_files(self):
""" Get all .deps file under module build folder. """
deps_files = []
for root, _, files in os.walk(self.d_folder, topdown=False):
for name in files:
if not name.endswith(".deps"):
continue
abspath = os.path.join(root, name)
deps_files.append(abspath)
return deps_files
@cached_property
def DepsCollection(self):
""" Collect all the dependency files list from all .deps files under a module's build folder """
includes = set()
targetname = [item[0].Name for item in self.TargetFileList.values()]
for abspath in self.deps_files:
try:
with open(abspath,"r") as fd:
lines = fd.readlines()
firstlineitems = lines[0].split(": ")
dependency_file = firstlineitems[1].strip(" \\\n")
dependency_file = dependency_file.strip('''"''')
if dependency_file:
if os.path.normpath(dependency_file +".deps") == abspath:
continue
filename = os.path.basename(dependency_file).strip()
if filename not in self.SourceFileList and filename not in targetname:
includes.add(dependency_file.strip())
for item in lines[1:]:
if item == DEP_FILE_TAIL:
continue
dependency_file = item.strip(" \\\n")
dependency_file = dependency_file.strip('''"''')
if os.path.normpath(dependency_file +".deps") == abspath:
continue
filename = os.path.basename(dependency_file).strip()
if filename in self.SourceFileList:
continue
if filename in targetname:
continue
includes.add(dependency_file.strip())
except Exception as e:
EdkLogger.error("build",FILE_NOT_FOUND, "%s doesn't exist" % abspath, ExtraData=str(e), RaiseError=False)
continue
rt = sorted(list(set([item.strip(' " \\\n') for item in includes])))
return rt
@cached_property
def SourceFileList(self):
""" Get a map of module's source files name to module's source files path """
source = {os.path.basename(item.File):item.Path for item in self.module_autogen.SourceFileList}
middle_file = {}
for afile in source:
if afile.upper().endswith(".VFR"):
middle_file.update({afile.split(".")[0]+".c":os.path.join(self.module_autogen.DebugDir,afile.split(".")[0]+".c")})
if afile.upper().endswith((".S","ASM")):
middle_file.update({afile.split(".")[0]+".i":os.path.join(self.module_autogen.OutputDir,afile.split(".")[0]+".i")})
if afile.upper().endswith(".ASL"):
middle_file.update({afile.split(".")[0]+".i":os.path.join(self.module_autogen.OutputDir,afile.split(".")[0]+".i")})
source.update({"AutoGen.c":os.path.join(self.module_autogen.OutputDir,"AutoGen.c")})
source.update(middle_file)
return source
@cached_property
def HasNamesakeSourceFile(self):
source_base_name = set([os.path.basename(item.File) for item in self.module_autogen.SourceFileList])
rt = len(source_base_name) != len(self.module_autogen.SourceFileList)
return rt
@cached_property
def CcPPCommandPathSet(self):
rt = set()
rt.add(self.module_autogen.BuildOption.get('CC',{}).get('PATH'))
rt.add(self.module_autogen.BuildOption.get('ASLCC',{}).get('PATH'))
rt.add(self.module_autogen.BuildOption.get('ASLPP',{}).get('PATH'))
rt.add(self.module_autogen.BuildOption.get('VFRPP',{}).get('PATH'))
rt.add(self.module_autogen.BuildOption.get('PP',{}).get('PATH'))
rt.add(self.module_autogen.BuildOption.get('APP',{}).get('PATH'))
rt.discard(None)
return rt
@cached_property
def TargetFileList(self):
""" Get a map of module's target name to a tuple of module's targets path and whose input file path """
targets = {}
targets["AutoGen.obj"] = (PathClass(os.path.join(self.module_autogen.OutputDir,"AutoGen.obj")),PathClass(os.path.join(self.module_autogen.DebugDir,"AutoGen.c")))
for item in self.module_autogen.Targets.values():
for block in item:
targets[block.Target.Path] = (block.Target,block.Inputs[0])
return targets
def GetRealTarget(self,source_file_abs):
""" Get the final target file based on source file abspath """
source_target_map = {item[1].Path:item[0].Path for item in self.TargetFileList.values()}
source_name_map = {item[1].File:item[0].Path for item in self.TargetFileList.values()}
target_abs = source_target_map.get(source_file_abs)
if target_abs is None:
if source_file_abs.strip().endswith(".i"):
sourcefilename = os.path.basename(source_file_abs.strip())
for sourcefile in source_name_map:
if sourcefilename.split(".")[0] == sourcefile.split(".")[0]:
target_abs = source_name_map[sourcefile]
break
else:
target_abs = source_file_abs
else:
target_abs = source_file_abs
return target_abs
def CreateDepsFileForMsvc(self, DepList):
""" Generate dependency files, .deps file from /showIncludes output message """
if not DepList:
return
ModuleDepDict = {}
current_source = ""
SourceFileAbsPathMap = self.SourceFileList
for line in DepList:
line = line.strip()
if self.HasNamesakeSourceFile:
for cc_cmd in self.CcPPCommandPathSet:
if cc_cmd in line:
if '''"'''+cc_cmd+'''"''' in line:
cc_options = line[len(cc_cmd)+2:].split()
else:
cc_options = line[len(cc_cmd):].split()
SourceFileAbsPathMap = {os.path.basename(item):item for item in cc_options if not item.startswith("/") and os.path.exists(item)}
if line in SourceFileAbsPathMap:
current_source = line
if current_source not in ModuleDepDict:
ModuleDepDict[SourceFileAbsPathMap[current_source]] = []
elif "Note: including file:" == line.lstrip()[:21]:
if not current_source:
EdkLogger.error("build",BUILD_ERROR, "Parse /showIncludes output failed. line: %s. \n" % line, RaiseError=False)
else:
ModuleDepDict[SourceFileAbsPathMap[current_source]].append(line.lstrip()[22:].strip())
for source_abs in ModuleDepDict:
if ModuleDepDict[source_abs]:
target_abs = self.GetRealTarget(source_abs)
dep_file_name = os.path.basename(source_abs) + ".deps"
SaveFileOnChange(os.path.join(os.path.dirname(target_abs),dep_file_name)," \\\n".join([target_abs+":"] + ['''"''' + item +'''"''' for item in ModuleDepDict[source_abs]]),False)
def UpdateDepsFileforNonMsvc(self):
""" Update .deps files.
1. Update target path to absolute path.
2. Update middle target to final target.
"""
for abspath in self.deps_files:
if abspath.endswith(".trim.deps"):
continue
try:
newcontent = []
with open(abspath,"r") as fd:
lines = fd.readlines()
if lines[-1] == DEP_FILE_TAIL:
continue
firstlineitems = lines[0].strip().split(" ")
if len(firstlineitems) > 2:
sourceitem = firstlineitems[1]
else:
sourceitem = lines[1].strip().split(" ")[0]
source_abs = self.SourceFileList.get(sourceitem,sourceitem)
firstlineitems[0] = self.GetRealTarget(source_abs)
p_target = firstlineitems
if not p_target[0].strip().endswith(":"):
p_target[0] += ": "
if len(p_target) == 2:
p_target[0] += lines[1]
newcontent.append(p_target[0])
newcontent.extend(lines[2:])
else:
line1 = " ".join(p_target).strip()
line1 += "\n"
newcontent.append(line1)
newcontent.extend(lines[1:])
newcontent.append("\n")
newcontent.append(DEP_FILE_TAIL)
with open(abspath,"w") as fw:
fw.write("".join(newcontent))
except Exception as e:
EdkLogger.error("build",FILE_NOT_FOUND, "%s doesn't exist" % abspath, ExtraData=str(e), RaiseError=False)
continue
def UpdateDepsFileforTrim(self):
""" Update .deps file which generated by trim. """
for abspath in self.deps_files:
if not abspath.endswith(".trim.deps"):
continue
try:
newcontent = []
with open(abspath,"r") as fd:
lines = fd.readlines()
if lines[-1] == DEP_FILE_TAIL:
continue
source_abs = lines[0].strip().split(" ")[0]
targetitem = self.GetRealTarget(source_abs.strip(" :"))
targetitem += ": "
targetitem += lines[1]
newcontent.append(targetitem)
newcontent.extend(lines[2:])
newcontent.append("\n")
newcontent.append(DEP_FILE_TAIL)
with open(abspath,"w") as fw:
fw.write("".join(newcontent))
except Exception as e:
EdkLogger.error("build",FILE_NOT_FOUND, "%s doesn't exist" % abspath, ExtraData=str(e), RaiseError=False)
continue