sravan/system76-edk2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

BaseTools: Improve the file saving and copying reliability

Browse Source 
BZ:https://bugzilla.tianocore.org/show_bug.cgi?id=2079

The Basetool CopyFileOnChange() and SaveFileOnChange()
functions might raise the IOError occasionally when build
in Windows with multi-process and build cache enabled.
The CopyFileOnChange() and SaveFileOnChange() might be invoked
in multiple sub-processes simultaneously, and this patch adds
global locks to sync these functions invoking which can
harden their reliability.

Cc: Liming Gao <liming.gao@intel.com>
Cc: Bob Feng <bob.c.feng@intel.com>
Signed-off-by: Steven Shi <steven.shi@intel.com>
Reviewed-by: Bob Feng <bob.c.feng@intel.com>
This commit is contained in:
Shi, Steven
2019-08-15 22:26:21 +08:00
committed by Feng, Bob C
parent d01a998612
commit 94459080c1
8 changed files with 119 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
BaseTools/Source/Python/Common/Misc.py Normal file → Executable file
View File

@ -448,7 +448,7 @@ def RemoveDirectory(Directory, Recursively=False):
# @retval True If the file content is changed and the file is renewed
# @retval False If the file content is the same
#
def SaveFileOnChange(File, Content, IsBinaryFile=True):
def SaveFileOnChange(File, Content, IsBinaryFile=True, FileLock=None):
if os.path.exists(File):
if IsBinaryFile:
@ -479,6 +479,13 @@ def SaveFileOnChange(File, Content, IsBinaryFile=True):
if IsBinaryFile:
OpenMode = "wb"
# use default file_lock if no input new lock
if not FileLock:
FileLock = GlobalData.file_lock
if FileLock:
FileLock.acquire()
if GlobalData.gIsWindows and not os.path.exists(File):
# write temp file, then rename the temp file to the real file
# to make sure the file be immediate saved to disk
@ -487,14 +494,26 @@ def SaveFileOnChange(File, Content, IsBinaryFile=True):
tempname = tf.name
try:
os.rename(tempname, File)
except:
EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
except IOError as X:
if GlobalData.gBinCacheSource:
EdkLogger.quiet("[cache error]:fails to save file with error: %s" % (X))
else:
EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
finally:
if FileLock:
FileLock.release()
else:
try:
with open(File, OpenMode) as Fd:
Fd.write(Content)
except IOError as X:
EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
if GlobalData.gBinCacheSource:
EdkLogger.quiet("[cache error]:fails to save file with error: %s" % (X))
else:
EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
finally:
if FileLock:
FileLock.release()
return True
@ -510,7 +529,7 @@ def SaveFileOnChange(File, Content, IsBinaryFile=True):
# @retval True The two files content are different and the file is copied
# @retval False No copy really happen
#
def CopyFileOnChange(SrcFile, Dst):
def CopyFileOnChange(SrcFile, Dst, FileLock=None):
if not os.path.exists(SrcFile):
return False
@ -531,6 +550,12 @@ def CopyFileOnChange(SrcFile, Dst):
if not os.access(DirName, os.W_OK):
EdkLogger.error(None, PERMISSION_FAILURE, "Do not have write permission on directory %s" % DirName)
# use default file_lock if no input new lock
if not FileLock:
FileLock = GlobalData.file_lock
if FileLock:
FileLock.acquire()
# os.replace and os.rename are the atomic operations in python 3 and 2.
# we use these two atomic operations to ensure the file copy is atomic:
# copy the src to a temp file in the dst same folder firstly, then
@ -546,9 +571,14 @@ def CopyFileOnChange(SrcFile, Dst):
if GlobalData.gIsWindows and os.path.exists(DstFile):
os.remove(DstFile)
os.rename(tempname, DstFile)
except IOError as X:
EdkLogger.error(None, FILE_COPY_FAILURE, ExtraData='IOError %s' % X)
if GlobalData.gBinCacheSource:
EdkLogger.quiet("[cache error]:fails to copy file with error: %s" % (X))
else:
EdkLogger.error(None, FILE_COPY_FAILURE, ExtraData='IOError %s' % X)
finally:
if FileLock:
FileLock.release()
return True