From 94459080c118049aba927ec0444ba5b750b7d2c9 Mon Sep 17 00:00:00 2001
From: "Shi, Steven" <steven.shi@intel.com>
Date: Thu, 15 Aug 2019 22:26:21 +0800
Subject: BaseTools: Improve the file saving and copying reliability

BZ:https://bugzilla.tianocore.org/show_bug.cgi?id=2079

The Basetool CopyFileOnChange() and SaveFileOnChange()
functions might raise the IOError occasionally when build
in Windows with multi-process and build cache enabled.
The CopyFileOnChange() and SaveFileOnChange() might be invoked
in multiple sub-processes simultaneously, and this patch adds
global locks to sync these functions invoking which can
harden their reliability.

Cc: Liming Gao <liming.gao@intel.com>
Cc: Bob Feng <bob.c.feng@intel.com>
Signed-off-by: Steven Shi <steven.shi@intel.com>
Reviewed-by: Bob Feng <bob.c.feng@intel.com>
---
 BaseTools/Source/Python/Common/Misc.py | 44 ++++++++++++++++++++++++++++------
 1 file changed, 37 insertions(+), 7 deletions(-)
 mode change 100644 => 100755 BaseTools/Source/Python/Common/Misc.py

(limited to 'BaseTools/Source/Python/Common/Misc.py')

diff --git a/BaseTools/Source/Python/Common/Misc.py b/BaseTools/Source/Python/Common/Misc.py
old mode 100644
new mode 100755
index 554ec010dd..4799635cc4
--- a/BaseTools/Source/Python/Common/Misc.py
+++ b/BaseTools/Source/Python/Common/Misc.py
@@ -448,7 +448,7 @@ def RemoveDirectory(Directory, Recursively=False):
 #   @retval     True            If the file content is changed and the file is renewed
 #   @retval     False           If the file content is the same
 #
-def SaveFileOnChange(File, Content, IsBinaryFile=True):
+def SaveFileOnChange(File, Content, IsBinaryFile=True, FileLock=None):
 
     if os.path.exists(File):
         if IsBinaryFile:
@@ -479,6 +479,13 @@ def SaveFileOnChange(File, Content, IsBinaryFile=True):
     if IsBinaryFile:
         OpenMode = "wb"
 
+    # use default file_lock if no input new lock
+    if not FileLock:
+        FileLock = GlobalData.file_lock
+    if FileLock:
+        FileLock.acquire()
+
+
     if GlobalData.gIsWindows and not os.path.exists(File):
         # write temp file, then rename the temp file to the real file
         # to make sure the file be immediate saved to disk
@@ -487,14 +494,26 @@ def SaveFileOnChange(File, Content, IsBinaryFile=True):
             tempname = tf.name
         try:
             os.rename(tempname, File)
-        except:
-            EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
+        except IOError as X:
+            if GlobalData.gBinCacheSource:
+                EdkLogger.quiet("[cache error]:fails to save file with error: %s" % (X))
+            else:
+                EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
+        finally:
+            if FileLock:
+                FileLock.release()
     else:
         try:
             with open(File, OpenMode) as Fd:
                 Fd.write(Content)
         except IOError as X:
-            EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
+            if GlobalData.gBinCacheSource:
+                EdkLogger.quiet("[cache error]:fails to save file with error: %s" % (X))
+            else:
+                EdkLogger.error(None, FILE_CREATE_FAILURE, ExtraData='IOError %s' % X)
+        finally:
+            if FileLock:
+                FileLock.release()
 
     return True
 
@@ -510,7 +529,7 @@ def SaveFileOnChange(File, Content, IsBinaryFile=True):
 #   @retval     True      The two files content are different and the file is copied
 #   @retval     False     No copy really happen
 #
-def CopyFileOnChange(SrcFile, Dst):
+def CopyFileOnChange(SrcFile, Dst, FileLock=None):
     if not os.path.exists(SrcFile):
         return False
 
@@ -531,6 +550,12 @@ def CopyFileOnChange(SrcFile, Dst):
         if not os.access(DirName, os.W_OK):
             EdkLogger.error(None, PERMISSION_FAILURE, "Do not have write permission on directory %s" % DirName)
 
+    # use default file_lock if no input new lock
+    if not FileLock:
+        FileLock = GlobalData.file_lock
+    if FileLock:
+        FileLock.acquire()
+
     # os.replace and os.rename are the atomic operations in python 3 and 2.
     # we use these two atomic operations to ensure the file copy is atomic:
     # copy the src to a temp file in the dst same folder firstly, then
@@ -546,9 +571,14 @@ def CopyFileOnChange(SrcFile, Dst):
             if GlobalData.gIsWindows and os.path.exists(DstFile):
                 os.remove(DstFile)
             os.rename(tempname, DstFile)
-
     except IOError as X:
-        EdkLogger.error(None, FILE_COPY_FAILURE, ExtraData='IOError %s' % X)
+        if GlobalData.gBinCacheSource:
+            EdkLogger.quiet("[cache error]:fails to copy file with error: %s" % (X))
+        else:
+            EdkLogger.error(None, FILE_COPY_FAILURE, ExtraData='IOError %s' % X)
+    finally:
+        if FileLock:
+            FileLock.release()
 
     return True
 
-- 
cgit v1.2.3