BaseTools: Improve the method of checking queue empty

Ref: https://bugzilla.tianocore.org/show_bug.cgi?id=2807

The Queue.empty() method is not reliable in the multiple
process runtime environment. This patch uses a new method
to check if all modules are processed and workers need
to be stopped. That is to add a None item at the bottom
of the queue. Worker check if it gets that None item to
know if all the module is processed.

Signed-off-by: Bob Feng <bob.c.feng@intel.com>
Cc: Liming Gao <liming.gao@intel.com>
Cc: Yuwei Chen <yuwei.chen@intel.com>
Cc: Lucy Yan <lucyyan@google.com>

Reviewed-by: Liming Gao <liming.gao@intel.com>

2 files changed, 22 insertions, 7 deletions

diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
index 563d91b421..017f676399 100755
--- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
+++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
@@ -24,6 +24,7 @@ import traceback
 import sys

 from AutoGen.DataPipe import MemoryDataPipe

 import logging

+import time

 

 def clearQ(q):

     try:

@@ -111,7 +112,11 @@ class AutoGenManager(threading.Thread):
                     break

                 if badnews == "Done":

                     fin_num += 1

+                elif badnews == "QueueEmpty":

+                    EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), badnews))

+                    self.TerminateWorkers()

                 else:

+                    EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), badnews))

                     self.Status = False

                     self.TerminateWorkers()

                 if fin_num == len(self.autogen_workers):

@@ -227,12 +232,21 @@ class AutoGenWorkerInProcess(mp.Process):
             PlatformMetaFile = self.GetPlatformMetaFile(self.data_pipe.Get("P_Info").get("ActivePlatform"),

                                              self.data_pipe.Get("P_Info").get("WorkspaceDir"))

             while True:

-                if self.module_queue.empty():

-                    break

                 if self.error_event.is_set():

                     break

                 module_count += 1

-                module_file,module_root,module_path,module_basename,module_originalpath,module_arch,IsLib = self.module_queue.get_nowait()

+                try:

+                    module_file,module_root,module_path,module_basename,module_originalpath,module_arch,IsLib = self.module_queue.get_nowait()

+                except Empty:

+                    EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), "Fake Empty."))

+                    time.sleep(0.01)

+                    continue

+                if module_file is None:

+                    EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), "Worker get the last item in the queue."))

+                    self.feedback_q.put("QueueEmpty")

+                    time.sleep(0.01)

+                    continue

+

                 modulefullpath = os.path.join(module_root,module_file)

                 taskname = " : ".join((modulefullpath,module_arch))

                 module_metafile = PathClass(module_file,module_root)

@@ -280,11 +294,11 @@ class AutoGenWorkerInProcess(mp.Process):
                     else:

                         self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "MakeCache", False))

 

-        except Empty:

-            pass

-        except:

+        except Exception as e:

+            EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), str(e)))

             self.feedback_q.put(taskname)

         finally:

+            EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), "Done"))

             self.feedback_q.put("Done")

             self.cache_q.put("CacheDone")

 

diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py
index 1ab1e60a64..59ceacfed0 100755
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -1217,7 +1217,7 @@ class Build():
             mqueue = mp.Queue()

             for m in AutoGenObject.GetAllModuleInfo:

                 mqueue.put(m)

-

+            mqueue.put((None,None,None,None,None,None,None))

             AutoGenObject.DataPipe.DataContainer = {"CommandTarget": self.Target}

             AutoGenObject.DataPipe.DataContainer = {"Workspace_timestamp": AutoGenObject.Workspace._SrcTimeStamp}

             AutoGenObject.CreateLibModuelDirs()

@@ -2174,6 +2174,7 @@ class Build():
             data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch))

             Pa.DataPipe.dump(data_pipe_file)

 

+            mqueue.put((None,None,None,None,None,None,None))

             autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList, cqueue)

 

             if not autogen_rt: