BaseTools: Add LogAgent to support multiple process Autogen

BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=1875

AutoGen processes race the logfile. To resolve this issue,
this patch create a LogAgent thread in main process to write
the log content to console or file, Other process will send
the log content to the LogAgent.

Cc: Liming Gao <liming.gao@intel.com>
Signed-off-by: Bob Feng <bob.c.feng@intel.com>
Acked-by: Laszlo Ersek <lersek@redhat.com>
Tested-by: Laszlo Ersek <lersek@redhat.com>
Acked-by: Liming Gao <liming.gao@intel.com>

BaseTools/Source/Python/AutoGen/AutoGenWorker.py

@@ -26,12 +26,76 @@ def clearQ(q):
             q.get_nowait()
     except Empty:
         pass
+import logging
+
+class LogAgent(threading.Thread):
+    def __init__(self,log_q,log_level,log_file=None):
+        super(LogAgent,self).__init__()
+        self.log_q = log_q
+        self.log_level = log_level
+        self.log_file = log_file
+    def InitLogger(self):
+        # For DEBUG level (All DEBUG_0~9 are applicable)
+        self._DebugLogger_agent = logging.getLogger("tool_debug_agent")
+        _DebugFormatter = logging.Formatter("[%(asctime)s.%(msecs)d]: %(message)s", datefmt="%H:%M:%S")
+        self._DebugLogger_agent.setLevel(self.log_level)
+        _DebugChannel = logging.StreamHandler(sys.stdout)
+        _DebugChannel.setFormatter(_DebugFormatter)
+        self._DebugLogger_agent.addHandler(_DebugChannel)
+
+        # For VERBOSE, INFO, WARN level
+        self._InfoLogger_agent = logging.getLogger("tool_info_agent")
+        _InfoFormatter = logging.Formatter("%(message)s")
+        self._InfoLogger_agent.setLevel(self.log_level)
+        _InfoChannel = logging.StreamHandler(sys.stdout)
+        _InfoChannel.setFormatter(_InfoFormatter)
+        self._InfoLogger_agent.addHandler(_InfoChannel)
+
+        # For ERROR level
+        self._ErrorLogger_agent = logging.getLogger("tool_error_agent")
+        _ErrorFormatter = logging.Formatter("%(message)s")
+        self._ErrorLogger_agent.setLevel(self.log_level)
+        _ErrorCh = logging.StreamHandler(sys.stderr)
+        _ErrorCh.setFormatter(_ErrorFormatter)
+        self._ErrorLogger_agent.addHandler(_ErrorCh)
+
+        if self.log_file:
+            if os.path.exists(self.log_file):
+                os.remove(self.log_file)
+            _Ch = logging.FileHandler(self.log_file)
+            _Ch.setFormatter(_DebugFormatter)
+            self._DebugLogger_agent.addHandler(_Ch)
+
+            _Ch= logging.FileHandler(self.log_file)
+            _Ch.setFormatter(_InfoFormatter)
+            self._InfoLogger_agent.addHandler(_Ch)
+
+            _Ch = logging.FileHandler(self.log_file)
+            _Ch.setFormatter(_ErrorFormatter)
+            self._ErrorLogger_agent.addHandler(_Ch)
+
+    def run(self):
+        self.InitLogger()
+        while True:
+            log_message = self.log_q.get()
+            if log_message is None:
+                break
+            if log_message.name == "tool_error":
+                self._ErrorLogger_agent.log(log_message.levelno,log_message.getMessage())
+            elif log_message.name == "tool_info":
+                self._InfoLogger_agent.log(log_message.levelno,log_message.getMessage())
+            elif log_message.name == "tool_debug":
+                self._DebugLogger_agent.log(log_message.levelno,log_message.getMessage())
+            else:
+                self._InfoLogger_agent.log(log_message.levelno,log_message.getMessage())
+
+    def kill(self):
+        self.log_q.put(None)
 class AutoGenManager(threading.Thread):
     def __init__(self,autogen_workers, feedback_q,error_event):
         super(AutoGenManager,self).__init__()
         self.autogen_workers = autogen_workers
         self.feedback_q = feedback_q
-        self.terminate = False
         self.Status = True
         self.error_event = error_event
     def run(self):
@@ -64,7 +128,7 @@ class AutoGenManager(threading.Thread):
     def kill(self):
         self.feedback_q.put(None)
 class AutoGenWorkerInProcess(mp.Process):
-    def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_lock, share_data,error_event):
+    def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_lock, share_data,log_q,error_event):
         mp.Process.__init__(self)
         self.module_queue = module_queue
         self.data_pipe_file_path =data_pipe_file_path
@@ -73,6 +137,7 @@ class AutoGenWorkerInProcess(mp.Process):
         self.PlatformMetaFileSet = {}
         self.file_lock = file_lock
         self.share_data = share_data
+        self.log_q = log_q
         self.error_event = error_event
     def GetPlatformMetaFile(self,filepath,root):
         try:
@@ -88,14 +153,11 @@ class AutoGenWorkerInProcess(mp.Process):
                     self.feedback_q.put(taskname + ":" + "load data pipe %s failed." % self.data_pipe_file_path)
                 self.data_pipe = MemoryDataPipe()
                 self.data_pipe.load(self.data_pipe_file_path)
-            EdkLogger.Initialize()
+            EdkLogger.LogClientInitialize(self.log_q)
             loglevel = self.data_pipe.Get("LogLevel")
             if not loglevel:
                 loglevel = EdkLogger.INFO
             EdkLogger.SetLevel(loglevel)
-            logfile = self.data_pipe.Get("LogFile")
-            if logfile:
-                EdkLogger.SetLogFile(logfile)
             target = self.data_pipe.Get("P_Info").get("Target")
             toolchain = self.data_pipe.Get("P_Info").get("ToolChain")
             archlist = self.data_pipe.Get("P_Info").get("ArchList")

BaseTools/Source/Python/AutoGen/DataPipe.py

@@ -154,7 +154,7 @@ class MemoryDataPipe(DataPipe):
         self.DataContainer = {"GuidDict": PlatformInfo.Platform._GuidDict}
 
         self.DataContainer = {"DatabasePath":GlobalData.gDatabasePath}
+
         self.DataContainer = {"FdfParser": True if GlobalData.gFdfParser else False}
 
         self.DataContainer = {"LogLevel": EdkLogger.GetLevel()}
-        self.DataContainer = {"LogFile": GlobalData.gOptions.LogFile if GlobalData.gOptions.LogFile is not None else ""}