more updates for encoding issues

Author: TimSVector

src/main/resources/scripts/fixup_reports.py

@@ -25,7 +25,7 @@
 from __future__ import division
 from __future__ import print_function
 
-import sys, os
+import sys, os, locale
 # adding path
 workspace = os.getenv("WORKSPACE")
 if workspace is None:
@@ -47,6 +47,7 @@
     pass
 
 import tee_print
+from vcast_utils import getVectorCASTEncoding
 
 def fixup_2020_soup(main_soup):
 
@@ -111,21 +112,28 @@ def fixup_2020_soup(main_soup):
     return main_soup
 
 def fixup_2020_reports(report_name):
+
+    lang, enc = getVectorCASTEncoding()
+
     with open(report_name, "rb") as fd:
+        raw = fd.read()
         try:
-            main_soup = BeautifulSoup(fd,features="lxml", from_encoding="utf-8")
+            main_soup = BeautifulSoup(raw, features="lxml", from_encoding=enc)
         except Exception as e:
-            import traceback
-            print( "EXCEPTION FORMAT PRINT:\n{}".format( e ) )
-            print( "EXCEPTION TRACE  PRINT:\n{}".format( "".join(traceback.format_exception(type(e), e, e.__traceback__))))
-            main_soup = BeautifulSoup(fd)
-        
+            try:
+                # Try UTF-8 first
+                main_soup = BeautifulSoup(raw, "lxml", from_encoding="utf-8")
+            except Exception:
+                # Fall back to system ACP (cp936 in China, cp1252 in US)
+                main_soup = BeautifulSoup(raw.decode(enc, errors="replace"), "lxml")
+    
     main_soup = fixup_2020_soup(main_soup)
 
-    with open(report_name,"wb") as fd:    
-        fd.write(main_soup.prettify(formatter="html",encoding="utf-8"))
-        
+    with open(report_name, "w", encoding=enc, errors="replace") as fd:
+        fd.write(main_soup.prettify(formatter="html"))
+    
 if __name__ == '__main__':
+
     report_name = sys.argv[1]
     fixup_2020_reports(report_name)
 

src/main/resources/scripts/generate_xml.py

@@ -1524,17 +1524,21 @@ def __get_testcase_execution_results(self, tc, classname, tc_name):
                 sections=[ "TESTCASE_SECTIONS"],
                 testcase_sections=["EXECUTION_RESULTS"])
 
-            with open(report_name,"rb") as fd:
+            with open(report_name, "rb") as fd:
                 out = fd.read()
 
-            out = out.decode(self.encFmt)
+            try:
+                # Prefer UTF-8 if possible
+                out = out.decode("utf-8")
+            except UnicodeDecodeError:
+                # Fallback to system/default encoding (e.g. cp936 in CN) with replace
+                out = out.decode(self.encFmt, errors="replace")
+                
             os.remove(report_name)
         except:
             out = "No execution results found"
             parse_traceback.parse(traceback.format_exc(), self.print_exc, self.compiler,  self.testsuite,  self.env,  self.build_dir)
 
-        #out = bytes(out, 'utf-8').decode('utf-8', 'ignore')
-
         return out
 
 ## GenerateXml

src/main/resources/scripts/incremental_build_report_aggregator.py

@@ -26,6 +26,7 @@
 import os
 import sys
 import shutil
+import locale
 
 # adding path
 workspace = os.getenv("WORKSPACE")
@@ -120,9 +121,6 @@ def parse_text_files(mpName):
 
 def parse_html_files(mpName):
 
-    # get the VC langaguge and encoding
-    lang, encFmt = getVectorCASTEncoding()
-
     if os.path.exists(mpName + "_rebuild.html"):
         os.remove(mpName + "_rebuild.html")
 
@@ -139,20 +137,30 @@ def parse_html_files(mpName):
 
     while keepLooping:
         try:
-            with open(report_file_list[0],"r", encoding='utf-8') as fd:
+            with open(report_file_list[0],"rb") as fd:
+
+                raw = fd.read()
+                lang, encFmt = getVectorCASTEncoding()
+
                 try:
-                    main_soup = BeautifulSoup((fd),features="lxml")
-                except:
-                    main_soup = BeautifulSoup(fd)
+                    main_soup = BeautifulSoup(fd,features="lxml", from_encoding=encFmt)
+                except Exception as e:
+                    try:
+                        # Try UTF-8 first
+                        main_soup = BeautifulSoup(raw, "lxml", from_encoding="utf-8")
+                    except Exception:
+                        # Fall back to system ACP (cp936 in China, cp1252 in US)
+                        main_soup = BeautifulSoup(raw, "lxml", from_encoding=encFmt)
 
             preserved_count = 0
             executed_count = 0
             total_count = 0
 
             if main_soup.find(id="report-title"):
                 main_manage_api_report = True
-        # New Manage reports have div with id=report-title
-        # Want second table (skip config data section)
+                
+                # New Manage reports have div with id=report-title
+                # Want second table (skip config data section)
                 main_row_list = main_soup.find_all('table')[1].tr.find_next_siblings()
                 main_count_list = main_row_list[-1].th.find_next_siblings()
             else:
@@ -270,16 +278,6 @@ def parse_html_files(mpName):
 
 if __name__ == "__main__":
 
-    encoding = "utf-8"
-    try:
-        if sys.stdout.encoding.lower() != encoding:
-            sys.stdout.reconfigure(encoding=encoding)
-        if sys.stderr.encoding.lower() != encoding:
-            sys.stderr.reconfigure(encoding=encoding)
-    except AttributeError:
-        print("except AttributeError")
-        pass
-
     parser = argparse.ArgumentParser()
     parser.add_argument('ManageProject')
     parser.add_argument('--rptfmt')

src/main/resources/scripts/managewait.py

@@ -41,8 +41,8 @@
 try:
     from safe_open import open
 except:
-    pass    
- 
+    pass
+
 from vcast_utils import getVectorCASTEncoding
 
 class ManageWait(object):
@@ -53,38 +53,50 @@ def __init__(self, verbose, command_line, wait_time, wait_loops, mpName = "", us
         self.command_line = command_line
         self.mpName = mpName
         self.useCI = useCI
+        self.stop_requested = False
 
         # get the VC langaguge and encoding
         self.lang, self.encFmt = getVectorCASTEncoding()
 
-        import sys
+        import sys, locale
 
         try:
-            sys.stdout.reconfigure(encoding="utf-8", errors="replace")
-            sys.stderr.reconfigure(encoding="utf-8", errors="replace")
+            sys.stdout.reconfigure(encoding=self.encFmt, errors="replace")
+            sys.stderr.reconfigure(encoding=self.encFmt, errors="replace")
         except AttributeError:
             import io
-            sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8", errors="replace")
-            sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding="utf-8", errors="replace")
+            sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding=self.encFmt, errors="replace")
+            sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding=self.encFmt, errors="replace")
+
 
-                    
     def enqueueOutput(self, io_target, queue, logfile):
-        while True:
+
+        while not self.stop_requested:
             line = io_target.readline()
+            if not line:
+                continue
             line = line.rstrip()
+
             if line == '':
                 continue
             output = ( datetime.now().strftime("%H:%M:%S.%f") + "  " + line + "\n" )
+
             if not self.silent:
                 try:
-                    print(line.decode(self.encFmt))
-                except:
+                    # Python 2: line may be str (bytes) - decode it
+                    if isinstance(line, bytes):
+                        print(line.decode(self.encFmt, "replace"))
+                    else:
+                        print(line)  # already unicode/str
+                except Exception:
                     pass
                 try:
                     logfile.write(output)
-                except:
-                    logfile.write(output.decode(self.encFmt))
-                    
+                except Exception:
+                    if isinstance(output, bytes):
+                        logfile.write(output.decode(self.encFmt, "replace"))
+                    else:
+                        logfile.write(output)
             queue.put(line)
 
     def startOutputThread(self, io_target, logfile):
@@ -98,20 +110,20 @@ def exec_manage_command(self, cmd_line, silent = False):
         if self.verbose:
             print (self.command_line)
         return self.exec_manage(silent)
-        
+
     def exec_manage(self, silent=False):
         try:
-            with open("command.log", 'a', encoding=self.encFmt) as logfile:
+            with open("command.log", 'a', encoding=self.encFmt, errors="replace") as logfile:
                 return self.__exec_manage(silent, logfile)
-        except:        
+        except:
             with open("command.log", 'a') as logfile:
                 return self.__exec_manage(silent, logfile)
-                
+
     def __exec_manage(self, silent, logfile):
         self.silent = silent
         callStr = os.environ.get('VECTORCAST_DIR') + os.sep + "manage " + self.command_line
         ret_out = ''
-        
+
         if self.verbose:
             logfile.write( "\nVerbose: %s\n" % callStr)
 
@@ -123,22 +135,22 @@ def __exec_manage(self, silent, logfile):
                 p = subprocess.Popen(callStr,stdout=subprocess.PIPE,stderr=subprocess.STDOUT, shell=True, universal_newlines=True, encoding=self.encFmt)
             except:
                 p = subprocess.Popen(callStr,stdout=subprocess.PIPE,stderr=subprocess.STDOUT, shell=True, universal_newlines=True)
-            
+
             self.startOutputThread(p.stdout, logfile)
-            
+
             license_outage = False
             edited_license_outage_msg = ""
             actual_license_outage_msg = ""
-            
+
             while p.poll() is None or not self.q.empty():
                 try:
                     while not self.q.empty():
-                        out_mgt = self.q.get(False) 
-                        
+                        out_mgt = self.q.get(False)
+
                         if len(out_mgt) > 0:
-                        
+
                             errors = ["Unable to obtain license",  "Licensed number of users already reached", "License server system does not support this feature"]
-         
+
                             if any(error in out_mgt for error in errors):
                                 license_outage = True
                                 # Change FLEXlm Error to FLEXlm Err.. to avoid Groovy script from
@@ -149,10 +161,13 @@ def __exec_manage(self, silent, logfile):
 
                             ret_out +=  out_mgt + "\n"
                     time.sleep(0.5)
-                    
+
                 except Empty:
                     pass
 
+            self.stop_requested = True
+            self.io_t.join()
+            
             # manage finished. Was there a license outage?
             if license_outage == True :
                 if loop_count < self.wait_loops:
@@ -168,17 +183,17 @@ def __exec_manage(self, silent, logfile):
                     break
             else :
                 break #leave outer while loop
-                
+
         try:
             ret_out = unicode(ret_out,self.encFmt)
         except:
             pass
-            
+
         return ret_out # checked in generate-results.py
- 
+
 ## main
 if __name__ == '__main__':
-    
+
     parser = argparse.ArgumentParser()
     parser.add_argument('-v', '--verbose',   help='Enable verbose output', action="store_true")
     parser.add_argument('--command_line',   help='Command line to pass to Manage', required=True)

src/main/resources/scripts/parallel_full_reports.py

@@ -25,6 +25,8 @@ def dump(obj):
             return str(obj)
 
 def generate_report(params):
+    
+    print(f"generate report {params}")
 
     key, env_name, env_is_monitored, env_orig_env_dir, env_relative_wd, env_is_coverapi, env_is_ut_api, vcm_file, workspace, jenkins_workspace, VCD = params
 
@@ -51,12 +53,14 @@ def generate_report(params):
             cmd = VCD + "/clicast -e " + env_name + " COVER REPORT AGGREGATE " + report_name
             process = subprocess.Popen(cmd.split(), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=build_dir)
             stdout, stderr = process.communicate()
+            print(cmd, stdout)
             result = process.returncode
 
         elif env_is_ut_api:
             cmd = VCD + "/clicast -e " + env_name + " REPORT CUSTOM FULL " + report_name
             process = subprocess.Popen(cmd.split(), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=build_dir)
             stdout, stderr = process.communicate()
+            print(cmd, stdout)
             result = process.returncode
 
         else:

src/main/resources/scripts/parse_console_for_cbt.py

@@ -193,9 +193,14 @@ def parse(self, console_log):
 
 if __name__ == '__main__':
 
-    with open(sys.argv[1],"r") as fd:
+    from vcast_utils import getVectorCASTEncoding
+
+    # get the VC langaguge and encoding
+    lang, encFmt = getVectorCASTEncoding()
+
+    with open(sys.argv[1], "r", encoding=encFmt, errors="replace") as fd:
         buildLogData = fd.readlines()
-        
+    
     parser = ParseConsoleForCBT(True)
     parser.parse(buildLogData)
     pprint(parser.parse(buildLogData), width=132)

src/main/resources/scripts/safe_open.py

@@ -25,6 +25,7 @@
 import os
 
 from io import open as _open
+from vcast_utils import getVectorCASTEncoding
 
 @contextlib.contextmanager
 def open(file, mode='r', buffering=-1, encoding=None, errors=None, newline=None, closefd=True, opener=None):
@@ -35,8 +36,9 @@ def open(file, mode='r', buffering=-1, encoding=None, errors=None, newline=None,
         except:
             fd = _open(file, mode, buffering, encoding, errors, newline, closefd)
     else:
-        encoding = "utf-8"
-        fd = _open(file, mode, buffering, encoding, errors, newline)
+        lang, encFmt = getVectorCASTEncoding()
+        
+        fd = _open(file, mode, buffering, encFmt, errors, newline)
 
     try:
         yield fd