updates for proper encoding

Author: TimSVector

src/main/resources/scripts/archive_extract_reports.py

@@ -26,7 +26,7 @@ def extract(verbose = False):
                             os.utime(full_path, (f.mtime, f.mtime))
                         except Exception as e:
                             if verbose:
-                                print("Could not set time for {}: {}".format(f.name, e)
+                                print("Could not set time for {}: {}".format(f.name, e))
 
 def archive(verbose = False):
     if os.path.exists(archive_name):

src/main/resources/scripts/cobertura.py

@@ -44,7 +44,9 @@
 
 fileList = []
 
-from vcast_utils import dump, checkVectorCASTVersion
+from vcast_utils import dump, checkVectorCASTVersion, getVectorCASTEncoding
+
+encFmt = getVectorCASTEncoding()
 
 def write_xml(x, name, verbose = False):
 
@@ -56,7 +58,8 @@ def write_xml(x, name, verbose = False):
 
     xml_str += etree.tostring(x,pretty_print=True).decode()
 
-    with open(name + ".xml", "w") as fd: fd.write(xml_str)
+    with open(name + ".xml", "wb") as fd: 
+        fd.write(xml_str.encode(encFmt,"replace"))
 
 def getCoveredFunctionCount(source):
     if len(source.functions) == 0:
@@ -730,8 +733,8 @@ def generateCoverageResults(inFile, azure = False, xml_data_dir = "xml_data", ve
     coverages.attrib['timestamp'] = str(datetime.now())
 
     tool_version = os.path.join(os.environ['VECTORCAST_DIR'], "DATA", "tool_version.txt")
-    with open(tool_version,"r") as fd:
-        ver = fd.read()
+    with open(tool_version,"rb") as fd:
+        ver = fd.read().decode(encFmt,"replace")
 
     coverages.attrib['version'] = "VectorCAST " + ver.rstrip()
 

src/main/resources/scripts/create_index_html.py

@@ -3,6 +3,9 @@
 import argparse
 import glob
 
+from vcast_utils import dump, checkVectorCASTVersion, getVectorCASTEncoding
+
+encFmt = getVectorCASTEncoding()
 
 class cd:
     """Context manager for changing the current working directory"""
@@ -17,8 +20,9 @@ def __exit__(self, etype, value, traceback):
         os.chdir(self.savedPath)
 
 def searchKeyword(search_string, filename):
-    with open(filename, "r") as f:
-        for line_number, line in enumerate(f, start=1):
+    with open(filename, "rb") as fd:
+        for line_number, line in enumerate(fd, start=1):
+            line = line.decode(encFmt, "replace")
             if search_string in line:
                 start_idx = line.find(search_string)
                 if start_idx != -1: 

src/main/resources/scripts/fixup_reports.py

@@ -113,24 +113,28 @@ def fixup_2020_soup(main_soup):
 
 def fixup_2020_reports(report_name):
 
-    lang, enc = getVectorCASTEncoding()
+    encFmt = getVectorCASTEncoding()
 
     with open(report_name, "rb") as fd:
-        raw = fd.read()
+        raw = fd.read().decode(encFmt, "replace")
+
+    try:
+        # First attempt: use whatever encoding was detected
+        main_soup = BeautifulSoup(raw, features="lxml")
+
+    except Exception as e:
         try:
-            main_soup = BeautifulSoup(raw, features="lxml", from_encoding=enc)
-        except Exception as e:
-            try:
-                # Try UTF-8 first
-                main_soup = BeautifulSoup(raw, "lxml", from_encoding="utf-8")
-            except Exception:
-                # Fall back to system ACP (cp936 in China, cp1252 in US)
-                main_soup = BeautifulSoup(raw.decode(enc, errors="replace"), "lxml")
+            # Try UTF-8 first as a fallback (should rarely fail if raw is text)
+            main_soup = BeautifulSoup(raw.encode("utf-8", "replace"), "lxml")
+        except Exception:
+            # Last resort: try system default encoding (ACP on Windows, etc.)
+            main_soup = BeautifulSoup(raw.encode(encFmt, "replace"), "lxml")
 
     main_soup = fixup_2020_soup(main_soup)
 
-    with open(report_name, "w", encoding=enc, errors="replace") as fd:
-        fd.write(main_soup.prettify(formatter="html"))
+    with open(report_name, "wb") as fd:
+        fd.write(main_soup.prettify(formatter="html").encode(encFmt, "replace"))
+
 
 if __name__ == '__main__':
 

src/main/resources/scripts/generate-results.py

@@ -38,6 +38,7 @@
 import tee_print
 
 from safe_open import open
+from vcast_utils import getVectorCASTEncoding
 
 # adding path
 workspace = os.getenv("WORKSPACE")
@@ -81,7 +82,9 @@
     from vector.apps.DataAPI.vcproject_api import VCProjectApi
 except:
     pass
-    
+
+encFmt = getVectorCASTEncoding()
+
 #global variables
 global verbose
 global print_exc
@@ -178,11 +181,14 @@ def readManageVersion(ManageFile):
     version = 14
     if os.path.isfile(ManageFile + ".vcm"):
         ManageFile = ManageFile + '.vcm'
-    with open(ManageFile, 'r') as projFile:
-        for line in projFile:
+        
+    with open(ManageFile, 'rb') as projFile:
+        for raw_line in projFile:                      # iterates lazily, line by line
+            line = raw_line.decode(encFmt, "replace")  # decode each line
             if 'version' in line and 'project' in line:
                 version = int(re.findall(r'\d+', line)[0])
                 break
+
     if verbose:
         print("Version of VectorCAST project file = %d" % version)
         print("(Levels change in version 17 (*maybe) and above)")
@@ -308,8 +314,8 @@ def fixup_css(report_name):
     if not need_fixup:
         return
 
-    with open(report_name,"r") as fd:
-        data = fd.read() 
+    with open(report_name,"rb") as fd:
+        data = fd.read().decode(encFmt,"replace")
 
     #fix up inline CSS because of Content Security Policy violation
     newData = data[: data.index("<style>")-1] +  """
@@ -323,8 +329,8 @@ def fixup_css(report_name):
     regex_str = r"<img alt=\"Vector\".*"
     newData =  re.sub(regex_str,"<img alt=\"Vector\" src=\"vectorcast.png\"/>",newData)
 
-    with open(report_name, "w") as fd:
-        fd.write(newData)
+    with open(report_name, "wb") as fd:
+        fd.write(newData.encode(encFmt,"replace"))
 
     workspace = os.getenv("WORKSPACE")
     if workspace is None:
@@ -577,20 +583,13 @@ def buildReports(FullManageProjectName = None,
                 use_archive_extract, report_only_failures, no_full_report,
                 useStartLine, teePrint, use_cte)
 
-        with open("unit_test_fail_count.txt", "w") as fd:
-            failed_str = str(failed_count)
-            try:
-                fd.write(unicode(failed_str))
-            except:
-                fd.write(failed_str)    
-           
-        with open("unit_test_passfail_count.txt", "w") as fd:
-            passfail_str = str(passed_count) + " " + str(failed_count)
-            try:
-                fd.write(unicode(passfail_str))
-            except:
-                fd.write(passfail_str)    
-           
+            with open("unit_test_fail_count.txt", "wb") as fd:
+                fd.write(str(failed_count).encode(encFmt, "replace"))
+
+            with open("unit_test_passfail_count.txt", "wb") as fd:
+                text = "{} {}".format(passed_count, failed_count)
+                fd.write(text.encode(encFmt, "replace"))
+                   
         if timing:
             print("XML and Individual reports: " + str(time.time()))
 
@@ -671,8 +670,8 @@ def buildReports(FullManageProjectName = None,
             print(out)
 
         # save the output of the manage command for debug purposes
-        with open("build.log","w") as fd:
-            fd.write(out)
+        with open("build.log","wb") as fd:
+            fd.write(out.encode(encFmt, "replace"))
 
         copyList = []
         jobName = ""
@@ -754,9 +753,9 @@ def buildReports(FullManageProjectName = None,
         passed_count = 0
         try:
             for file in glob.glob("xml_data/test_results_*.xml"):
-                with open(file,"r") as fd:
-                    lines = fd.readlines()
-            
+                with open(file,"rb") as fd:
+                    lines = [line.decode(encFmt, "replace") for line in fd.readlines()]
+
                 for line in lines:
                     if "failures" in line:
                         failed_count += int(line.split("\"")[5])
@@ -766,20 +765,12 @@ def buildReports(FullManageProjectName = None,
             teePrint.teePrint ("   *INFO: Problem parsing test results file for unit testcase failure count: " + file)
             if print_exc:  traceback.print_exc()
 
-        with open("unit_test_fail_count.txt", "w") as fd:
-            failed_str = str(failed_count)
-            try:
-                fd.write(unicode(failed_str))
-            except:
-                fd.write(failed_str)    
+            with open("unit_test_fail_count.txt", "wb") as fd:
+                fd.write(str(failed_count).encode(encFmt, "replace"))
 
-        with open("unit_test_passfail_count.txt", "w") as fd:
-            passfail_str = str(passed_count) + " " + str(failed_count)
-            try:
-                fd.write(unicode(passfail_str))
-            except:
-                fd.write(passfail_str)    
-           
+            with open("unit_test_passfail_count.txt", "wb") as fd:
+                text = "{} {}".format(passed_count, failed_count)
+                fd.write(text.encode(encFmt, "replace"))
 
         for file in copyList:
 
@@ -837,8 +828,8 @@ def buildReports(FullManageProjectName = None,
 
     try:
         tool_version = os.path.join(os.environ['VECTORCAST_DIR'], "DATA", "tool_version.txt")
-        with open(tool_version,"r") as fd:
-            ver = fd.read()
+        with open(tool_version,"rb") as fd:
+            ver = fd.read().decode(encFmt, "replace")
 
         if ver.startswith("19 "):
             need_fixup = True
@@ -871,8 +862,9 @@ def new_init(self):
 
 
     if args.buildlog and os.path.exists(args.buildlog):
-        with open(args.buildlog,"r") as fd:
-            buildLogData = fd.readlines()
+        with open(args.buildlog,"rb") as fd:
+            buildLogData = [line.decode(encFmt, "replace") for line in fd.readlines()]
+            
         cbt = ParseConsoleForCBT(verbose)
         cbtDict = cbt.parse(buildLogData)
 

src/main/resources/scripts/generate_lcov.py

@@ -40,11 +40,13 @@
 import subprocess
 import argparse
 
-from vcast_utils import dump, checkVectorCASTVersion
+from vcast_utils import dump, checkVectorCASTVersion, getVectorCASTEncoding
 try:
     from safe_open import open
 except:
     pass
+    
+encFmt = getVectorCASTEncoding()
 
 fileList = []
 
@@ -114,9 +116,9 @@ def has_branches_covered(line):
 
 def get_function_name_line_number(file_path, function, initial_guess):
 
-    with open(file_path,"r") as fd:
-        lines = fd.readlines()
-
+    with open(file_path,"rb") as fd:
+        lines = [line.decode(encFmt, "replace") for line in fd.readlines()]
+        
     line_number_closest_so_far = initial_guess;
     delta = 9999999999;
 
@@ -323,7 +325,8 @@ def generateCoverageResults(inFile, xml_data_dir = "xml_data", verbose = False,
         os.makedirs(lcov_data_dir)
 
     pathToInfo = os.path.join(lcov_data_dir, name + ".info")
-    with open(pathToInfo, "w") as fd: fd.write(output)
+    with open(pathToInfo, "wb") as fd: 
+        fd.write(output.encode(encFmt, "replace"))
 
     cmdStr = "genhtml " + pathToInfo + " --output-directory out"
     cmdArr = cmdStr.split()

src/main/resources/scripts/generate_pclp_reports.py

@@ -19,7 +19,9 @@
 
 from pprint import pprint
 
-from vcast_utils import checkVectorCASTVersion
+from vcast_utils import checkVectorCASTVersion, getVectorCASTEncoding
+    
+encFmt = getVectorCASTEncoding()
 
 try:
     from safe_open import open
@@ -45,13 +47,17 @@ def parse_msgs(filename):
     directoryName = os.path.dirname(filename)
 
     try:
-        basepath = os.environ['WORKSPACE'].replace("\\","/") + "/"
+        basepath = os.environ['CI_PROJECT_DIR'].replace("\\","/") + "/"
     except:
-        basepath = os.getcwd().replace("\\","/") + "/"
-    os.environ['VCAST_RPTS_CUSTOM_CSS']=r'vc_scripts\css\tooltip.css'
+        try:
+            basepath = os.environ['WORKSPACE'].replace("\\","/") + "/"
+        except:
+            basepath = os.getcwd().replace("\\","/") + "/"
+    
+    os.environ['VCAST_RPTS_CUSTOM_CSS']= basepath + "/vc_scripts/css/tooltip.css"
 
-    with open(filename, "r") as fd:
-        pcplXmlData = fd.read()
+    with open(filename, "rb") as fd:
+        pcplXmlData = fd.read().decode(encFmt, "replace")
 
     index = pcplXmlData.find('<')
 
@@ -274,9 +280,9 @@ def generate_source():
         if os.path.isfile(fname + ".vcast.bak"):
             fname = fname + ".vcast.bak"
 
-        with open(fname , 'r') as fh:
+        with open(fname, 'rb') as fh:
             # read and replace the line ending for consistency
-            contents = fh.read()
+            contents = fh.read().encode(encFmnt, "replace")
             contents = contents.replace("\r\n", "\n").replace("\r","\n")
             for lineno, line in enumerate(contents.splitlines(), start=1):
                 lineno_str = str(lineno)
@@ -483,11 +489,8 @@ def emit_gitlab(msgs):
 # Driver
 
 def write_output(output, filename):
-    with open(filename, 'w') as file:
-        try:
-            file.write(output)
-        except:
-            file.write(output.decode('utf-8'))
+    with open(filename, 'wb') as file:
+        file.write(output.encode(encFmt, "replace"))
 
 def generate_reports(input_xml, output_text = None, output_html = None, output_json = None, output_gitlab = None, full_mp_name = None):
 

src/main/resources/scripts/generate_sonarqube_pclp_reports.py

@@ -24,6 +24,10 @@
 except:
     pass
 
+from vcast_utils import getVectorCASTEncoding
+    
+encFmt = getVectorCASTEncoding()
+
 # PC-lint Plus message representation and parsing
 
 class Message:
@@ -299,8 +303,8 @@ def emit_gitlab(msgs):
 # Driver
 
 def write_output(output, filename):
-    with open(filename, 'w') as file:
-        file.write(output)
+    with open(filename, 'wb') as file:
+        file.write(output.encode(encFmt, "replace"))
 
 def generate_reports(input_xml, output_text = None, output_html = None, output_json = None, output_gitlab = None):
     msgs = parse_msgs(input_xml)