Query suite updates (#172)

* Suite updates to add additional queries

* update windows_driver_mustfix.qls

* update readme

* qlpack version update

* update query packs

* update github action

* update query suites with missing query

* update version

* sort query suites

* add missing queries

* Fix opaqueid fields. For some reason opaque-id causes issues with sarif reader.

* fix changed file

Author: jacob-ronstadt

.github/workflows/build-codeql.yaml

@@ -14,7 +14,7 @@ on:
   workflow_dispatch:
 
 env:
-  CODEQL_VERSION: 2.20.1
+  CODEQL_VERSION: 2.20.4
 
 jobs:
   build:
@@ -52,11 +52,11 @@ jobs:
 
     - name: Build must-fix driver suite
       shell: cmd
-      run: .\codeql-cli\codeql.cmd query compile --check-only windows_mustfix_partial.qls
+      run: .\codeql-cli\codeql.cmd query compile --check-only mustfix.qls
 
     - name: Build recommended driver suite
       shell: cmd
-      run: .\codeql-cli\codeql.cmd query compile --check-only windows_recommended_partial.qls
+      run: .\codeql-cli\codeql.cmd query compile --check-only recommended.qls
 
     - name: Build CA ported queries
       shell: cmd
@@ -196,15 +196,15 @@ jobs:
       shell: pwsh
       run: 
         $qlpack_diff = git diff HEAD~1:src/qlpack.yml src/qlpack.yml;
-        $rec_diff = git diff HEAD~1:src/windows-driver-suites/windows_recommended_partial.qls src/windows-driver-suites/windows_recommended_partial.qls;
-        $mf_diff = git diff HEAD~1:src/windows-driver-suites/windows_mustfix_partial.qls src/windows-driver-suites/windows_mustfix_partial.qls;
+        $rec_diff = git diff HEAD~1:src/windows-driver-suites/recommended.qls src/windows-driver-suites/recommended.qls;
+        $mf_diff = git diff HEAD~1:src/windows-driver-suites/mustfix.qls src/windows-driver-suites/mustfix.qls;
         if (!$qlpack_diff -and ($rec_diff -or $mf_diff)) { "Query suite file updated without updating qlpack version"; exit 1 }
 
         $last_qlpack_commit = git log -n 1 --pretty=format:%H -- src/qlpack.yml;
         $qlpack_changes =git show $last_qlpack_commit -- .\src\qlpack.yml;
 
-        $last_mf_commit = git log -n 1 --pretty=format:%H -- src/windows-driver-suites/windows_mustfix_partial.qls;
-        $last_rec_commit = git log -n 1 --pretty=format:%H -- src/windows-driver-suites/windows_recommended_partial.qls;
+        $last_mf_commit = git log -n 1 --pretty=format:%H -- src/windows-driver-suites/mustfix.qls;
+        $last_rec_commit = git log -n 1 --pretty=format:%H -- src/windows-driver-suites/recommended.qls;
         $commits_since_qlpack_change = [int](git rev-list --count HEAD...$last_qlpack_commit);
         $commits_since_mf_change = [int](git rev-list --count HEAD...$last_mf_commit);
         $commits_since_rec_change = [int](git rev-list --count HEAD...$last_rec_commit);
@@ -214,7 +214,7 @@ jobs:
 
         try{$old_qlpack_version = [version]($qlpack_changes -match "-version").Substring(10);} catch {"Changed qlpack.yml without updating version"; exit 1 }
         try{$new_qlpack_version = [version]($qlpack_changes -match "\+version").Substring(10);} catch {"Changed qlpack.yml without updating version"; exit 1 }
-        if ($new_qlpack_version -gt $old_qlpack_version) { exit 0 } else { "qlpack.yml version not incremented"; exit 1 }
+        if ($new_qlpack_version -gt $old_qlpack_version) { exit 0 } else { "qlpack.yml version not incremented. Previously updated to version $old_qlpack_version, current version $new_qlpack_version"; exit 1 }
 
   test-create-dvl:
     runs-on: windows-latest

README.md

@@ -6,7 +6,7 @@ This repository contains open-source components for supplemental use in developi
 
 |  CodeQL CLI version      | microsoft/windows-drivers qlpack version | codeql/cpp-queries version  |Associated Repo Branch|
 |--------------------------|-------------------------------------------|------------------------|------------------------|
-| 2.15.4                   | latest                                    | latest |main |
+| latest                   | latest                                    | latest |main |
 
 ### For Windows Hardware Compatibility Program Use
 

src/codeql-pack.lock.yml

@@ -11,6 +11,8 @@ dependencies:
     version: 1.0.16
   codeql/ssa:
     version: 1.0.16
+  codeql/suite-helpers:
+    version: 1.0.16
   codeql/tutorial:
     version: 1.0.16
   codeql/typeflow:
@@ -21,4 +23,6 @@ dependencies:
     version: 2.0.3
   codeql/xml:
     version: 1.0.16
+  microsoft/cpp-queries:
+    version: 0.0.2
 compiled: false

src/drivers/general/queries/PointerVariableSize/PointerVariableSize.ql

@@ -21,7 +21,7 @@
 
 import cpp
 
-from SizeofExprOperator e, VariableAccess va, AddressOfExpr a
+from SizeofExprOperator e, VariableAccess va
 where
   va = e.getExprOperand() and
   va.getTarget().getUnspecifiedType() instanceof PointerType

src/drivers/kmdf/queries/experimental/DeviceInitApi/DeviceInitApi.ql

@@ -7,7 +7,7 @@
  * @kind path-problem
  * @problem.severity error
  * @precision medium
- * @id cpp/windows/wdk/kmdf/DeviceInitApi
+ * @id cpp/windows/wdk/kmdf/device-init-api
  * @tags correctness
  * @query-version v1
  */

src/drivers/test/build_create_analyze_test.py

@@ -189,7 +189,7 @@ def upload_results_to_azure(file_to_upload, file_name, file_directory):
     file_service = FileService(connection_string=args.connection_string)
     file_service.create_file_from_path(share_name=args.share_name, file_name=file_name, directory_name=file_directory, local_file_path=file_to_upload, content_settings=ContentSettings(content_type='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'))
 
-def download_file_from_azure(file_to_download, file_name, file_directory):
+def download_file_from_azure(out_file_path, file_name, file_directory):
     """
     Downloads a file from Azure.
 
@@ -200,7 +200,7 @@ def download_file_from_azure(file_to_download, file_name, file_directory):
         None
     """
     file_service = FileService(connection_string=args.connection_string)
-    file = file_service.get_file_to_path(share_name=args.share_name, file_name=file_name, directory_name=file_directory, file_path=file_to_download)
+    file = file_service.get_file_to_path(share_name=args.share_name, file_name=file_name, directory_name=file_directory, file_path=out_file_path)
     return file.name
 
 def get_git_root():
@@ -250,7 +250,7 @@ def find_ql_test_paths(directory, extension):
         # exclude wfp folder until correct test template is added
         ignore_paths = ["wfp", "QueryTemplate", "TestTemplate", ".vs"]
         root = root.replace("\\", "/")
-        if any(path in ignore_paths for x in root.split("/")): 
+        if any(path in ignore_paths for path in root.split("/")): 
             print_conditionally("Skipping: " + root)
             continue
         if "TestTemplate" in root:
@@ -265,6 +265,7 @@ def find_ql_test_paths(directory, extension):
                     ql_obj = ql_test_attributes(use_ntifs=use_ntifs, use_cpp=use_cpp)
                     ql_obj.set_use_cpp(use_cpp)
                     ql_files_map[os.path.join(root, file)] = ql_obj
+                    print_conditionally("Found: " + os.path.join(root, file))
 
     return ql_files_map
 
@@ -780,14 +781,14 @@ def run_tests_external_drivers(ql_tests_dict):
     result_file = "external_drivers_results.xlsx"
     with pd.ExcelWriter(result_file) as writer:
         health_df.to_excel(writer, sheet_name="Results")
-        codeql_version_df.to_excel(writer, sheet_name="CodeQL Version")
-        codeql_packs_df.to_excel(writer, sheet_name="CodeQL Packs")
-        system_info_df.to_excel(writer, sheet_name="System Info")
+        local_codeql_version_df.to_excel(writer, sheet_name="Local CodeQL Version")
+        local_codeql_packs_df.to_excel(writer, sheet_name="Local CodeQL Packs")
+        local_system_info_df.to_excel(writer, sheet_name="Local System Info")
     with pd.ExcelWriter("detailed" + result_file) as writer:
         detailed_health_df.to_excel(writer, sheet_name="Results")
-        codeql_version_df.to_excel(writer, sheet_name="CodeQL Version")
-        codeql_packs_df.to_excel(writer, sheet_name="CodeQL Packs")
-        system_info_df.to_excel(writer, sheet_name="System Info")
+        local_codeql_version_df.to_excel(writer, sheet_name="Local CodeQL Version")
+        local_codeql_packs_df.to_excel(writer, sheet_name="Local CodeQL Packs")
+        local_system_info_df.to_excel(writer, sheet_name="Local System Info")
     if args.compare_results:
         compare_health_results("detailed"+result_file)
         compare_health_results(result_file)
@@ -838,9 +839,8 @@ def compare_health_results(curr_results_path):
     try:
         prev_results = 'azure-'+curr_results_path
         print_conditionally("Downloading previous results from Azure: " + prev_results)
-        temp_file = download_file_from_azure(file_to_download=prev_results, 
+        _ = download_file_from_azure(out_file_path=prev_results, 
                         file_name=curr_results_path, file_directory="")
-        print_conditionally("Downloaded previous results: " + temp_file)
 
     except Exception as e:
         if "ResourceNotFound" in str(e):
@@ -853,7 +853,7 @@ def compare_health_results(curr_results_path):
     prev_results_df = pd.read_excel(prev_results, index_col=0, sheet_name=0) 
     prev_results_codeql_version_df = pd.read_excel(prev_results, index_col=0, sheet_name=1)
     prev_results_codeql_packs_df = pd.read_excel(prev_results, index_col=0, sheet_name=2)
-    prev_results_system_info_df = pd.read_excel(prev_results, index_col=0, sheet_name=3)
+    prev_results_local_system_info_df = pd.read_excel(prev_results, index_col=0, sheet_name=3)
     curr_results_df = pd.read_excel(curr_results_path, index_col=0, sheet_name=0)
     print_conditionally("Comparing results...")
     print_conditionally("Previous results: ", prev_results)
@@ -879,16 +879,29 @@ def compare_health_results(curr_results_path):
 
     with pd.ExcelWriter("diff" + curr_results_path) as writer:
         diff_results.to_excel(writer, sheet_name="Diff")
-        codeql_version_df.to_excel(writer, sheet_name="Current CodeQL Version")
-        codeql_packs_df.to_excel(writer, sheet_name="Current CodeQL Packs")
-        system_info_df.to_excel(writer, sheet_name="Current System Info")
-        prev_results_codeql_version_df.to_excel(writer, sheet_name="Previous CodeQL Version")
-        prev_results_codeql_packs_df.to_excel(writer, sheet_name="Previous CodeQL Packs")
-        prev_results_system_info_df.to_excel(writer, sheet_name="Previous System Info")
+        local_codeql_version_df.to_excel(writer, sheet_name="Local CodeQL Version")
+        local_codeql_packs_df.to_excel(writer, sheet_name="Local CodeQL Packs")
+        local_system_info_df.to_excel(writer, sheet_name="Local System Info")
+        prev_results_codeql_version_df.to_excel(writer, sheet_name="Last Stored CodeQL Version")
+        prev_results_codeql_packs_df.to_excel(writer, sheet_name="Last Stored CodeQL Packs")
+        prev_results_local_system_info_df.to_excel(writer, sheet_name="Last Stored System Info")
         print_conditionally("Saved diff results")
 
     if not args.local_result_storage:
         # upload new results to Azure
+        if args.overwrite_azure_results:
+            print("!! Overwriting Azure results !!")
+            print("Type 'yes' to confirm")
+            confirm = input()
+            if confirm != "yes":
+                print("Exiting")
+                exit(1)
+            else:
+                double_confirm = input("Are you sure?")
+                if double_confirm != "yes":
+                    print("Exiting")
+                    exit(1)
+                    
         if args.overwrite_azure_results:
             print_conditionally("Uploading results")
             upload_results_to_azure(file_to_upload=curr_results_path, 
@@ -934,14 +947,14 @@ def run_tests(ql_tests_dict):
     result_file = "functiontestresults.xlsx"
     with pd.ExcelWriter(result_file) as writer:
         health_df.to_excel(writer, sheet_name="Results")
-        codeql_version_df.to_excel(writer, sheet_name="CodeQL Version")
-        codeql_packs_df.to_excel(writer, sheet_name="CodeQL Packs")
-        system_info_df.to_excel(writer, sheet_name="System Info")
+        local_codeql_version_df.to_excel(writer, sheet_name="Local CodeQL Version")
+        local_codeql_packs_df.to_excel(writer, sheet_name="Local CodeQL Packs")
+        local_system_info_df.to_excel(writer, sheet_name="Local System Info")
     with pd.ExcelWriter("detailed"+result_file) as writer:
         detailed_health_df.to_excel(writer, sheet_name="Results")
-        codeql_version_df.to_excel(writer, sheet_name="CodeQL Version")
-        codeql_packs_df.to_excel(writer, sheet_name="CodeQL Packs")
-        system_info_df.to_excel(writer, sheet_name="System Info")
+        local_codeql_version_df.to_excel(writer, sheet_name="Local CodeQL Version")
+        local_codeql_packs_df.to_excel(writer, sheet_name="Local CodeQL Packs")
+        local_system_info_df.to_excel(writer, sheet_name="Local System Info")
     if args.compare_results:
         compare_health_results("detailed"+result_file)
         compare_health_results(result_file)
@@ -1008,26 +1021,18 @@ def find_sln_file(path):
     parser.add_argument('--overwrite_azure_results', help='Overwrite Azure results',action='store_true',required=False,)
     parser.add_argument('--build_database_only', help='Build database only',action='store_true',required=False,)
     args = parser.parse_args()
-    
-    if args.overwrite_azure_results:
-        print("Overwriting Azure results")
-        print("Type 'yes' to confirm")
-        confirm = input()
-        if confirm != "yes":
-            print("Exiting")
-            exit(1)
-            
+           
     if args.codeql_path:
         codeql_path = args.codeql_path
     else:
         codeql_path = "codeql"
 
     codeql_version = subprocess.run([codeql_path, "version"], capture_output=True) # test codeql is working
-    codeql_version_df = pd.DataFrame([x for x in codeql_version.stdout.decode().split('\n')])
+    local_codeql_version_df = pd.DataFrame([x for x in codeql_version.stdout.decode().split('\n')])
     codeql_packs = subprocess.run([codeql_path, "resolve", "qlpacks"], capture_output=True) 
-    codeql_packs_df = pd.DataFrame([x for x in codeql_packs.stdout.decode().split('\n')])
+    local_codeql_packs_df = pd.DataFrame([x for x in codeql_packs.stdout.decode().split('\n')])
     system_info = subprocess.run(["systeminfo"], capture_output=True) 
-    system_info_df = pd.DataFrame([x for x in system_info.stdout.decode().split('\n')])
+    local_system_info_df = pd.DataFrame([x for x in system_info.stdout.decode().split('\n')])
 
 
     if args.compare_results_no_build:

src/microsoft/Likely Bugs/Boundary Violations/PaddingByteInformationDisclosure.ql

@@ -2,16 +2,16 @@
 // Licensed under the MIT license.
 
 /**
+ * @id cpp/paddingbyteinformationdisclosure
  * @name Possible information leakage from uninitialized padding bytes.
  * @description A newly allocated struct or class that is initialized member-by-member may
  *              leak information if it includes padding bytes.
  * @kind problem
  * @problem.severity warning
  * @tags security
  *       external/cwe/cwe-200
- * @opaque-id SM02320
+ * @opaqueid SM02320
  * @microsoft.severity Important
- * @id cpp/paddingbyteinformationdisclosure
  */
 
 import cpp

src/microsoft/Likely Bugs/Conversion/BadOverflowGuard.ql

@@ -2,6 +2,7 @@
 // Licensed under the MIT license.
 
 /**
+ * @id cpp/badoverflowguard
  * @name Bad overflow check
  * @description Checking for overflow of an addition by comparing against one
  *              of the arguments of the addition fails if the size of all the
@@ -12,9 +13,9 @@
  * @tags security
  *       external/cwe/cwe-190
  *       external/cwe/cwe-191
- * @opaque-id SM02324
  * @microsoft.severity Important
- * @id cpp/badoverflowguard
+ * @opaqueid SM02324
+ * 
  */
 
 import cpp

src/microsoft/Likely Bugs/Conversion/InfiniteLoop.ql

@@ -2,6 +2,7 @@
 // Licensed under the MIT license.
 
 /**
+ * @id cpp/infiniteloop
  * @name Comparison of narrow type with wide type in loop condition
  * @description Comparisons between types of different widths in a loop
  *              condition can cause the loop to fail to terminate.
@@ -11,9 +12,8 @@
  *       security
  *       external/cwe/cwe-190
  *       external/cwe/cwe-197
- * @opaque-id SM02323
+ * @opaqueid SM02323
  * @microsoft.severity Important
- * @id cpp/infiniteloop
  */
 
 import cpp

src/microsoft/Likely Bugs/UninitializedPtrField.ql

@@ -2,16 +2,16 @@
 // Licensed under the MIT license.
 
 /**
+ * @id cpp/uninitializedptrfield
  * @name Dereference of potentially uninitialized pointer field
  * @description A pointer field which was not initialized during or since class 
  *              construction will cause a null pointer dereference.
  * @kind problem
  * @problem.severity warning
  * @tags security
  *       external/cwe/cwe-476
- * @opaque-id SM02310
+ * @opaqueid SM02310
  * @microsoft.severity Important
- * @id cpp/uninitializedptrfield
  */
 
 import cpp