feat: Normalize git repository URLs for API queries on the new API (#3986)

For #3830
Change the new `AffectedVersions` entities to store a normalized Git
repository URL so that queries with the new API logic so that queries
for tags aren't dependent on the exact repository URL:
- remove the protocol/scheme
- There are currently only 16 unique repositories in OSV (test instance)
that don't use the `https://` scheme, and only 3[^1] of these repos have
vulns with both `http://` & `https://`
- remove the `.git` extension
- Mostly, OSS-Fuzz and CURL uses GitHub repos with the `.git` extension,
while our CVE's do not.

I will need to do a re-put of all the GIT records in the test instance
to repopulate the names in the `AffectedVersions` entities.

This doesn't yet fix the issue on production - it's too complicated to
try fix with the current/old querying logic, it'll just be fixed when
the migration is complete.

[^1]: git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
([http](https://osv.dev/vulnerability/CVE-2019-19352),
[https](https://osv.dev/vulnerability/CVE-2023-32255))
git.musl-libc.org/git/musl
([http](https://osv.dev/vulnerability/CVE-2017-15650),
[https](https://osv.dev/vulnerability/CVE-2025-26519))
git.savannah.gnu.org/git/wget.git
([http](https://osv.dev/vulnerability/CVE-2016-7098),
[https](https://osv.dev/vulnerability/CVE-2018-20483))

Author: michaelkedar

gcp/api/server.py

@@ -26,7 +26,6 @@
 import time
 import concurrent.futures
 from typing import Callable
-from urllib.parse import urlparse
 
 from collections import defaultdict
 
@@ -110,38 +109,6 @@
 # ----
 
 
-def _normalize_git_repo_url(repo_url: str) -> str:
-  """Normalize git repository URL for matching by removing protocol/scheme.
-  
-  This enables matching git repositories regardless of whether they use
-  http, https, git, or other protocols. For example:
-  - http://git.musl-libc.org/git/musl
-  - https://git.musl-libc.org/git/musl
-  - git://git.musl-libc.org/git/musl
-  
-  Will all normalize to: git.musl-libc.org/git/musl
-  
-  Args:
-    repo_url: The git repository URL to normalize
-    
-  Returns:
-    The normalized URL without protocol/scheme
-  """
-  if not repo_url:
-    return repo_url
-
-  try:
-    parsed = urlparse(repo_url)
-    # Remove scheme and reconstruct without it
-    # Keep netloc (hostname) and path
-    normalized = parsed.netloc + parsed.path
-
-    # Remove trailing slash
-    return normalized.rstrip('/')
-  except Exception:
-    return repo_url
-
-
 def ndb_context(func):
   """Wrapper to create an NDB context."""
 
@@ -1082,8 +1049,8 @@ def _is_version_affected(affected_packages,
 
         # Normalize both URLs for comparison to handle protocol differences
         # (http vs https vs git://, etc.)
-        normalized_package_name = _normalize_git_repo_url(package_name)
-        normalized_repo_url = _normalize_git_repo_url(repo_url)
+        normalized_package_name = osv.normalize_repo_package(package_name)
+        normalized_repo_url = osv.normalize_repo_package(repo_url)
 
         if normalized_package_name != normalized_repo_url:
           continue

gcp/api/server_new.py

@@ -61,7 +61,12 @@ def query_package(context,
   if not package_name:
     return []
 
-  query = osv.AffectedVersions.query(osv.AffectedVersions.name == package_name)
+  query = osv.AffectedVersions.query(
+      osv.AffectedVersions.name.IN([
+          package_name,
+          # Also query the normalized name in case this is a GIT repo.
+          osv.normalize_repo_package(package_name)
+      ]))
   if ecosystem:
     query = query.filter(osv.AffectedVersions.ecosystem == ecosystem)
   query = query.order(osv.AffectedVersions.vuln_id)
@@ -83,7 +88,7 @@ def query_package(context,
     affected: osv.AffectedVersions = it.next()
     if affected.vuln_id == last_matched_id:
       continue
-    if not version or affected_affects(version, affected):
+    if not version or affected_affects(package_name, version, affected):
       if include_details:
         bugs.append(get_vuln_async(affected.vuln_id))
       else:
@@ -94,8 +99,16 @@ def query_package(context,
   return bugs
 
 
-def affected_affects(version: str, affected: osv.AffectedVersions) -> bool:
+def affected_affects(name: str, version: str,
+                     affected: osv.AffectedVersions) -> bool:
   """Check if a given version is affected by the AffectedVersions entry."""
+  # Make sure the package name correctly matches this entity.
+  if affected.ecosystem != 'GIT' and name != affected.name:
+    return False
+  if (affected.ecosystem == 'GIT' and
+      osv.normalize_repo_package(name) != affected.name):
+    return False
+
   if len(affected.versions) > 0:
     return _match_versions(version, affected)
   if len(affected.events) > 0:

gcp/api/server_test.py

@@ -15,7 +15,7 @@
 
 import unittest
 
-from server import should_skip_bucket, _normalize_git_repo_url
+from server import should_skip_bucket
 
 
 class ServerTest(unittest.TestCase):
@@ -41,38 +41,6 @@ def test_should_skip_bucket(self):
     for path, expected in test_cases:
       self.assertEqual(expected, should_skip_bucket(path))
 
-  def test_normalize_git_repo_url(self):
-    """Test _normalize_git_repo_url function."""
-    test_cases = [
-        # protocol normalization
-        ('http://git.musl-libc.org/git/musl', 'git.musl-libc.org/git/musl'),
-        ('https://git.musl-libc.org/git/musl', 'git.musl-libc.org/git/musl'),
-        ('git://git.musl-libc.org/git/musl', 'git.musl-libc.org/git/musl'),
-
-        # github examples
-        ('http://github.com/user/repo', 'github.com/user/repo'),
-        ('https://github.com/user/repo', 'github.com/user/repo'),
-        ('git://github.com/user/repo', 'github.com/user/repo'),
-
-        # trailing slash
-        ('https://github.com/user/repo/', 'github.com/user/repo'),
-        ('http://git.example.com/path/', 'git.example.com/path'),
-
-        # .git suffix preserved
-        ('https://github.com/user/repo.git', 'github.com/user/repo.git'),
-        ('http://git.example.com/repo.git', 'git.example.com/repo.git'),
-
-        # edge cases
-        ('', ''),
-        ('invalid-url', 'invalid-url'),
-        ('http://', ''),
-        ('https://hostname', 'hostname'),
-    ]
-
-    for repo_url, expected in test_cases:
-      with self.subTest(repo_url=repo_url):
-        self.assertEqual(expected, _normalize_git_repo_url(repo_url))
-
 
 if __name__ == '__main__':
   unittest.main()

osv/models.py

@@ -1212,7 +1212,7 @@ def affected_from_bug(entity: Bug) -> list[AffectedVersions]:
             AffectedVersions(
                 vuln_id=entity.db_id,
                 ecosystem='GIT',
-                name=repo_url,
+                name=normalize_repo_package(repo_url),
                 versions=affected.versions,
             ))
 
@@ -1245,6 +1245,36 @@ def diff_affected_versions(
   return added, removed
 
 
+def normalize_repo_package(repo_url: str) -> str:
+  """Normalize the repo_url for use with GIT AffectedVersions entities.
+  
+  Removes the scheme/protocol and the .git extension, and trailing slashes.
+  
+  For example:
+    - 'http://git.musl-libc.org/git/musl' (e.g. CVE-2017-15650)
+      and 'https://git.musl-libc.org/git/musl' (e.g. CVE-2025-26519)
+      both become 'git.musl-libc.org/git/musl'
+    - 'https://github.com/curl/curl.git' (e.g. CURL-CVE-2024-2004)
+      and 'https://github.com/curl/curl' (e.g. CVE-2025-5025)
+      both become 'github.com/curl/curl'
+  """
+  if not repo_url:
+    return repo_url
+
+  try:
+    parsed = urlparse(repo_url)
+    # Remove scheme and reconstruct without it
+    # Keep netloc (hostname) and path
+    normalized = parsed.netloc + parsed.path
+
+    # Remove trailing slash
+    normalized = normalized.rstrip('/')
+    normalized = normalized.removesuffix('.git')
+    return normalized
+  except Exception:
+    return repo_url
+
+
 # --- Indexer entities ---
 
 

osv/models_test.py

@@ -174,7 +174,7 @@ def test_bug_post_put(self):
         models.AffectedVersions(
             vuln_id=vuln_id,
             ecosystem='GIT',
-            name='https://github.com/test/test',
+            name='github.com/test/test',
             versions=['v1', 'v2']),
         models.AffectedVersions(
             vuln_id=vuln_id,
@@ -375,6 +375,38 @@ def test_oss_fuzz_private(self):
     blob = bucket.get_blob(os.path.join(gcs.VULN_PB_PATH, f'{vuln_id}.pb'))
     self.assertIsNone(blob)
 
+  def test_normalize_repo(self):
+    """Test normalize_repo_package function."""
+    test_cases = [
+        # protocol normalization
+        ('http://git.musl-libc.org/git/musl', 'git.musl-libc.org/git/musl'),
+        ('https://git.musl-libc.org/git/musl', 'git.musl-libc.org/git/musl'),
+        ('git://git.musl-libc.org/git/musl', 'git.musl-libc.org/git/musl'),
+
+        # github examples
+        ('http://github.com/user/repo', 'github.com/user/repo'),
+        ('https://github.com/user/repo', 'github.com/user/repo'),
+        ('git://github.com/user/repo', 'github.com/user/repo'),
+
+        # trailing slash
+        ('https://github.com/user/repo/', 'github.com/user/repo'),
+        ('http://git.example.com/path/', 'git.example.com/path'),
+
+        # .git suffix removed
+        ('https://github.com/user/repo.git', 'github.com/user/repo'),
+        ('http://git.example.com/repo.git', 'git.example.com/repo'),
+
+        # edge cases
+        ('', ''),
+        ('invalid-url', 'invalid-url'),
+        ('http://', ''),
+        ('https://hostname', 'hostname'),
+    ]
+
+    for repo_url, expected in test_cases:
+      with self.subTest(repo_url=repo_url):
+        self.assertEqual(expected, models.normalize_repo_package(repo_url))
+
 
 def setUpModule():
   """Set up the test module."""