add sigstore signing integration test (#324)

* add integration test marker

By default `hatch test` wont run integration tests, but they can be
run with `hatch test -m integration`.

Signed-off-by: Spencer Schrock <[email protected]>

* add signing test using public OIDC beacon

Signed-off-by: Spencer Schrock <[email protected]>

* add integration workflow

Signed-off-by: Spencer Schrock <[email protected]>

---------

Signed-off-by: Spencer Schrock <[email protected]>

Author: spencerschrock

.github/workflows/integration.yml

@@ -0,0 +1,47 @@
+# Copyright 2024 The Sigstore Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Run integration tests
+on:
+  pull_request:
+    branches: [main]
+    types: [opened, synchronize]
+    paths-ignore:
+      - '**/*.md'
+      - '*.md'
+  workflow_dispatch:
+
+permissions: {}
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  model-signing-integration-test:
+    name: Signing with Python ${{ matrix.python-version }} on ${{ startsWith(matrix.os, 'macos-') && 'macOS' || startsWith(matrix.os, 'windows-') && 'Windows' || 'Linux' }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false # Don't cancel other jobs if one fails
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ['3.10', '3.11', '3.12', '3.13']
+    steps:
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+    - name: Set up Hatch
+      uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc
+    - name: Run integration tests
+      run: |
+        set -euxo pipefail
+        hatch test -c -py ${{ matrix.python-version }} -m integration

pyproject.toml

@@ -61,6 +61,7 @@ packages = ["src/model_signing"]
 installer = "pip"
 parallel = true
 randomize = true
+extra-args = ["-m", "not integration"]
 
 [[tool.hatch.envs.hatch-test.matrix]]
 python = ["3.10", "3.11", "3.12", "3.13"]
@@ -119,6 +120,9 @@ skip_empty = true
 
 # Add support for testing via the old `pytest .` way, too.
 [tool.pytest.ini_options]
+markers = [
+  "integration: mark a test as an integration test.",
+]
 pythonpath = "src"
 
 [tool.ruff]

tests/api_test.py

@@ -0,0 +1,101 @@
+# Copyright 2024 The Sigstore Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for the top level API."""
+
+from base64 import b64decode
+from datetime import datetime
+from datetime import timedelta
+import json
+import os
+import subprocess
+from tempfile import TemporaryDirectory
+import time
+
+import pytest
+
+from model_signing import api as model_signing_api
+
+
+_MIN_VALIDITY = timedelta(minutes=1)
+_MAX_RETRY_TIME = timedelta(minutes=5)
+_RETRY_SLEEP_SECS = 30
+
+
+class DangerousPublicOIDCBeacon:
+    """Fetches and validates tokens from Sigstore's testing beacon repo."""
+
+    def __init__(self):
+        self._token = ""
+
+    def _fetch(self) -> None:
+        # the git approach is apparently fresher than https://raw.githubusercontent.com
+        # https://github.com/sigstore-conformance/extremely-dangerous-public-oidc-beacon/issues/17
+        git_url = "https://github.com/sigstore-conformance/extremely-dangerous-public-oidc-beacon.git"
+        with TemporaryDirectory() as tmpdir:
+            base_cmd = [
+                "git",
+                "clone",
+                "--quiet",
+                "--single-branch",
+                "--branch=current-token",
+                "--depth=1",
+            ]
+            subprocess.run(base_cmd + [git_url, tmpdir], check=True)
+            token_path = os.path.join(tmpdir, "oidc-token.txt")
+            with open(token_path) as f:
+                self._token = f.read().rstrip()
+
+    def _expiration(self) -> datetime:
+        payload = self._token.split(".")[1]
+        payload += "=" * (4 - len(payload) % 4)
+        payload_json = json.loads(b64decode(payload))
+        return datetime.fromtimestamp(payload_json["exp"])
+
+
+@pytest.fixture
+def sigstore_oidc_beacon_token():
+    beacon = DangerousPublicOIDCBeacon()
+    start = datetime.now()
+    while True:
+        now = datetime.now()
+        deadline = now + _MIN_VALIDITY
+        beacon._fetch()
+        exp = beacon._expiration()
+        if deadline < exp:
+            return beacon._token
+        if now > start + _MAX_RETRY_TIME:
+            break
+        time.sleep(_RETRY_SLEEP_SECS)
+    pytest.fail("unable to fetch token within time limit")
+
+
+class TestSigstoreSigning:
+    @pytest.mark.integration
+    def test_sign_and_verify(
+        self, sigstore_oidc_beacon_token, sample_model_folder, tmp_path
+    ):
+        sc = model_signing_api.SigningConfig()
+        sc.set_sigstore_signer(
+            use_staging=True, identity_token=sigstore_oidc_beacon_token
+        )
+        signature_path = tmp_path / "model.sig"
+        sc.sign(sample_model_folder, signature_path)
+        expected_identity = "https://github.com/sigstore-conformance/extremely-dangerous-public-oidc-beacon/.github/workflows/extremely-dangerous-oidc-beacon.yml@refs/heads/main"
+        model_signing_api.verify(
+            sample_model_folder,
+            signature_path,
+            identity=expected_identity,
+            use_staging=True,
+        )