Implement bulk upload releases (#1187)

* Update pre-commit configuration and clean up Dockerfiles and workflows

- Added new hooks to the pre-commit configuration for better code quality checks.
- Cleaned up whitespace in Dockerfiles and docker-compose.yml for consistency.
- Removed unnecessary blank lines in GitHub Actions workflows to enhance readability.

* Update caniuse-lite dependency in package-lock.json to version 1.0.30001720, ensuring compatibility with the latest features and improvements.

* Refactor upload validation tests to use mocker for improved readability and consistency. Updated test functions to replace monkeypatching with mocker.patch, enhancing the clarity of mock setups and ensuring better integration with pytest's mocking capabilities.

* Refactor SCSS files to use `@use` instead of `@import` for improved modularity and maintainability. Updated color adjustments to utilize `color.adjust` for better control over color lightness in hover states.

* Add bulk release functionality for uploads

- Introduced BulkReleaseForm to handle bulk release submissions.
- Implemented BulkReleaseView to manage the release of multiple uploads at once, including validation and error handling.
- Added a new bulk release template for user interaction.
- Updated project detail view to link to the bulk release feature when multiple uploads are present.
- Created comprehensive tests for bulk release functionality, ensuring robust validation and release processes.

* Refactor mock_user fixture to utilize mocker for improved testing consistency. This change enhances the clarity of mock setups in the test suite.

* Refactor BulkReleaseView to inherit from ProjectMixin and MethodView

- Updated the inheritance structure of BulkReleaseView to improve functionality and maintainability.
- Adjusted tests to verify the new inheritance and decorators, ensuring proper permission checks and method availability.

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Enhance bulk release tests for improved clarity and coverage

- Refactored test functions in test_bulk_release.py to utilize consistent mocking practices, improving readability and maintainability.
- Added comprehensive tests for various scenarios in the bulk release process, including form validation, release success, and error handling.
- Updated test_upload_validation.py to streamline HTTP error handling and ensure proper warning treatment for 404 errors.
- Ensured all tests are integrated with Flask context for better coverage and reliability.

* Formatting.

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: jezdez

.github/workflows/cleanup-cache.yml

@@ -14,7 +14,7 @@ jobs:
   cleanup:
     runs-on: ubuntu-latest
     if: github.repository_owner == 'jazzband'
-    
+
     steps:
       - name: Log in to Container Registry
         uses: docker/login-action@v3
@@ -27,13 +27,13 @@ jobs:
         run: |
           # Get all image versions
           PACKAGE_NAME=$(echo "${{ env.BASE_IMAGE_NAME }}" | cut -d'/' -f2-)
-          
+
           # Get all versions of the base image package
           VERSIONS=$(gh api \
             -H "Accept: application/vnd.github.v3+json" \
             "/orgs/${{ github.repository_owner }}/packages/container/$PACKAGE_NAME/versions" \
             --jq '.[].id' | head -n -5)  # Keep the 5 most recent versions
-          
+
           # Delete old versions (keep the latest 5)
           for version_id in $VERSIONS; do
             echo "Deleting version ID: $version_id"
@@ -43,4 +43,4 @@ jobs:
               "/orgs/${{ github.repository_owner }}/packages/container/$PACKAGE_NAME/versions/$version_id" || true
           done
         env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} 
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test.yml

@@ -137,7 +137,7 @@ jobs:
           # Create coverage output directory
           mkdir -p $PWD/coverage-data
           chmod 777 $PWD/coverage-data
-          
+
           # Run tests with volume mount for coverage output
           docker compose run --rm \
             -e COVERAGE_FILE \

.pre-commit-config.yaml

@@ -1,4 +1,11 @@
 repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: trailing-whitespace
+      - id: check-yaml
+      - id: check-added-large-files
+      - id: check-merge-conflict
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: v0.6.5
     hooks:

Dockerfile.app

@@ -35,4 +35,4 @@ USER 10001
 
 EXPOSE 5000
 
-ENTRYPOINT ["/app/docker-entrypoint.sh", "--"] 
+ENTRYPOINT ["/app/docker-entrypoint.sh", "--"]

Dockerfile.base

@@ -60,4 +60,4 @@ LABEL deps_hash=$DEPS_HASH
 
 RUN chown -R 10001:10001 /app
 
-USER 10001 
+USER 10001

docker-compose.yml

@@ -13,7 +13,7 @@ services:
     platform: linux/amd64
     # Use JAZZBAND_IMAGE if set (for CI), otherwise build locally
     image: ${JAZZBAND_IMAGE:-}
-    build: 
+    build:
       context: .
       dockerfile: ${DOCKERFILE:-Dockerfile}
     volumes:

jazzband/projects/forms.py

@@ -145,5 +145,16 @@ def add_global_error(self, *messages):
         self.global_errors.extend(messages)
 
 
+class BulkReleaseForm(TwoFactorAuthValidation, ProjectNameForm):
+    submit = SubmitField("Bulk Release")
+
+    def __init__(self, *args, **kwargs):
+        self.global_errors = []
+        super().__init__(*args, **kwargs)
+
+    def add_global_error(self, *messages):
+        self.global_errors.extend(messages)
+
+
 class DeleteForm(TwoFactorAuthValidation, ProjectNameForm):
     submit = SubmitField("Delete")

jazzband/projects/views.py

@@ -26,7 +26,6 @@
 from packaging.utils import canonicalize_name as safe_name
 from packaging.version import parse as parse_version
 import requests
-from requests.exceptions import HTTPError
 from sqlalchemy import desc, nullsfirst, nullslast
 from sqlalchemy.sql.expression import func
 from werkzeug.security import safe_join
@@ -39,7 +38,7 @@
 from ..exceptions import eject
 from ..members.decorators import member_required
 from ..tasks import spinach
-from .forms import DeleteForm, ReleaseForm, UploadForm
+from .forms import BulkReleaseForm, DeleteForm, ReleaseForm, UploadForm
 from .models import Project, ProjectMembership, ProjectUpload
 from .tasks import send_new_upload_notifications, update_upload_ordering
 
@@ -469,8 +468,8 @@ def validate_upload(self, timeout=10):
             if e.response.status_code == 404:
                 # File not yet available - likely CDN delay
                 warning = (
-                    f"File not yet visible on PyPI API. This is likely due to CDN "
-                    f"propagation delays and should resolve shortly."
+                    "File not yet visible on PyPI API. This is likely due to CDN "
+                    "propagation delays and should resolve shortly."
                 )
                 logger.warning(warning)
                 warnings.append(warning)
@@ -724,6 +723,184 @@ def post(self, name, upload_id):
             return context
 
 
+class BulkReleaseView(ProjectMixin, MethodView):
+    """
+    A view to release all uploads of a given version at once.
+    """
+
+    methods = ["GET", "POST"]
+    decorators = [login_required, templated()]
+
+    def project_query(self, name):
+        """Override to include authorization logic for project leads."""
+        projects = super().project_query(name)
+        if current_user_is_roadie():
+            return projects
+        return projects.filter(
+            Project.membership.any(user=current_user, is_lead=True),
+        )
+
+    def get_unreleased_uploads_for_version(self, version):
+        """Get all unreleased uploads for a specific version."""
+        return self.project.uploads.filter_by(version=version, released_at=None).all()
+
+    def validate_uploads_bulk(self, uploads, timeout=10):
+        """
+        Validate multiple uploads at once.
+        Returns (overall_success, all_errors, all_warnings) tuple.
+        """
+        all_errors = []
+        all_warnings = []
+
+        for upload in uploads:
+            # Create a temporary UploadReleaseView to use its validation logic
+            temp_view = UploadReleaseView()
+            temp_view.upload = upload
+            temp_view.project = self.project
+
+            success, errors, warnings = temp_view.validate_upload(timeout)
+
+            # Prefix errors/warnings with filename for clarity
+            for error in errors:
+                all_errors.append(f"{upload.filename}: {error}")
+            for warning in warnings:
+                all_warnings.append(f"{upload.filename}: {warning}")
+
+        # Overall success if no blocking errors
+        return len(all_errors) == 0, all_errors, all_warnings
+
+    def release_uploads_bulk(self, uploads):
+        """
+        Release multiple uploads using twine.
+        Returns (success, twine_outputs) tuple.
+        """
+        twine_outputs = []
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            # Copy all files to temp directory
+            upload_paths = []
+            for upload in uploads:
+                upload_path = os.path.join(tmpdir, upload.filename)
+                shutil.copy(upload.full_path, upload_path)
+                upload_paths.append(upload_path)
+
+            # Run twine upload on all files at once
+            files_arg = " ".join(f'"{path}"' for path in upload_paths)
+            twine_run = delegator.run(f"twine upload {files_arg}")
+            twine_outputs.append(twine_run)
+
+        return twine_run.return_code == 0, twine_outputs
+
+    def get(self, name, version):
+        if not current_app.config["RELEASE_ENABLED"]:
+            message = "Releasing is currently out of service"
+            flash(message)
+            return self.redirect_to_project()
+
+        uploads = self.get_unreleased_uploads_for_version(version)
+
+        if not uploads:
+            flash(f"No unreleased uploads found for version {version}")
+            return self.redirect_to_project()
+
+        # Check if any uploads are already released
+        released_uploads = (
+            self.project.uploads.filter_by(version=version)
+            .filter(ProjectUpload.released_at.isnot(None))
+            .all()
+        )
+
+        bulk_release_form = BulkReleaseForm(project_name=self.project.name)
+
+        return {
+            "project": self.project,
+            "version": version,
+            "uploads": uploads,
+            "released_uploads": released_uploads,
+            "bulk_release_form": bulk_release_form,
+        }
+
+    def post(self, name, version):
+        if not current_app.config["RELEASE_ENABLED"]:
+            message = "Releasing is currently out of service"
+            flash(message)
+            return self.redirect_to_project()
+
+        uploads = self.get_unreleased_uploads_for_version(version)
+
+        if not uploads:
+            flash(f"No unreleased uploads found for version {version}")
+            return self.redirect_to_project()
+
+        bulk_release_form = BulkReleaseForm(project_name=self.project.name)
+
+        context = {
+            "project": self.project,
+            "version": version,
+            "uploads": uploads,
+            "bulk_release_form": bulk_release_form,
+        }
+
+        if bulk_release_form.validate_on_submit():
+            # Validate all uploads first
+            validation_success, all_errors, all_warnings = self.validate_uploads_bulk(
+                uploads
+            )
+
+            # Add validation errors to form (these will block release)
+            if all_errors:
+                bulk_release_form.add_global_error(*all_errors)
+
+            # Show warnings as flash messages (these won't block)
+            for warning in all_warnings:
+                flash(warning, category="warning")
+
+            # Proceed with bulk release if no blocking errors
+            if not all_errors:
+                # Release all uploads at once
+                release_success, twine_outputs = self.release_uploads_bulk(uploads)
+
+                if release_success:
+                    # Mark all uploads as released
+                    release_time = datetime.utcnow()
+                    released_count = 0
+
+                    for upload in uploads:
+                        upload.released_at = release_time
+                        upload.save()
+                        released_count += 1
+
+                    if all_warnings:
+                        message = (
+                            f"Successfully released {released_count} uploads for version {version} to PyPI. "
+                            f"Note: Some validation warnings were encountered (see above)."
+                        )
+                    else:
+                        message = f"Successfully released {released_count} uploads for version {version} to PyPI."
+
+                    flash(message, category="success")
+                    logger.info(
+                        f"Bulk release successful: {released_count} uploads for {self.project.name} v{version}"
+                    )
+                    return self.redirect_to_project()
+                else:
+                    error = f"Bulk release of version {version} failed."
+                    bulk_release_form.add_global_error(error)
+                    logger.error(
+                        error,
+                        extra={
+                            "data": {
+                                "twine_outputs": [
+                                    output.out for output in twine_outputs
+                                ]
+                            }
+                        },
+                    )
+                    context.update({"twine_outputs": twine_outputs})
+
+        return context
+
+
 # /projects/test-project/1/delete
 projects.add_url_rule(
     "/<name>/upload/<upload_id>/delete", view_func=UploadDeleteView.as_view("delete")
@@ -742,6 +919,10 @@ def post(self, name, upload_id):
 projects.add_url_rule(
     "/<name>/upload/<upload_id>/release", view_func=UploadReleaseView.as_view("release")
 )
+# /projects/test-project/release/1.0.0
+projects.add_url_rule(
+    "/<name>/release/<version>", view_func=BulkReleaseView.as_view("bulk_release")
+)
 # /projects/test-project/join
 projects.add_url_rule("/<name>/join", view_func=JoinView.as_view("join"))
 # /projects/test-project/leave