Remove - from pkg and add comments

Author: viveksinghggits

.evergreen-functions.yml

@@ -560,7 +560,7 @@ functions:
           MACOS_NOTARY_KEY: ${macos_notary_keyid}
           MACOS_NOTARY_SECRET: ${macos_notary_secret}
         working_dir: src/github.com/mongodb/mongodb-kubernetes
-        binary: scripts/dev/run_python.sh scripts/release/kubectl-mongodb/python/build_kubectl_plugin.py
+        binary: scripts/dev/run_python.sh scripts/release/kubectl_mongodb/python/build_kubectl_plugin.py
 
   promote_kubectl_plugin_and_release:
     - command: subprocess.exec
@@ -584,7 +584,7 @@ functions:
           MACOS_NOTARY_KEY: ${macos_notary_keyid}
           MACOS_NOTARY_SECRET: ${macos_notary_secret}
         working_dir: src/github.com/mongodb/mongodb-kubernetes
-        binary: scripts/dev/run_python.sh scripts/release/kubectl-mongodb/python/promote_kubectl_plugin.py --release_version ${release_version} --staging_commit ${staging_commit_sha}
+        binary: scripts/dev/run_python.sh scripts/release/kubectl_mongodb/python/promote_kubectl_plugin.py --release_version ${release_version} --staging_commit ${staging_commit_sha}
 
   build_and_push_appdb_database:
     - command: subprocess.exec

.evergreen.yml

@@ -421,7 +421,7 @@ tasks:
           expansion_name: GH_TOKEN
       - func: promote_kubectl_plugin_and_release
         vars:
-          release_version: 6.0.0
+          release_version: 9.0.9
           staging_commit_sha: 68a45e3b9f9b2b000754926f
 
   - name: build_test_image

.goreleaser.yaml

@@ -19,9 +19,9 @@ builds:
     hooks:
       # This will notarize Apple binaries and replace goreleaser bins with the notarized ones
       post:
-        - cmd: ./scripts/release/kubectl-mongodb/kubectl_mac_notarize.sh
+        - cmd: ./scripts/release/kubectl_mongodb/kubectl_mac_notarize.sh
           output: true
-        - cmd: ./scripts/release/kubectl-mongodb/sign.sh {{ .Path }}
+        - cmd: ./scripts/release/kubectl_mongodb/sign.sh {{ .Path }}
           env:
             - GRS_USERNAME={{ .Env.GRS_USERNAME }}
             - GRS_PASSWORD={{ .Env.GRS_PASSWORD }}
@@ -30,7 +30,7 @@ builds:
             - SIGNING_IMAGE_URI={{ .Env.SIGNING_IMAGE_URI }}
             - ARTIFACTORY_USERNAME=mongodb-enterprise-kubernetes-operator
             - ARTIFACTORY_PASSWORD={{ .Env.ARTIFACTORY_PASSWORD }}
-        - cmd: ./scripts/release/kubectl-mongodb/verify.sh {{ .Path }} && echo "VERIFIED OK"
+        - cmd: ./scripts/release/kubectl_mongodb/verify.sh {{ .Path }} && echo "VERIFIED OK"
 
 archives:
   - format: tar.gz

requirements.txt

@@ -35,6 +35,7 @@ botocore==1.40.7
 boto3==1.40.7
 python-frontmatter==1.1.0
 python-on-whales==0.78.0
+PyGithub==2.7.0
 
 # from kubeobject
 freezegun==1.5.5

…ongodb/install_istio_separate_network.sh …ongodb/install_istio_separate_network.shscripts/release/kubectl-mongodb/install_istio_separate_network.sh renamed to scripts/release/kubectl_mongodb/install_istio_separate_network.sh scripts/release/kubectl-mongodb/install_istio_separate_network.sh renamed to scripts/release/kubectl_mongodb/install_istio_separate_network.sh

@@ -1,26 +1,31 @@
 import argparse
 import hashlib
 import os
+import subprocess
 import sys
 import tarfile
-import subprocess
 from pathlib import Path
-import boto3
 
+import boto3
 from botocore.exceptions import ClientError, NoCredentialsError, PartialCredentialsError
-# from github import Github, GithubException
+from github import Github, GithubException
+
+from lib.base_logger import logger
 
-GITHUB_REPO = "mongodb/mongodb-kubernetes"
 GITHUB_TOKEN = os.environ.get("GH_TOKEN")
+GITHUB_REPO = "mongodb/mongodb-kubernetes"
+
+AWS_REGION = "eu-north-1"
+
+STAGING_S3_BUCKET_NAME = "mongodb-kubernetes-dev"
+RELEASE_S3_BUCKET_NAME = "mongodb-kubernetes-staging"
+
+KUBECTL_PLUGIN_BINARY_NAME = "kubectl-mongodb"
+S3_BUCKET_KUBECTL_PLUGIN_SUBPATH = KUBECTL_PLUGIN_BINARY_NAME
 
 LOCAL_ARTIFACTS_DIR = "artifacts"
 CHECKSUMS_PATH = f"{LOCAL_ARTIFACTS_DIR}/checksums.txt"
 
-DEV_S3_BUCKET_NAME = "mongodb-kubernetes-dev"
-STAGING_S3_BUCKET_NAME = "mongodb-kubernetes-staging"
-
-S3_BUCKET_KUBECTL_PLUGIN_SUBPATH = "kubectl-mongodb"
-AWS_REGION = "eu-north-1"
 
 def main():
     parser = argparse.ArgumentParser()
@@ -44,57 +49,69 @@ def main():
 
     promote_artifacts(artifacts, args.release_version)
 
-    # upload_assets_to_github_release(artifacts_tar, args.release_version)
+    upload_assets_to_github_release(artifacts, args.release_version)
 
+
+# generate_checksums generates checksums for the artifacts that we are going to upload to github release as assets.
+# It's formatted: checksum  artifact_name
 def generate_checksums(artifacts: list[str]):
     checksums_path = Path(CHECKSUMS_PATH)
 
     with checksums_path.open("w") as out_file:
         for artifact in artifacts:
             artifact_path = Path(artifact)
             if not artifact_path.is_file() or not artifact_path.name.endswith(".tar.gz"):
-                print(f"skipping invalid tar file: {artifact_path}")
+                logger.info(f"skipping invalid tar file: {artifact_path}")
                 continue
 
             sha256 = hashlib.sha256()
             with open(artifact_path, "rb") as f:
-                for chunk in iter(lambda : f.read(8192), b""):
+                # read chunk of 8192 bites until end of file (b"") is received
+                for chunk in iter(lambda: f.read(8192), b""):
                     sha256.update(chunk)
 
             checksum_line = f"{sha256.hexdigest()}  {artifact_path.name}"
-            out_file.write(checksum_line+"\n")
+            out_file.write(checksum_line + "\n")
 
-    print(f"Checksums written to {checksums_path}")
+    logger.info(f"Checksums written to {checksums_path}")
     all_artifacts = list(artifacts) + [str(checksums_path.resolve())]
-    return  all_artifacts
+    return all_artifacts
+
 
+# promote_artifacts promotes (copies) the downloaded staging artifacts to release S3 bucket.
 def promote_artifacts(artifacts: list[str], release_version: str):
     s3_client = boto3.client("s3", region_name=AWS_REGION)
     for file in artifacts:
-        if not os.path.isfile(file) or not file.endswith(('.tar.gz', '.txt')):
-            print(f"skipping invalid or non-tar file: {file}")
+        if not os.path.isfile(file) or not file.endswith((".tar.gz", ".txt")):
+            logger.info(f"Skipping invalid or non-tar/checksum file: {file}")
             continue
 
         file_name = os.path.basename(file)
         s3_key = os.path.join(S3_BUCKET_KUBECTL_PLUGIN_SUBPATH, release_version, file_name)
 
         try:
-            s3_client.upload_file(file, STAGING_S3_BUCKET_NAME, s3_key)
+            s3_client.upload_file(file, RELEASE_S3_BUCKET_NAME, s3_key)
         except ClientError as e:
-            print(f"failed to upload the file {file}: {e}")
+            logger.debug(f"failed to upload the file {file}: {e}")
             sys.exit(1)
 
-    print("artifacts were promoted to release bucket successfully")
+    logger.info("Artifacts were promoted to release bucket successfully")
 
 
+# notarize_artifacts notarizes the darwin goreleaser binaries in-place.
 def notarize_artifacts(release_version: str):
-    notarize_result = subprocess.run(["scripts/release/kubectl-mongodb/kubectl_mac_notarize.sh", release_version], capture_output=True, text=True)
+    notarize_result = subprocess.run(
+        ["scripts/release/kubectl_mongodb/kubectl_mac_notarize.sh", release_version], capture_output=True, text=True
+    )
     if notarize_result.returncode == 0:
-        print("notarization of artifacts was successful")
+        logger.info("Notarization of artifacts was successful")
     else:
-        print(f"notarization of artifacts failed. \nstdout: {notarize_result.stdout} \nstderr: {notarize_result.stderr}")
+        logger.debug(
+            f"Notarization of artifacts failed. \nstdout: {notarize_result.stdout} \nstderr: {notarize_result.stderr}"
+        )
         sys.exit(1)
 
+
 # sign_and_verify_artifacts iterates over the goreleaser artifacts, that have been downloaded from S3, and
 # signs and verifies them.
 def sign_and_verify_artifacts():
@@ -110,18 +127,27 @@ def sign_and_verify_artifacts():
                 file_path = os.path.join(subdir_path, file)
 
                 if os.path.isfile(file_path):
-                    sign_result = subprocess.run(["scripts/release/kubectl-mongodb/sign.sh", file_path], capture_output=True, text=True)
+                    # signing an already signed artifact fails with `Signature already exixts. Displaying proof`.
+                    sign_result = subprocess.run(
+                        ["scripts/release/kubectl_mongodb/sign.sh", file_path], capture_output=True, text=True
+                    )
                     if sign_result.returncode == 0:
-                        print(f"artifact {file_path} was signed successfully")
+                        logger.info(f"Artifact {file_path} was signed successfully")
                     else:
-                        print(f"signing the artifact {file_path} failed. \nstdout: {sign_result.stdout} \nstderr: {sign_result.stderr}")
+                        logger.debug(
+                            f"Signing the artifact {file_path} failed. \nstdout: {sign_result.stdout} \nstderr: {sign_result.stderr}"
+                        )
                         sys.exit(1)
 
-                    verify_result = subprocess.run(["scripts/release/kubectl-mongodb/verify.sh", file_path], capture_output=True, text=True)
+                    verify_result = subprocess.run(
+                        ["scripts/release/kubectl_mongodb/verify.sh", file_path], capture_output=True, text=True
+                    )
                     if verify_result.returncode == 0:
-                        print(f"artifact {file_path} was verified successfully")
+                        logger.info(f"Artifact {file_path} was verified successfully")
                     else:
-                        print(f"verification of the artifact {file_path} failed. \nstdout: {verify_result.stdout} \nstderr: {verify_result.stderr}")
+                        logger.debug(
+                            f"Verification of the artifact {file_path} failed. \nstdout: {verify_result.stdout} \nstderr: {verify_result.stderr}"
+                        )
                         sys.exit(1)
 
 
@@ -142,18 +168,19 @@ def s3_artifacts_path_to_local_path(release_version: str, commit_sha: str):
     }
 
 
-# download_artifacts_from_s3 downloads the staging artifacts from S3 and puts them in the local dir LOCAL_ARTIFACTS_DIR
+# download_artifacts_from_s3 downloads the staging artifacts (only that ones that we would later promote) from S3 and puts
+# them in the local dir LOCAL_ARTIFACTS_DIR.
 # ToDo: if the artifacts are not present at correct location, this is going to fail silently, we should instead fail this
 def download_artifacts_from_s3(release_version: str, commit_sha: str):
-    print(f"\nStarting download of artifacts from S3 bucket: {DEV_S3_BUCKET_NAME}")
+    logger.info(f"Starting download of artifacts from staging S3 bucket: {STAGING_S3_BUCKET_NAME}")
 
     try:
         s3_client = boto3.client("s3", region_name=AWS_REGION)
     except (NoCredentialsError, PartialCredentialsError):
-        print("ERROR: AWS credentials were not set.")
+        logger.debug("ERROR: AWS credentials were not set.")
         sys.exit(1)
     except Exception as e:
-        print(f"An error occurred connecting to S3: {e}")
+        logger.debug(f"An error occurred connecting to S3: {e}")
         sys.exit(1)
 
     artifacts_to_promote = s3_artifacts_path_to_local_path(release_version, commit_sha)
@@ -165,7 +192,7 @@ def download_artifacts_from_s3(release_version: str, commit_sha: str):
     for s3_artifact_dir, local_subdir in artifacts_to_promote.items():
         try:
             paginator = s3_client.get_paginator("list_objects_v2")
-            pages = paginator.paginate(Bucket=DEV_S3_BUCKET_NAME, Prefix=s3_artifact_dir)
+            pages = paginator.paginate(Bucket=STAGING_S3_BUCKET_NAME, Prefix=s3_artifact_dir)
             for page in pages:
                 # "Contents" corresponds to the directory in the S3 bucket
                 if "Contents" not in page:
@@ -186,20 +213,21 @@ def download_artifacts_from_s3(release_version: str, commit_sha: str):
                     # Create the local directory structure if it doesn't exist
                     os.makedirs(os.path.dirname(final_local_path), exist_ok=True)
 
-                    print(f"Downloading {s3_key} to {final_local_path}")
-                    s3_client.download_file(DEV_S3_BUCKET_NAME, s3_key, final_local_path)
+                    logger.info(f"Downloading staging artifact {s3_key} to {final_local_path}")
+                    s3_client.download_file(STAGING_S3_BUCKET_NAME, s3_key, final_local_path)
                     download_count += 1
 
         except ClientError as e:
-            print(f"ERROR: Failed to list or download from prefix '{s3_artifact_dir}'. S3 Client Error: {e}")
+            logger.debug(f"ERROR: Failed to list or download from prefix '{s3_artifact_dir}'. S3 Client Error: {e}")
             return False
 
-    print("All the artifacts have been downloaded successfully.")
+    logger.info("All the artifacts have been downloaded successfully.")
     return True
 
 
+# create_tarballs creates `.tar.gz` archives for the artifacts that before promoting them.
 def create_tarballs():
-    print(f"\nCreating archives for subdirectories in {LOCAL_ARTIFACTS_DIR}")
+    logger.info(f"Creating archives for subdirectories in {LOCAL_ARTIFACTS_DIR}")
     created_archives = []
     original_cwd = os.getcwd()
     try:
@@ -213,47 +241,51 @@ def create_tarballs():
                     tar.add(dir_name)
 
                 full_archive_path = os.path.join(original_cwd, LOCAL_ARTIFACTS_DIR, archive_name)
-                print(f"Successfully created archive at {full_archive_path}")
+                logger.info(f"Successfully created archive at {full_archive_path}")
                 created_archives.append(full_archive_path)
 
     except Exception as e:
-        print(f"ERROR: Failed to create tar.gz archives: {e}")
-        return []
+        logger.debug(f"ERROR: Failed to create tar.gz archives: {e}")
+        sys.exit(1)
     finally:
         os.chdir(original_cwd)
 
     return created_archives
 
 
-# def upload_assets_to_github_release(asset_paths, release_version: str):
-#     if not GITHUB_TOKEN:
-#         print("ERROR: GITHUB_TOKEN environment variable not set.")
-#         sys.exit(1)
-#
-#     try:
-#         g = Github(GITHUB_TOKEN)
-#         repo = g.get_repo(GITHUB_REPO)
-#     except GithubException as e:
-#         print(f"ERROR: Could not connect to GitHub or find repository '{GITHUB_REPO}', Error {e}.")
-#         sys.exit(1)
-#
-#     try:
-#         release = repo.get_release(release_version)
-#     except GithubException as e:
-#         print(
-#             f"ERROR: Could not find release with tag '{release_version}'. Please ensure release exists already. Error: {e}"
-#         )
-#         return
-#
-#     for asset_path in asset_paths:
-#         asset_name = os.path.basename(asset_path)
-#         print(f"Uploading artifact '{asset_name}' to github release as asset")
-#         try:
-#             release.upload_asset(path=asset_path, name=asset_name, content_type="application/gzip")
-#         except GithubException as e:
-#             print(f"ERROR: Failed to upload asset {asset_name}. Error: {e}")
-#         except Exception as e:
-#             print(f"An unexpected error occurred during upload of {asset_name}: {e}")
+# upload_assets_to_github_release uploads the release artifacts (downloaded notarized/signed staging artifacts) to
+# the github release as assets.
+def upload_assets_to_github_release(asset_paths, release_version: str):
+    if not GITHUB_TOKEN:
+        logger.info("ERROR: GITHUB_TOKEN environment variable not set.")
+        sys.exit(1)
+
+    try:
+        g = Github(GITHUB_TOKEN)
+        repo = g.get_repo(GITHUB_REPO)
+    except GithubException as e:
+        logger.info(f"ERROR: Could not connect to GitHub or find repository '{GITHUB_REPO}', Error {e}.")
+        sys.exit(1)
+
+    try:
+        release = repo.get_release(release_version)
+    except GithubException as e:
+        logger.debug(
+            f"ERROR: Could not find release with tag '{release_version}'. Please ensure release exists already. Error: {e}"
+        )
+        sys.exit(2)
+
+    for asset_path in asset_paths:
+        asset_name = os.path.basename(asset_path)
+        logger.info(f"Uploading artifact '{asset_name}' to github release as asset")
+        try:
+            release.upload_asset(path=asset_path, name=asset_name, content_type="application/gzip")
+        except GithubException as e:
+            logger.debug(f"ERROR: Failed to upload asset {asset_name}. Error: {e}")
+            sys.exit(2)
+        except Exception as e:
+            logger.debug(f"An unexpected error occurred during upload of {asset_name}: {e}")
+            sys.exit(2)
 
 
 if __name__ == "__main__":