diff --git a/.evergreen-functions.yml b/.evergreen-functions.yml index d7b2414c37..4b0131b342 100644 --- a/.evergreen-functions.yml +++ b/.evergreen-functions.yml @@ -507,14 +507,13 @@ functions: params: working_dir: src/github.com/mongodb/mongodb-kubernetes binary: scripts/release/publish_helm_chart.sh - + build_multi_cluster_binary: - command: subprocess.exec - type: setup params: working_dir: src/github.com/mongodb/mongodb-kubernetes - binary: scripts/evergreen/build_multi_cluster_kubeconfig_creator.sh - + binary: scripts/dev/run_python.sh scripts/release/kubectl-mongodb/python/build_kubectl_plugin.py + build_and_push_appdb_database: - command: subprocess.exec params: @@ -854,6 +853,7 @@ functions: curl -fL "${goreleaser_pro_tar_gz}" --output goreleaser_Linux_x86_64.tar.gz tar -xf goreleaser_Linux_x86_64.tar.gz chmod 755 ./goreleaser + sudo cp goreleaser /usr/local/bin/ install_macos_notarization_service: - command: shell.exec diff --git a/.evergreen.yml b/.evergreen.yml index 3a25ef758a..eb2a827829 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -82,6 +82,7 @@ variables: - func: clone - func: switch_context - func: setup_building_host_minikube + - func: install_goreleaser - func: build_multi_cluster_binary - &setup_group_multi_cluster @@ -91,6 +92,7 @@ variables: - func: download_kube_tools - func: switch_context - func: setup_building_host + - func: install_goreleaser - func: build_multi_cluster_binary - &setup_and_teardown_task_cloudqa @@ -346,6 +348,7 @@ tasks: commands: - func: clone - func: setup_building_host + - func: install_goreleaser - func: build_multi_cluster_binary - func: pipeline vars: @@ -355,6 +358,7 @@ tasks: commands: - func: clone - func: setup_building_host + - func: install_goreleaser - func: build_multi_cluster_binary - func: pipeline vars: @@ -373,6 +377,7 @@ tasks: add_to_path: - ${workdir}/bin command: scripts/evergreen/setup_minikube_host.sh + - func: install_goreleaser - func: build_multi_cluster_binary - func: build_test_image_ibm diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 70b220d431..884ae3fc40 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -14,6 +14,8 @@ builds: goarch: - amd64 - arm64 + - s390x + - ppc64le hooks: # This will notarize Apple binaries and replace goreleaser bins with the notarized ones post: diff --git a/build_info.json b/build_info.json index 0c1f6a5384..4a56e98704 100644 --- a/build_info.json +++ b/build_info.json @@ -349,14 +349,14 @@ "binaries": { "kubectl-mongodb": { "patch": { - "s3-store": "s3://kubectl-mongodb/dev", + "s3-store": "mongodb-kubernetes-dev", "platforms": [ "linux/amd64" ] }, "staging": { - "sign": true, - "s3-store": "s3://kubectl-mongodb/staging", + "sign": false, + "s3-store": "mongodb-kubernetes-staging", "platforms": [ "darwin/amd64", "darwin/arm64", @@ -366,7 +366,7 @@ }, "release": { "sign": true, - "s3-store": "s3://kubectl-mongodb/prod", + "s3-store": "mongodb-kubernetes-release", "platforms": [ "darwin/amd64", "darwin/arm64", diff --git a/scripts/release/kubectl-mongodb/python/build_kubectl_plugin.py b/scripts/release/kubectl-mongodb/python/build_kubectl_plugin.py new file mode 100755 index 0000000000..5e0d9b3bb4 --- /dev/null +++ b/scripts/release/kubectl-mongodb/python/build_kubectl_plugin.py @@ -0,0 +1,152 @@ +import os +import subprocess +import sys + +import boto3 +from botocore.exceptions import ClientError, NoCredentialsError, PartialCredentialsError + +from lib.base_logger import logger +from scripts.release.build.build_info import ( + load_build_info, +) + +AWS_REGION = "eu-north-1" +KUBECTL_PLUGIN_BINARY_NAME = "kubectl-mongodb" +S3_BUCKET_KUBECTL_PLUGIN_SUBPATH = KUBECTL_PLUGIN_BINARY_NAME + +GORELEASER_DIST_DIR = "dist" + + +def run_goreleaser(): + try: + command = ["goreleaser", "build", "--snapshot", "--clean", "--skip", "post-hooks"] + + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1) + + for log in iter(process.stdout.readline, ""): + print(log, end="") + + process.stdout.close() + exit_code = process.wait() + + if exit_code != 0: + logger.debug(f"GoReleaser command failed with exit code {exit_code}.") + sys.exit(1) + + logger.info("GoReleaser build completed successfully!") + + except FileNotFoundError: + logger.debug( + "ERROR: 'goreleaser' command not found. Please ensure goreleaser is installed and in your system's PATH." + ) + sys.exit(1) + except Exception as e: + logger.debug(f"An unexpected error occurred while running `goreleaser build`: {e}") + sys.exit(1) + + +# upload_artifacts_to_s3 uploads the artifacts that are generated by goreleaser to S3 bucket at a specific path. +# The S3 bucket and version are figured out and passed to this function based on BuildScenario. +def upload_artifacts_to_s3(s3_bucket: str, version: str): + if not os.path.isdir(GORELEASER_DIST_DIR): + logger.info(f"ERROR: GoReleaser dist directory '{GORELEASER_DIST_DIR}' not found.") + sys.exit(1) + + try: + s3_client = boto3.client("s3", region_name=AWS_REGION) + except (NoCredentialsError, PartialCredentialsError): + logger.debug("ERROR: Failed to create S3 client. AWS credentials not found.") + sys.exit(1) + except Exception as e: + logger.debug(f"An error occurred connecting to S3: {e}") + sys.exit(1) + + uploaded_files = 0 + # iterate over all the files generated by goreleaser in the dist directory and upload them to S3 + for subdir in os.listdir(GORELEASER_DIST_DIR): + subdir_path = os.path.join(GORELEASER_DIST_DIR, subdir) + if not os.path.isdir(subdir_path): + continue # not a directory + + for filename in os.listdir(subdir_path): + local_file_path = os.path.join(subdir_path, filename) + if not os.path.isfile(local_file_path): + continue + + s3_key = s3_path(local_file_path, version) + logger.info(f"Uploading artifact {local_file_path} to s3://{s3_bucket}/{s3_key}") + try: + s3_client.upload_file(local_file_path, s3_bucket, s3_key) + logger.info(f"Successfully uploaded the artifact {filename}") + uploaded_files += 1 + except Exception as e: + logger.debug(f"ERROR: Failed to upload file {filename}: {e}") + sys.exit(1) + + if uploaded_files > 0: + logger.info(f"Successfully uploaded {uploaded_files} kubectl-mongodb plugin artifacts to S3.") + + +# s3_path returns the path where the artifacts should be uploaded to in S3 object store. +# For dev workflows it's going to be `kubectl-mongodb/{evg-patch-id}/{goreleaser-artifact}`, +# for staging workflows it would be `kubectl-mongodb/{commit-sha}/{goreleaser-artifact}`. +# The `version` string has the correct version (either patch id or commit sha), based on the BuildScenario. +def s3_path(local_path: str, version: str): + return f"{S3_BUCKET_KUBECTL_PLUGIN_SUBPATH}/{version}/{local_path}" + + +def s3_and_local_plugin_path(version: str) -> dict[str, str]: + s3_common_path = f"{S3_BUCKET_KUBECTL_PLUGIN_SUBPATH}/{version}/dist" + local_common_path = "docker/mongodb-kubernetes-tests" + # path in s3 : local path where tests image expects the binary + return { + f"{s3_common_path}/kubectl-mongodb_linux_amd64_v1/kubectl-mongodb": f"{local_common_path}/multi-cluster-kube-config-creator_amd64", + f"{s3_common_path}/kubectl-mongodb_linux_arm64/kubectl-mongodb": f"{local_common_path}/multi-cluster-kube-config-creator_arm64", + f"{s3_common_path}/kubectl-mongodb_linux_ppc64le/kubectl-mongodb": f"{local_common_path}/multi-cluster-kube-config-creator_ppc64le", + f"{s3_common_path}/kubectl-mongodb_linux_s390x/kubectl-mongodb": f"{local_common_path}/multi-cluster-kube-config-creator_s390x", + } + + +# download_plugin_for_tests_image downloads the plugin for all the architectures and places them to the paths configured in +# s3_and_local_plugin_path +def download_plugin_for_tests_image(s3_bucket: str, version: str): + try: + s3_client = boto3.client("s3", region_name=AWS_REGION) + except Exception as e: + logger.debug(f"An error occurred connecting to S3 to download kubectl plugin for tests image: {e}") + sys.exit(1) + + plugin_path = f"{S3_BUCKET_KUBECTL_PLUGIN_SUBPATH}/{version}/dist/kubectl-mongodb_linux_amd64_v1/kubectl-mongodb" + for plugin_path, local_path in s3_and_local_plugin_path(version).items(): + logger.info(f"Downloading s3://{s3_bucket}/{plugin_path} to {local_path}") + try: + s3_client.download_file(s3_bucket, plugin_path, local_path) + # change the file's permissions to make file executable + os.chmod(local_path, 0o755) + + logger.info(f"Successfully downloaded artifact to {local_path}") + except ClientError as e: + if e.response["Error"]["Code"] == "404": + logger.debug(f"ERROR: Artifact not found at s3://{s3_bucket}/{plugin_path} ") + else: + logger.debug(f"ERROR: Failed to download artifact. S3 Client Error: {e}") + sys.exit(1) + except Exception as e: + logger.debug(f"An unexpected error occurred during download: {e}") + sys.exit(1) + + +def main(): + build_scenario = os.environ.get("BUILD_SCENARIO") + kubectl_plugin_build_info = load_build_info(build_scenario).binaries[KUBECTL_PLUGIN_BINARY_NAME] + + run_goreleaser() + + version = os.environ.get("OPERATOR_VERSION") + upload_artifacts_to_s3(kubectl_plugin_build_info.s3_store, version) + + download_plugin_for_tests_image(kubectl_plugin_build_info.s3_store, version) + + +if __name__ == "__main__": + main() diff --git a/scripts/release/tests/build_info_test.py b/scripts/release/tests/build_info_test.py index 1e02535770..a2863af515 100644 --- a/scripts/release/tests/build_info_test.py +++ b/scripts/release/tests/build_info_test.py @@ -85,7 +85,7 @@ def test_load_build_info_development(): }, binaries={ "kubectl-mongodb": BinaryInfo( - s3_store="s3://kubectl-mongodb/dev", + s3_store="mongodb-kubernetes-dev", platforms=["linux/amd64"], ) }, @@ -179,7 +179,7 @@ def test_load_build_info_patch(): }, binaries={ "kubectl-mongodb": BinaryInfo( - s3_store="s3://kubectl-mongodb/dev", + s3_store="mongodb-kubernetes-dev", platforms=["linux/amd64"], ) }, @@ -293,9 +293,9 @@ def test_load_build_info_staging(): }, binaries={ "kubectl-mongodb": BinaryInfo( - s3_store="s3://kubectl-mongodb/staging", + s3_store="mongodb-kubernetes-staging", platforms=["darwin/amd64", "darwin/arm64", "linux/amd64", "linux/arm64"], - sign=True, + sign=False, ) }, helm_charts={ @@ -397,7 +397,7 @@ def test_load_build_info_release(): }, binaries={ "kubectl-mongodb": BinaryInfo( - s3_store="s3://kubectl-mongodb/prod", + s3_store="mongodb-kubernetes-release", platforms=["darwin/amd64", "darwin/arm64", "linux/amd64", "linux/arm64"], sign=True, )