Merge remote-tracking branch 'origin/search/public-preview' into fealebenpae/enterprise-search

# Conflicts:
#	.evergreen-tasks.yml
#	controllers/operator/mongodbsearch_controller.go
#	controllers/search_controller/mongodbsearch_reconcile_helper.go
#	controllers/search_controller/mongodbsearch_reconcile_helper_test.go
#	controllers/search_controller/search_construction.go
#	mongodb-community-operator/controllers/replica_set_controller.go

Author: fealebenpae

.evergreen-functions.yml

@@ -51,10 +51,22 @@ functions:
 
   ### Setup Functions ###
 
+  setup_jq: &setup_jq
+    command: subprocess.exec
+    type: setup
+    params:
+      <<: *e2e_include_expansions_in_env
+      add_to_path:
+        - ${workdir}/bin
+      working_dir: src/github.com/mongodb/mongodb-kubernetes
+      binary: scripts/evergreen/setup_jq.sh
+
   setup_context: &setup_context # Running the first switch is important to fill the workdir and other important initial env vars
     command: shell.exec
     type: setup
     params:
+      add_to_path:
+        - ${workdir}/bin
       shell: bash
       working_dir: src/github.com/mongodb/mongodb-kubernetes
       <<: *e2e_include_expansions_in_env
@@ -103,6 +115,7 @@ functions:
       type: setup
       params:
         command: "git config --global user.email '[email protected]'"
+    - *setup_jq # we need jq in the context
     - *setup_context
 
   setup_kubectl: &setup_kubectl
@@ -112,13 +125,6 @@ functions:
       working_dir: src/github.com/mongodb/mongodb-kubernetes
       binary: scripts/evergreen/setup_kubectl.sh
 
-  setup_jq: &setup_jq
-    command: subprocess.exec
-    type: setup
-    params:
-      working_dir: src/github.com/mongodb/mongodb-kubernetes
-      binary: scripts/evergreen/setup_jq.sh
-
   setup_shellcheck:
     command: subprocess.exec
     type: setup
@@ -225,7 +231,7 @@ functions:
       working_dir: src/github.com/mongodb/mongodb-kubernetes
       add_to_path:
         - ${workdir}/bin
-      binary: scripts/dev/configure_docker_auth.sh
+      binary: scripts/dev/configure_container_auth.sh
 
   setup_evg_host: &setup_evg_host
     command: subprocess.exec
@@ -256,16 +262,35 @@ functions:
   # Configures docker authentication to ECR and RH registries.
   setup_building_host:
     - *switch_context
+    - *python_venv
     - *setup_aws
-    - *configure_docker_auth
     - *setup_evg_host
-    - *python_venv
+    - *configure_docker_auth
 
-  prune_docker_resources:
+  # This differs for normal evg_host as we require minikube instead of kind for
+  # IBM machines also install aws cli via pip instead and use podman
+  setup_building_host_minikube:
+    - *switch_context
     - command: subprocess.exec
       type: setup
       params:
-        command: "docker system prune -a -f"
+        working_dir: src/github.com/mongodb/mongodb-kubernetes
+        add_to_path:
+          - ${workdir}/bin
+        command: scripts/evergreen/setup_minikube_host.sh
+
+  prune_docker_resources:
+    - command: shell.exec
+      type: setup
+      params:
+        shell: bash
+        script: |
+          if command -v docker >/dev/null 2>&1; then
+            echo "Docker found, pruning docker resources..."
+            docker system prune -a -f
+          else
+            echo "Docker not found, skipping docker resource pruning"
+          fi
 
   # the task configures the set of tools necessary for any task working with K8 cluster:
   # installs kubectl, jq, kind (if necessary), configures docker authentication
@@ -327,20 +352,19 @@ functions:
         shell: bash
         working_dir: src/github.com/mongodb/mongodb-kubernetes
         script: |
-          source .generated/context.export.env
-          scripts/evergreen/e2e/setup_cloud_qa.py create
+          scripts/dev/run_python.sh scripts/evergreen/e2e/setup_cloud_qa.py create
     # The additional switch is needed, since we now have created the needed OM exports.
     - *switch_context
 
   teardown_cloud_qa:
     - command: shell.exec
       type: setup
       params:
+        continue_on_err: true
         shell: bash
         working_dir: src/github.com/mongodb/mongodb-kubernetes
         script: |
-          source .generated/context.export.env
-          scripts/evergreen/e2e/setup_cloud_qa.py delete
+          scripts/dev/run_python.sh scripts/evergreen/e2e/setup_cloud_qa.py delete
 
   dump_diagnostic_information_from_all_namespaces:
     - command: subprocess.exec
@@ -392,7 +416,7 @@ functions:
         add_to_path:
           - ${workdir}/bin
         working_dir: src/github.com/mongodb/mongodb-kubernetes
-        binary: scripts/evergreen/run_python.sh scripts/update_supported_dockerfiles.py
+        binary: scripts/dev/run_python.sh scripts/update_supported_dockerfiles.py
     - command: subprocess.exec
       type: setup
       params:
@@ -420,6 +444,7 @@ functions:
   upload_e2e_logs:
     - command: s3.put
       params:
+        continue_on_err: true
         aws_key: ${enterprise_aws_access_key_id}
         aws_secret: ${enterprise_aws_secret_access_key}
         local_files_include_filter:
@@ -486,7 +511,7 @@ functions:
         include_expansions_in_env:
           - image_version
           - rh_pyxis
-        binary: scripts/evergreen/run_python.sh scripts/preflight_images.py --image ${image_name} --submit "${preflight_submit}"
+        binary: scripts/dev/run_python.sh scripts/preflight_images.py --image ${image_name} --submit "${preflight_submit}"
 
   build_multi_cluster_binary:
     - command: subprocess.exec
@@ -504,7 +529,31 @@ functions:
           - ${workdir}/bin
           - ${workdir}
 
+  build_test_image_ibm:
+    - *switch_context
+    - command: subprocess.exec
+      params:
+        shell: bash
+        working_dir: src/github.com/mongodb/mongodb-kubernetes
+        include_expansions_in_env:
+          - version_id
+        add_to_path:
+          - ${workdir}/bin
+        binary: scripts/evergreen/e2e/build_tests_image_ibm.sh
+
   pipeline:
+    - *switch_context
+    - command: subprocess.exec
+      retry_on_failure: true
+      type: setup
+      params:
+        shell: bash
+        <<: *e2e_include_expansions_in_env
+        working_dir: src/github.com/mongodb/mongodb-kubernetes
+        binary: scripts/dev/run_python.sh scripts/release/pipeline_main.py --parallel ${image_name} ${all_agents} ${build_scenario}
+
+  # TODO: CLOUDP-335471 ; once all image builds are made with the new atomic pipeline, remove the following function
+  legacy_pipeline:
     - *switch_context
     - command: shell.exec
       type: setup
@@ -538,7 +587,7 @@ functions:
         shell: bash
         <<: *e2e_include_expansions_in_env
         working_dir: src/github.com/mongodb/mongodb-kubernetes
-        binary: scripts/evergreen/run_python.sh pipeline.py --include ${image_name} --parallel --sign
+        binary: scripts/dev/run_python.sh pipeline.py --include ${image_name} --parallel --sign
 
   teardown_cloud_qa_all:
     - *switch_context
@@ -549,7 +598,7 @@ functions:
         working_dir: src/github.com/mongodb/mongodb-kubernetes
         script: |
           source .generated/context.export.env
-          scripts/evergreen/run_python.sh scripts/evergreen/e2e/setup_cloud_qa.py delete_all
+          scripts/dev/run_python.sh scripts/evergreen/e2e/setup_cloud_qa.py delete_all
 
   # Updates current expansions with variables from release.json file.
   # Use e.g. ${mongoDbOperator} afterwards.
@@ -596,7 +645,7 @@ functions:
         add_to_path:
           - ${workdir}/bin
         # Below script deletes agent images created for an Evergreen patch older than 1 day
-        command: scripts/evergreen/run_python.sh scripts/evergreen/periodic-cleanup-aws.py
+        command: scripts/dev/run_python.sh scripts/evergreen/periodic-cleanup-aws.py
 
   ### Test Functions ###
 
@@ -695,7 +744,7 @@ functions:
         working_dir: src/github.com/mongodb/mongodb-kubernetes
         script: |
           source .generated/context.export.env
-          scripts/evergreen/run_python.sh scripts/evergreen/e2e/performance/create_variants.py ${variant} ${size}> evergreen_tasks.json
+          scripts/dev/run_python.sh scripts/evergreen/e2e/performance/create_variants.py ${variant} ${size}> evergreen_tasks.json
           echo "tasks to run:"
           cat evergreen_tasks.json
     - command: generate.tasks

.evergreen-periodic-builds.yaml

@@ -19,12 +19,6 @@ variables:
       - func: switch_context
 
 tasks:
-  - name: periodic_build_operator
-    commands:
-      - func: pipeline
-        vars:
-          image_name: operator-daily
-
   - name: periodic_teardown_aws
     commands:
       - func: cleanup_aws
@@ -33,134 +27,7 @@ tasks:
     commands:
       - func: teardown_cloud_qa_all
 
-  - name: periodic_build_init_appdb
-    commands:
-      - func: pipeline
-        vars:
-          image_name: init-appdb-daily
-
-  - name: periodic_build_init_database
-    commands:
-      - func: pipeline
-        vars:
-          image_name: init-database-daily
-
-  - name: periodic_build_init_opsmanager
-    commands:
-      - func: pipeline
-        vars:
-          image_name: init-ops-manager-daily
-
-  - name: periodic_build_database
-    commands:
-      - func: pipeline
-        vars:
-          image_name: database-daily
-
-  - name: periodic_build_sbom_cli
-    commands:
-      - func: pipeline
-        vars:
-          image_name: cli
-
-  - name: periodic_build_ops_manager_6
-    commands:
-      - func: pipeline
-        vars:
-          image_name: ops-manager-6-daily
-
-  - name: periodic_build_ops_manager_7
-    commands:
-      - func: pipeline
-        vars:
-          image_name: ops-manager-7-daily
-
-  - name: periodic_build_ops_manager_8
-    commands:
-      - func: pipeline
-        vars:
-          image_name: ops-manager-8-daily
-
-  # the periodic agent builds are more commented in the pipeline.py file.
-  # The gist is - we want to split up the periodic build on as many machines as possible
-  # To speed up the builds as we have too many agents due to the matrix build.
-  # For now its one without operator suffix and the last 3. This only works as long as we
-  # only have operator versions we support (minor version), as soon as we have multiple patch versions -
-  # this won't work anymore and we will need a dynamic solution.
-  - name: periodic_build_agent
-    exec_timeout_secs: 43200
-    commands:
-      - func: enable_QEMU
-      - func: pipeline
-        vars:
-          image_name: mongodb-agent-daily
-
-  - name: periodic_build_agent_1
-    exec_timeout_secs: 43200
-    commands:
-      - func: enable_QEMU
-      - func: pipeline
-        vars:
-          image_name: mongodb-agent-1-daily
-
-  - name: periodic_build_agent_2
-    exec_timeout_secs: 43200
-    commands:
-      - func: enable_QEMU
-      - func: pipeline
-        vars:
-          image_name: mongodb-agent-2-daily
-
-  - name: periodic_build_agent_3
-    exec_timeout_secs: 43200
-    commands:
-      - func: enable_QEMU
-      - func: pipeline
-        vars:
-          image_name: mongodb-agent-3-daily
-
-  #TODO should we still build the community operator?
-  - name: periodic_build_community_operator
-    commands:
-      - func: enable_QEMU
-      - func: pipeline
-        vars:
-          image_name: mongodb-kubernetes-operator-daily
-
-  - name: periodic_build_readiness_probe
-    commands:
-      - func: pipeline
-        vars:
-          image_name: readinessprobe-daily
-
-  - name: periodic_build_version_upgrade_post_start_hook
-    commands:
-      - func: pipeline
-        vars:
-          image_name: operator-version-upgrade-post-start-hook-daily
-
 task_groups:
-  - name: periodic_build_task_group
-    max_hosts: -1
-    <<: *setup_group
-    tasks:
-      - periodic_build_operator
-      - periodic_build_readiness_probe
-      - periodic_build_version_upgrade_post_start_hook
-      - periodic_build_init_appdb
-      - periodic_build_init_database
-      - periodic_build_init_opsmanager
-      - periodic_build_ops_manager_6
-      - periodic_build_ops_manager_7
-      - periodic_build_ops_manager_8
-      - periodic_build_database
-      - periodic_build_community_operator
-      - periodic_build_sbom_cli
-      - periodic_build_agent
-      - periodic_build_agent_1
-      - periodic_build_agent_2
-      - periodic_build_agent_3
-
   - name: periodic_teardown_task_group
     <<: *setup_group
     tasks:
@@ -175,22 +42,3 @@ buildvariants:
       - ubuntu2204-small
     tasks:
       - name: periodic_teardown_task_group
-
-  - name: periodic_build
-    display_name: periodic_build
-    tags: [ "periodic_build" ]
-    run_on:
-      - release-ubuntu2204-large # This is required for CISA attestation https://jira.mongodb.org/browse/DEVPROD-17780
-    tasks:
-      - name: periodic_build_task_group
-
-  - name: preflight_release_images_check_only
-    display_name: preflight_release_images_check_only
-    tags: [ "periodic_build" ]
-    depends_on:
-      - name: "*"
-        variant: periodic_build
-    run_on:
-      - rhel90-large
-    tasks:
-      - name: preflight_images_task_group