Recreate kind cluster on every evg host reboot (#32)

# Summary

Evergreen hosts reboot every day or weekend (depending on your
configuration). After every reboot, inter-cluster connectivity might be
broken for some reason. Recreating the clusters is the only solution we
have so far.
This is a systemd service that runs on every boot and recreates all
clusters (including the kind-kind for single cluster tests).
Our tunnel command will now also get the kubeconfig from the host
otherwise the tunnel won't open to the (new) ports of the recreated
clusters.

The systemd service is not created by default. This needs to be
explicitly done by running
```
evg_host.sh configure --auto-reboot
```

Remove the architecture flag, and instead left it to be inferred from
`uname`.

## Proof of Work

Tested locally, but it would be nice if someone can checkout this branch
and try it for themselves.

## Checklist
- [ ] Have you linked a jira ticket and/or is the ticket in the title?
- [ ] Have you checked whether your jira ticket required DOCSP changes?
- [ ] Have you checked for release_note changes?

Author: lucian-tosa

scripts/dev/evg_host.sh

@@ -39,14 +39,24 @@ kubeconfig_path="${HOME}/.operator-dev/evg-host.kubeconfig"
 
 configure() {
   shift 1
-  arch=${1-"amd64"}
+  auto_recreate="false"
+
+  # Parse arguments
+  while [[ $# -gt 0 ]]; do
+    case $1 in
+      --auto-recreate)
+        auto_recreate="true"
+        shift
+        ;;
+      *)
+        echo "Unknown argument: $1"
+        echo "Usage: configure [--auto-recreate]"
+        exit 1
+        ;;
+    esac
+  done
 
-  echo "Configuring EVG host ${EVG_HOST_NAME} (${host_url}) with architecture ${arch}"
-
-  if [[ "${cmd}" == "configure" && "${arch}" != "amd64" && "${arch}" != "arm64" ]]; then
-    echo "'configure' command supports the following architectures: 'amd64', 'arm64'"
-    exit 1
-  fi
+  echo "Configuring EVG host ${EVG_HOST_NAME} (${host_url}) (auto_recreate: ${auto_recreate})"
 
   ssh -T -q "${host_url}" "sudo chown ubuntu:ubuntu ~/.docker || true; mkdir -p ~/.docker"
   if [[ -f "${HOME}/.docker/config.json" ]]; then
@@ -56,7 +66,7 @@ configure() {
 
   sync
 
-  ssh -T -q "${host_url}" "cd ~/mongodb-kubernetes; scripts/dev/switch_context.sh root-context; scripts/dev/setup_evg_host.sh ${arch}"
+  ssh -T -q "${host_url}" "cd ~/mongodb-kubernetes; scripts/dev/switch_context.sh root-context; scripts/dev/setup_evg_host.sh ${auto_recreate}"
 }
 
 sync() {
@@ -100,7 +110,7 @@ get-kubeconfig() {
 
 recreate-kind-clusters() {
   DELETE_KIND_NETWORK=${DELETE_KIND_NETWORK:-"false"}
-  configure "${1-"amd64"}" 2>&1| prepend "evg_host.sh configure"
+  configure 2>&1| prepend "evg_host.sh configure"
   echo "Recreating kind clusters on ${EVG_HOST_NAME} (${host_url})..."
   # shellcheck disable=SC2088
   ssh -T "${host_url}" "cd ~/mongodb-kubernetes; DELETE_KIND_NETWORK=${DELETE_KIND_NETWORK} scripts/dev/recreate_kind_clusters.sh"
@@ -111,7 +121,7 @@ recreate-kind-clusters() {
 recreate-kind-cluster() {
   shift 1
   cluster_name=$1
-  configure "${1-"amd64"}" 2>&1| prepend "evg_host.sh configure"
+  configure 2>&1| prepend "evg_host.sh configure"
   echo "Recreating kind cluster ${cluster_name} on ${EVG_HOST_NAME} (${host_url})..."
   # shellcheck disable=SC2088
   ssh -T "${host_url}" "cd ~/mongodb-kubernetes; scripts/dev/recreate_kind_cluster.sh ${cluster_name}"
@@ -121,6 +131,7 @@ recreate-kind-cluster() {
 
 tunnel() {
   shift 1
+  get-kubeconfig
   # shellcheck disable=SC2016
   api_servers=$(yq '.contexts[].context.cluster as $cluster | .clusters[] | select(.name == $cluster).cluster.server' < "${kubeconfig_path}" | sed 's/https:\/\///g')
   echo "Extracted the following API server urls from ${kubeconfig_path}: ${api_servers}"
@@ -187,7 +198,7 @@ PREREQUISITES:
 
 COMMANDS:
   recreate-kind-clusters                all-you-need to configure host and kind clusters; deletes and recreates all kind clusters (for single and multi runs)
-  configure <architecture>              installs on a host: calls sync, switches context, installs necessary software
+  configure [--auto-recreate]           installs on a host: calls sync, switches context, installs necessary software
   sync                                  rsync of project directory
   recreate-kind-cluster test-cluster    executes scripts/dev/recreate_kind_cluster.sh test-cluster and executes get-kubeconfig
   remote-prepare-local-e2e-run          executes prepare-local-e2e on the remote evg host
@@ -202,7 +213,7 @@ COMMANDS:
 
 case ${cmd} in
 configure) configure "$@" ;;
-recreate-kind-clusters) recreate-kind-clusters "${1-"amd64"}";;
+recreate-kind-clusters) recreate-kind-clusters "$@" ;;
 recreate-kind-cluster) recreate-kind-cluster "$@" ;;
 get-kubeconfig) get-kubeconfig ;;
 remote-prepare-local-e2e-run) remote-prepare-local-e2e-run ;;

scripts/dev/kindclusters.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=Recreate kind clusters at boot
+After=docker.service
+Requires=docker.service
+
+[Service]
+Type=oneshot
+WorkingDirectory=/home/ubuntu/mongodb-kubernetes
+Environment=HOME=/home/ubuntu
+ExecStart=/home/ubuntu/mongodb-kubernetes/scripts/dev/recreate_kind_clusters.sh
+RemainAfterExit=yes
+
+[Install]
+WantedBy=multi-user.target

scripts/dev/setup_evg_host.sh

@@ -4,6 +4,8 @@
 
 set -Eeou pipefail
 
+source scripts/funcs/install
+
 set_limits() {
   echo "Increasing fs.inotify.max_user_instances"
   sudo sysctl -w fs.inotify.max_user_instances=8192
@@ -26,16 +28,26 @@ set_limits() {
 EOF
 }
 
-# retrieve arch variable off the shell command line
-ARCH=${1-"amd64"}
+set_auto_recreate() {
+  echo "Creating systemd service for recreating kind clusters on reboot"
+
+  sudo cp /home/ubuntu/mongodb-kubernetes/scripts/dev/kindclusters.service /etc/systemd/system/kindclusters.service
+  sudo systemctl enable kindclusters.service
+}
+
+# Detect architecture from the environment
+ARCH=$(detect_architecture)
+echo "Detected architecture: ${ARCH}"
 
 download_kind() {
   scripts/evergreen/setup_kind.sh /usr/local
 }
 
-download_curl() {
-  echo "Downloading curl..."
-  curl -s -o kubectl -L https://dl.k8s.io/release/"$(curl -L -s https://dl.k8s.io/release/stable.txt)"/bin/linux/"${ARCH}"/kubectl
+download_kubectl() {
+  kubectl_version=$(curl --retry 5 -Ls https://dl.k8s.io/release/stable.txt)
+  echo "Downloading kubectl ${kubectl_version}..."
+
+  curl --retry 5 -LOs "https://dl.k8s.io/release/${kubectl_version}/bin/linux/${ARCH}/kubectl"
   chmod +x kubectl
   sudo mv kubectl /usr/local/bin/kubectl
 }
@@ -51,7 +63,12 @@ download_helm() {
 
 set_limits
 download_kind &
-download_curl &
+download_kubectl &
 download_helm &
 
+AUTO_RECREATE=${1:-false}
+if [[ "${AUTO_RECREATE}" == "true" ]]; then
+  set_auto_recreate &
+fi
+
 wait

scripts/evergreen/setup_kind.sh

@@ -15,7 +15,7 @@ latest_version="v0.29.0"
 if [[ "${arch_suffix}" == "amd64" || "${arch_suffix}" == "arm64" ]]; then
   mkdir -p "${PROJECT_DIR}/bin/"
   echo "Saving kind to ${PROJECT_DIR}/bin"
-  curl --retry 3 --silent -L "https://github.com/kubernetes-sigs/kind/releases/download/${latest_version}/kind-${os}-${arch_suffix}" -o kind
+  curl --retry 5 -L "https://github.com/kubernetes-sigs/kind/releases/download/${latest_version}/kind-${os}-${arch_suffix}" -o kind
 
   chmod +x kind
   sudo mv kind "${PROJECT_DIR}/bin"

scripts/evergreen/setup_kubectl.sh

@@ -12,8 +12,10 @@ bindir="${PROJECT_DIR}/bin"
 tmpdir="${PROJECT_DIR}/tmp"
 mkdir -p "${bindir}" "${tmpdir}"
 
-echo "Downloading latest kubectl for ${ARCH}"
-curl -s --retry 3 -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/${ARCH}/kubectl"
+kubectl_version=$(curl --retry 5 -Ls https://dl.k8s.io/release/stable.txt)
+echo "Downloading kubectl ${kubectl_version} for ${ARCH}"
+
+curl --retry 5 -LOs "https://dl.k8s.io/release/${kubectl_version}/bin/linux/${ARCH}/kubectl"
 chmod +x kubectl
 echo "kubectl version --client"
 ./kubectl version --client