[CI] Attach pod disruption budgets to runner pods (#523)

This patch adds some pod disruption budgets to runner pods that just
sets the minimum number of available pods to the maximum. This ensure
that the number of pods that k8s calculates can be disrupted is zero.
This means that when GKE is updating the node pool, it must wait an hour
before forcibly evicting the pod, giving it time to finish. Before this,
when GKE wanted to upgrade a node, it would forcibly evict the pod very
quickly (theoretically after the grace period which has a default of
30s) not realizing it is stateful.

Author: boomanaiden154

premerge/pod_disruption_budget.yaml

@@ -0,0 +1,10 @@
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: runner-set-pdb
+  namespace: ${ runner_set_name }
+spec:
+  minAvailable: ${ min_pod_count }
+  selector:
+    matchLabels:
+      actions.github.com/scale-set-name: ${ runner_set_name }

premerge/premerge_resources/main.tf

@@ -258,6 +258,36 @@ resource "kubernetes_service_account" "windows_2022_object_cache_ksa" {
   depends_on = [kubernetes_namespace.llvm_premerge_windows_2022_runners]
 }
 
+# We set up pod disruption budgets here. We need one per namespace and we need
+# to set the min pod count to the maximum number of runner pods that can
+# possibly exist so we never have a number of disruptible pods greater than
+# zero.
+
+resource "kubernetes_manifest" "linux_runners_disruption_budget" {
+  manifest   = yamldecode(templatefile("pod_disruption_budget.yaml", { runner_set_name : "llvm-premerge-linux-runners", min_pod_count : 16 }))
+  depends_on = [kubernetes_namespace.llvm_premerge_linux_runners]
+}
+
+resource "kubernetes_manifest" "windows_2022_runners_disruption_budget" {
+  manifest   = yamldecode(templatefile("pod_disruption_budget.yaml", { runner_set_name : "llvm-premerge-windows-2022-runners", min_pod_count : 16 }))
+  depends_on = [kubernetes_namespace.llvm_premerge_linux_runners]
+}
+
+resource "kubernetes_manifest" "libcxx_runners_disruption_budget" {
+  manifest   = yamldecode(templatefile("pod_disruption_budget.yaml", { runner_set_name : "llvm-premerge-libcxx-runners", min_pod_count : 32 }))
+  depends_on = [kubernetes_namespace.llvm_premerge_linux_runners]
+}
+
+resource "kubernetes_manifest" "libcxx_release_runners_disruption_budget" {
+  manifest   = yamldecode(templatefile("pod_disruption_budget.yaml", { runner_set_name : "llvm-premerge-libcxx-release-runners", min_pod_count : 32 }))
+  depends_on = [kubernetes_namespace.llvm_premerge_linux_runners]
+}
+
+resource "kubernetes_manifest" "libcxx_next_runners_disruption_budget" {
+  manifest   = yamldecode(templatefile("pod_disruption_budget.yaml", { runner_set_name : "llvm-premerge-libcxx-next-runners", min_pod_count : 32 }))
+  depends_on = [kubernetes_namespace.llvm_premerge_linux_runners]
+}
+
 resource "kubernetes_namespace" "grafana" {
   metadata {
     name = "grafana"