Skip to content

Commit d1a3db6

Browse files
streamnsightstreamnsight
authored
Merge pull request #2 from oracle-quickstart/bufix/k8sprovider
Bugfix/k8sprovider
2 parents b622110 + 48701bd commit d1a3db6

File tree

10 files changed

+751
-368
lines changed

10 files changed

+751
-368
lines changed

‎.gitignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
.terraform
2+
.terraform.lock.hcl
3+
terraform.tfstate
4+
terraform.tfstate.backup

‎cluster_autoscaler.yaml‎

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
---
2+
apiVersion: v1
3+
kind: ServiceAccount
4+
metadata:
5+
labels:
6+
k8s-addon: cluster-autoscaler.addons.k8s.io
7+
k8s-app: cluster-autoscaler
8+
name: cluster-autoscaler
9+
namespace: kube-system
10+
---
11+
apiVersion: rbac.authorization.k8s.io/v1
12+
kind: ClusterRole
13+
metadata:
14+
name: cluster-autoscaler
15+
labels:
16+
k8s-addon: cluster-autoscaler.addons.k8s.io
17+
k8s-app: cluster-autoscaler
18+
rules:
19+
- apiGroups: [""]
20+
resources: ["events", "endpoints"]
21+
verbs: ["create", "patch"]
22+
- apiGroups: [""]
23+
resources: ["pods/eviction"]
24+
verbs: ["create"]
25+
- apiGroups: [""]
26+
resources: ["pods/status"]
27+
verbs: ["update"]
28+
- apiGroups: [""]
29+
resources: ["endpoints"]
30+
resourceNames: ["cluster-autoscaler"]
31+
verbs: ["get", "update"]
32+
- apiGroups: [""]
33+
resources: ["nodes"]
34+
verbs: ["watch", "list", "get", "patch", "update"]
35+
- apiGroups: [""]
36+
resources:
37+
- "pods"
38+
- "services"
39+
- "replicationcontrollers"
40+
- "persistentvolumeclaims"
41+
- "persistentvolumes"
42+
- "namespaces"
43+
verbs: ["watch", "list", "get"]
44+
- apiGroups: ["extensions"]
45+
resources: ["replicasets", "daemonsets"]
46+
verbs: ["watch", "list", "get"]
47+
- apiGroups: ["policy"]
48+
resources: ["poddisruptionbudgets"]
49+
verbs: ["watch", "list"]
50+
- apiGroups: ["apps"]
51+
resources: ["statefulsets", "replicasets", "daemonsets"]
52+
verbs: ["watch", "list", "get"]
53+
- apiGroups: ["storage.k8s.io"]
54+
resources: ["storageclasses", "csinodes", "csistoragecapacities", "csidrivers"]
55+
verbs: ["watch", "list", "get"]
56+
- apiGroups: ["batch", "extensions"]
57+
resources: ["jobs"]
58+
verbs: ["get", "list", "watch", "patch"]
59+
- apiGroups: ["coordination.k8s.io"]
60+
resources: ["leases"]
61+
verbs: ["create"]
62+
- apiGroups: ["coordination.k8s.io"]
63+
resourceNames: ["cluster-autoscaler"]
64+
resources: ["leases"]
65+
verbs: ["get", "update"]
66+
---
67+
apiVersion: rbac.authorization.k8s.io/v1
68+
kind: Role
69+
metadata:
70+
name: cluster-autoscaler
71+
namespace: kube-system
72+
labels:
73+
k8s-addon: cluster-autoscaler.addons.k8s.io
74+
k8s-app: cluster-autoscaler
75+
rules:
76+
- apiGroups: [""]
77+
resources: ["configmaps"]
78+
verbs: ["create","list","watch"]
79+
- apiGroups: [""]
80+
resources: ["configmaps"]
81+
resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"]
82+
verbs: ["delete", "get", "update", "watch"]
83+
84+
---
85+
apiVersion: rbac.authorization.k8s.io/v1
86+
kind: ClusterRoleBinding
87+
metadata:
88+
name: cluster-autoscaler
89+
labels:
90+
k8s-addon: cluster-autoscaler.addons.k8s.io
91+
k8s-app: cluster-autoscaler
92+
roleRef:
93+
apiGroup: rbac.authorization.k8s.io
94+
kind: ClusterRole
95+
name: cluster-autoscaler
96+
subjects:
97+
- kind: ServiceAccount
98+
name: cluster-autoscaler
99+
namespace: kube-system
100+
101+
---
102+
apiVersion: rbac.authorization.k8s.io/v1
103+
kind: RoleBinding
104+
metadata:
105+
name: cluster-autoscaler
106+
namespace: kube-system
107+
labels:
108+
k8s-addon: cluster-autoscaler.addons.k8s.io
109+
k8s-app: cluster-autoscaler
110+
roleRef:
111+
apiGroup: rbac.authorization.k8s.io
112+
kind: Role
113+
name: cluster-autoscaler
114+
subjects:
115+
- kind: ServiceAccount
116+
name: cluster-autoscaler
117+
namespace: kube-system
118+
119+
---
120+
apiVersion: apps/v1
121+
kind: Deployment
122+
metadata:
123+
name: cluster-autoscaler
124+
namespace: kube-system
125+
labels:
126+
app: cluster-autoscaler
127+
spec:
128+
replicas: 3
129+
selector:
130+
matchLabels:
131+
app: cluster-autoscaler
132+
template:
133+
metadata:
134+
labels:
135+
app: cluster-autoscaler
136+
annotations:
137+
prometheus.io/scrape: 'true'
138+
prometheus.io/port: '8085'
139+
spec:
140+
serviceAccountName: cluster-autoscaler
141+
containers:
142+
- image: ${image}
143+
name: cluster-autoscaler
144+
resources:
145+
limits:
146+
cpu: 100m
147+
memory: 300Mi
148+
requests:
149+
cpu: 100m
150+
memory: 300Mi
151+
command:
152+
- ./cluster-autoscaler
153+
- --v=4
154+
- --stderrthreshold=info
155+
- --cloud-provider=oci-oke
156+
- --max-node-provision-time=25m
157+
${node_pools_settings}
158+
- --scale-down-delay-after-add=10m
159+
- --scale-down-unneeded-time=10m
160+
- --unremovable-node-recheck-timeout=5m
161+
- --balance-similar-node-groups
162+
- --balancing-ignore-label=displayName
163+
- --balancing-ignore-label=hostname
164+
- --balancing-ignore-label=internal_addr
165+
- --balancing-ignore-label=oci.oraclecloud.com/fault-domain
166+
imagePullPolicy: "Always"
167+
env:
168+
- name: OKE_USE_INSTANCE_PRINCIPAL
169+
value: "true"
170+
- name: OCI_SDK_APPEND_USER_AGENT
171+
value: "oci-oke-cluster-autoscaler"
172+
---
173+
apiVersion: policy/v1
174+
kind: PodDisruptionBudget
175+
metadata:
176+
name: coredns-pdb
177+
namespace: kube-system
178+
spec:
179+
maxUnavailable: 1
180+
selector:
181+
matchLabels:
182+
k8s-app: kube-dns

‎datasources.tf‎

Lines changed: 0 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -43,36 +43,6 @@ data "oci_identity_availability_domains" "ADs" {
4343
compartment_id = var.tenancy_ocid
4444
}
4545

46-
# data "oci_containerengine_cluster_kube_config" "KubeConfig" {
47-
# cluster_id = oci_containerengine_cluster.oci_oke_cluster.id
48-
# token_version = var.cluster_kube_config_token_version
49-
# }
50-
51-
52-
# locals {
53-
# gpu = {
54-
# sources = [for s in data.oci_containerengine_node_pool_option.oci_oke_node_pool_option.sources : s if length(regexall(".*Gen2-GPU.*", s.source_name)) > 0]
55-
# shapes = {
56-
# BM = [for s in data.oci_containerengine_node_pool_option.oci_oke_node_pool_option.shapes : s if length(regexall("BM[.]GPU.*", s)) > 0]
57-
# VM = [for s in data.oci_containerengine_node_pool_option.oci_oke_node_pool_option.shapes : s if length(regexall("VM[.]GPU.*", s)) > 0]
58-
# }
59-
# }
60-
# arm = {
61-
# sources = [for s in data.oci_containerengine_node_pool_option.oci_oke_node_pool_option.sources : s if length(regexall(".*aarch64.*", s.source_name)) > 0]
62-
# shapes = {
63-
# BM = [for s in data.oci_containerengine_node_pool_option.oci_oke_node_pool_option.shapes : s if length(regexall("BM[.]Standard[.]A1.*", s)) > 0]
64-
# VM = [for s in data.oci_containerengine_node_pool_option.oci_oke_node_pool_option.shapes : s if length(regexall("VM[.]Standard[.]A1.*", s)) > 0]
65-
# }
66-
# }
67-
# x86 = {
68-
# sources = [for s in data.oci_containerengine_node_pool_option.oci_oke_node_pool_option.sources : s if length(regexall(".*(aarch64|Gen2-GPU).*", s.source_name)) == 0]
69-
# shapes = {
70-
# BM = [for s in data.oci_containerengine_node_pool_option.oci_oke_node_pool_option.shapes : s if length(regexall(".*(GPU|A1).*", s)) == 0 && length(regexall("BM.*", s)) > 0]
71-
# VM = [for s in data.oci_containerengine_node_pool_option.oci_oke_node_pool_option.shapes : s if length(regexall(".*(GPU|A1).*", s)) == 0 && length(regexall("VM.*", s)) > 0]
72-
# }
73-
# }
74-
# }
75-
7646
data "oci_limits_limit_definitions" "limit_def" {
7747
compartment_id = var.tenancy_ocid
7848
service_name = "compute"
@@ -104,22 +74,6 @@ data "oci_core_shapes" "valid_shapes" {
10474
availability_domain = data.oci_identity_availability_domains.ADs.availability_domains[count.index].name
10575
}
10676

107-
# output valid_shapes {
108-
# value = data.oci_core_shapes.valid_shapes.*.shapes
109-
# }
110-
111-
# output "shape_ad_availability" {
112-
# value = local.shape_ad_availability
113-
# }
114-
115-
# output "limits" {
116-
# value = local.availability_map
117-
# }
118-
119-
# output "limits_definitions" {
120-
# value = local.limits_definitions
121-
# }
122-
12377
resource "random_string" "deploy_id" {
12478
length = 4
12579
special = false

‎helm_cert_manager.tf‎

Lines changed: 69 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -8,24 +8,75 @@ locals {
88
enable_cert_manager = var.enable_flink ? true : var.enable_cert_manager
99
}
1010

11-
resource "helm_release" "cert_manager" {
12-
count = local.enable_cert_manager ? 1 : 0
13-
name = "cert-manager"
14-
repository = "https://charts.jetstack.io"
15-
chart = "cert-manager"
16-
version = "1.8.2"
17-
namespace = "cert-manager"
18-
create_namespace = true
19-
wait = true # wait to allow the webhook be properly configured
20-
21-
set {
22-
name = "installCRDs"
23-
value = true
11+
# resource "helm_release" "cert_manager" {
12+
# count = local.enable_cert_manager ? 1 : 0
13+
# name = "cert-manager"
14+
# repository = "https://charts.jetstack.io"
15+
# chart = "cert-manager"
16+
# version = "1.8.2"
17+
# namespace = "cert-manager"
18+
# create_namespace = true
19+
# wait = true # wait to allow the webhook be properly configured
20+
21+
# set {
22+
# name = "installCRDs"
23+
# value = true
24+
# }
25+
26+
# set {
27+
# name = "webhook.timeoutSeconds"
28+
# value = "30"
29+
# }
30+
# depends_on = [oci_containerengine_cluster.oci_oke_cluster]
31+
# }
32+
33+
34+
resource "null_resource" "cert_manager" {
35+
count = local.enable_cert_manager ? 1 : 0
36+
37+
provisioner "local-exec" {
38+
command = "mkdir -p ~/.kube/ && oci ce cluster create-kubeconfig --cluster-id $CLUSTER_ID --file ~/.kube/config --region us-sanjose-1 --token-version 2.0.0 --kube-endpoint $ENDPOINT_TYPE"
39+
40+
environment = {
41+
CLUSTER_ID = oci_containerengine_cluster.oci_oke_cluster.id
42+
ENDPOINT_TYPE = var.is_endpoint_public ? "PUBLIC_ENDPOINT" : "PRIVATE_ENDPOINT"
43+
}
44+
}
45+
46+
provisioner "local-exec" {
47+
command = "helm repo add cert-manager https://charts.jetstack.io"
48+
}
49+
50+
51+
provisioner "local-exec" {
52+
command = "kubectl create ns cert-manager"
53+
}
54+
55+
provisioner "local-exec" {
56+
command = "helm install cert-manager cert-manager/cert-manager -n cert-manager --version 1.8.2 --set installCRDs=true --set webhook.timeoutSeconds=30 --wait"
57+
}
58+
59+
provisioner "local-exec" {
60+
when = destroy
61+
command = "helm uninstall cert-manager -n cert-manager"
62+
on_failure = continue
2463
}
2564

26-
set {
27-
name = "webhook.timeoutSeconds"
28-
value = "30"
65+
provisioner "local-exec" {
66+
when = destroy
67+
command = "helm repo remove cert-manager"
68+
on_failure = continue
2969
}
30-
depends_on = [oci_containerengine_cluster.oci_oke_cluster]
31-
}
70+
71+
provisioner "local-exec" {
72+
when = destroy
73+
command = "kubectl delete ns cert-manager"
74+
on_failure = continue
75+
}
76+
77+
depends_on = [
78+
oci_containerengine_cluster.oci_oke_cluster,
79+
oci_containerengine_node_pool.oci_oke_node_pool
80+
]
81+
82+
}

0 commit comments

Comments
 (0)