Skip to content

Commit e480db7

Browse files
smarunichsmarunich
authored
support for external-dns as addon for all three cloud providers (#247)
1. The sub-domain `$cluster_name.$external_dns_$cloud_dns_zone` for example `*.aks-r161rc0p3-eastus-0.azure.sandbox.tetrate.io` 2. The credentials for service account are done under the `external-dns` module itself 3. Support for all three cloud providers 4. external-dns is implemented as addon
1 parent a32973a commit e480db7

File tree

34 files changed

+1005
-29
lines changed

34 files changed

+1005
-29
lines changed

‎Makefile‎

Lines changed: 43 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,45 @@ monitoring: ## Deploys the TSB monitoring stack
169169
cd "../.."; \
170170
'
171171

172+
.PHONY: external-dns
173+
external-dns: external-dns_gcp external-dns_aws external-dns_azure ## Deploys external-dns
174+
external-dns_%:
175+
@echo "Deploying external-dns..."
176+
@$(MAKE) $*_k8s
177+
@/bin/sh -c '\
178+
index=0; \
179+
jq -r '.$*_k8s_regions[]' terraform.tfvars.json | while read -r region; do \
180+
echo "cloud=$* region=$$region cluster_id=$$index"; \
181+
cd "addons/$*/external-dns"; \
182+
terraform workspace new $*-$$index-$$region; \
183+
terraform workspace select $*-$$index-$$region; \
184+
terraform init; \
185+
terraform apply ${terraform_apply_args} -var-file="../../../terraform.tfvars.json" -var=cloud=$* -var=cluster_id=$$index; \
186+
terraform workspace select default; \
187+
index=$$((index+1)); \
188+
cd "../.."; \
189+
done; \
190+
'
191+
192+
destroy_external-dns: destroy_external-dns_gcp destroy_external-dns_aws destroy_external-dns_azure ## Destroys external-dns
193+
destroy_external-dns_%:
194+
@echo "Deploying external-dns..."
195+
@$(MAKE) $*_k8s
196+
@/bin/sh -c '\
197+
index=0; \
198+
jq -r '.$*_k8s_regions[]' terraform.tfvars.json | while read -r region; do \
199+
echo "cloud=$* region=$$region cluster_id=$$index"; \
200+
cd "addons/$*/external-dns"; \
201+
terraform workspace new $*-$$index-$$region; \
202+
terraform workspace select $*-$$index-$$region; \
203+
terraform init; \
204+
terraform destroy ${terraform_apply_args} -var-file="../../../terraform.tfvars.json" -var=cloud=$* -var=cluster_id=$$index; \
205+
terraform workspace select default; \
206+
index=$$((index+1)); \
207+
cd "../.."; \
208+
done; \
209+
'
210+
172211
.PHONY: destroy
173212
destroy: destroy_remote destroy_local
174213

@@ -179,12 +218,14 @@ destroy_remote: ## Destroy the environment
179218
fqdn=`jq -r '.tsb_fqdn' terraform.tfvars.json`; \
180219
address=`jq -r "if .ingress_ip.value != \"\" then .ingress_ip.value else .ingress_hostname.value end" outputs/terraform_outputs/terraform-tsb-mp.json`; \
181220
cd "tsb/fqdn/$$cloud"; \
221+
terraform init; \
182222
terraform destroy ${terraform_apply_args} -var-file="../../../terraform.tfvars.json" -var=address=$$address -var=fqdn=$$fqdn; \
183223
[ $$? -ne 0 ] && exit 1; \
184224
rm -rf terraform.tfstate.d/; \
185225
rm -rf terraform.tfstate; \
186226
cd "../../.."; \
187227
'
228+
@$(MAKE) destroy_external-dns
188229
@$(MAKE) destroy_gcp destroy_aws destroy_azure
189230

190231
.PHONY: destroy_local
@@ -201,7 +242,8 @@ destroy_%:
201242
echo "cloud=$* region=$$region cluster_id=$$index"; \
202243
cd "infra/$*"; \
203244
terraform workspace select $*-$$index-$$region; \
204-
terraform destroy ${terraform_destroy_args} -var-file="../../terraform.tfvars.json" -var=$*_k8s_region=$$region -var=cluster_id=$$index; \
245+
cluster_name=`terraform output cluster_name | jq . -r`; \
246+
terraform destroy ${terraform_destroy_args} -var-file="../../terraform.tfvars.json" -var=$*_k8s_region=$$region -var=cluster_id=$$index -var=cluster_name=$$cluster_name; \
205247
[ $$? -eq 0 ] && terraform workspace select default && terraform workspace delete ${terraform_workspace_args} $*-$$index-$$region; \
206248
index=$$((index+1)); \
207249
cd "../.."; \

‎README.md‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,11 @@ The `Makefile` in this directory provides ability to fastforward to anypoint of
1919
B[make k8s] --> CC[make azure_k8s]
2020
B[make k8s] --> CCC[make gcp_k8s]
2121
C[make aws_k8s] --> D[make tsb_mp]
22+
C[make aws_k8s] --> E[make external-dns_aws]
2223
CC[make azure_k8s] --> D[make tsb_mp]
24+
CC[make azure_k8s] --> EE[make external-dns_azure]
2325
CCC[make gcp_k8s] --> D[make tsb_mp]
26+
CCC[make gcp_k8s] --> EEE[make external-dns_gcp]
2427
D[make tsb_mp] --> DD[make tsb_cp]
2528
D[make tsb_mp] --> G[make argocd]
2629
D[make tsb_mp] --> H[make monitoring]
@@ -118,13 +121,14 @@ The completion of the above steps will result in:
118121

119122
## Deployment Scenarios
120123

121-
[Infra Staging](./infra/README.md)<br>
122-
[TSB MP Fastforward](./tsb/README.md#tsb_mp)<br>
123-
[TSB CP Fastforward](./tsb/README.md#tsb_cp)<br>
124+
* [Infrastructure Staging](./infra/README.md)<br>
125+
* [TSB Management Plane Rollout](./tsb/README.md#tsb_mp)<br>
126+
* [TSB Control Plane Cluster Onboarding](./tsb/README.md#tsb_cp)<br>
124127

125-
## Use Cases
128+
## Use Cases and Addons
126129

127-
[ArgoCD GitOps](./addons/README.md#argocd)
130+
* [ArgoCD GitOps](./addons/README.md#argocd)
131+
* [external-dns](./addons/README.md#external-dns)
128132

129133
## Destroy
130134

‎addons/README.md‎

Lines changed: 77 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99

1010
## ArgoCD
1111

12-
Deploy Argo CD for gitops demo
12+
Deploys Argo CD for gitops demo
1313

1414
```bash
1515
# Deploys Argocd on all Clusters
@@ -24,7 +24,7 @@ For details about the deployed applications, take a look at the manifests in the
2424

2525
## TSB monitoring stack
2626

27-
Deploy the TSB monitoring stack to have metrics and dashboards showing the operational status
27+
Deploys the TSB monitoring stack to have metrics and dashboards showing the operational status
2828
of the different TSB components.
2929

3030
```bash
@@ -36,6 +36,81 @@ make monitoring
3636
in the `tsb-monitoring` namespace. The username is `admin` and the password can be found in the
3737
`outputs/terraform_outputs/terraform-monitoring.json` file (defaults to the `tsb_password` if set).
3838

39+
## external-dns
40+
41+
Deploys external-dns per k8s cluster, where the DNS domain equals to `$var.cluster_name`.`$var.external_dns_$cloud_dns_zone`.
42+
For example, where the cluster name is `gke-r161rc1p1-us-east1-0` and `var.external_dns_gcp_dns_zone` is set to `gcp.sandbox.tetrate.io` - the DNS domain will equal to `gke-r161rc1p1-us-east1-0.gcp.sandbox.tetrate.io`, and the sample DNS record will equal to `test3.gke-r161rc1p1-us-east1-0.gcp.sandbox.tetrate.io`.
43+
44+
### General Defaults
45+
46+
```hcl
47+
variable "external_dns_annotation_filter" {
48+
default = ""
49+
}
50+
51+
variable "external_dns_label_filter" {
52+
default = ""
53+
}
54+
55+
variable "external_dns_sources" {
56+
default = "service"
57+
}
58+
59+
variable "external_dns_interval" {
60+
default = "5s"
61+
}
62+
```
63+
64+
#### GCP Defaults
65+
66+
```hcl
67+
variable "external_dns_gcp_dns_zone" {
68+
default = "gcp.sandbox.tetrate.io"
69+
}
70+
```
71+
72+
#### Azure Defaults
73+
74+
```hcl
75+
variable "external_dns_azure_dns_zone" {
76+
default = "azure.sandbox.tetrate.io"
77+
}
78+
```
79+
80+
### Deploy
81+
82+
Based on the cloud `external_dns_$cloud_dns_zone` variable have to be set or overwritten in `terraform.tfvars` file.
83+
> NOTE: AWS does not have a default external-dns zone set.
84+
85+
terraform.tfvars.json:
86+
```json
87+
...
88+
"external_dns_aws_dns_zone": "aws.sandbox.tetrate.io"
89+
...
90+
```
91+
92+
```bash
93+
make external_dns
94+
```
95+
96+
or per cloud, for example GCP:
97+
98+
```bash
99+
make external_dns_gcp
100+
```
101+
102+
### Destroy
103+
104+
```bash
105+
make destroy_external_dns
106+
```
107+
108+
or per cloud, for example GCP:
109+
110+
```bash
111+
make destroy_external_dns_gcp
112+
```
113+
39114
### Module Overview
40115

41116
#### module.argocd (`make argocd`)

‎addons/aws/external-dns/main.tf‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
data "terraform_remote_state" "infra" {
2+
backend = "local"
3+
config = {
4+
path = "../../../infra/${var.cloud}/terraform.tfstate.d/${var.cloud}-${var.cluster_id}-${local.k8s_regions[var.cluster_id]}/terraform.tfstate"
5+
}
6+
}
7+
8+
module "external_dns" {
9+
source = "../../../modules/addons/aws/external-dns"
10+
name_prefix = "${var.name_prefix}-${var.cluster_id}"
11+
cluster_name = data.terraform_remote_state.infra.outputs.cluster_name
12+
k8s_host = data.terraform_remote_state.infra.outputs.host
13+
k8s_cluster_ca_certificate = data.terraform_remote_state.infra.outputs.cluster_ca_certificate
14+
k8s_client_token = data.terraform_remote_state.infra.outputs.token
15+
oidc_provider_arn = data.terraform_remote_state.infra.outputs.oidc_provider_arn
16+
cluster_oidc_issuer_url = data.terraform_remote_state.infra.outputs.cluster_oidc_issuer_url
17+
vpc_id = data.terraform_remote_state.infra.outputs.vpc_id
18+
region = local.k8s_regions[var.cluster_id]
19+
tags = local.default_tags
20+
dns_zone = var.external_dns_aws_dns_zone
21+
sources = var.external_dns_sources
22+
annotation_filter = var.external_dns_annotation_filter
23+
label_filter = var.external_dns_label_filter
24+
interval = var.external_dns_interval
25+
output_path = var.output_path
26+
}

‎addons/aws/external-dns/variables.tf‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
variable "cloud" {
2+
default = null
3+
}
4+
5+
variable "cluster_id" {
6+
default = null
7+
}
8+
9+
variable "name_prefix" {
10+
description = "name prefix"
11+
}
12+
13+
variable "output_path" {
14+
default = "../../../outputs"
15+
}
16+
17+
variable "tsb_image_sync_username" {
18+
}
19+
20+
variable "aws_k8s_regions" {
21+
default = []
22+
}
23+
24+
locals {
25+
k8s_regions = var.aws_k8s_regions
26+
}
27+
28+
variable "tetrate_owner" {
29+
}
30+
variable "tetrate_team" {
31+
}
32+
variable "tetrate_purpose" {
33+
default = "demo"
34+
}
35+
variable "tetrate_lifespan" {
36+
default = "oneoff"
37+
}
38+
variable "tetrate_customer" {
39+
default = "internal"
40+
}
41+
42+
locals {
43+
default_tags = {
44+
"tetrate:owner" = coalesce(var.tetrate_owner, replace(var.tsb_image_sync_username, "/\\W+/", "-"))
45+
"tetrate:team" = var.tetrate_team
46+
"tetrate:purpose" = var.tetrate_purpose
47+
"tetrate:lifespan" = var.tetrate_lifespan
48+
"tetrate:customer" = var.tetrate_customer
49+
"environment" = var.name_prefix
50+
}
51+
}
52+
53+
variable "external_dns_annotation_filter" {
54+
default = ""
55+
}
56+
57+
variable "external_dns_label_filter" {
58+
default = ""
59+
}
60+
61+
variable "external_dns_sources" {
62+
default = "service"
63+
}
64+
65+
variable "external_dns_interval" {
66+
default = "5s"
67+
}
68+
69+
variable "external_dns_aws_dns_zone" {
70+
}
71+

‎addons/azure/external-dns/main.tf‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
data "terraform_remote_state" "infra" {
2+
backend = "local"
3+
config = {
4+
path = "../../../infra/${var.cloud}/terraform.tfstate.d/${var.cloud}-${var.cluster_id}-${local.k8s_regions[var.cluster_id]}/terraform.tfstate"
5+
}
6+
}
7+
8+
provider "azurerm" {
9+
features {}
10+
11+
#https://github.com/hashicorp/terraform-provider-azurerm/issues/13776
12+
/* default_tags {
13+
tags = local.default_tags
14+
} */
15+
}
16+
17+
module "external_dns" {
18+
source = "../../../modules/addons/azure/external-dns"
19+
name_prefix = "${var.name_prefix}-${var.cluster_id}"
20+
cluster_name = data.terraform_remote_state.infra.outputs.cluster_name
21+
k8s_host = data.terraform_remote_state.infra.outputs.host
22+
k8s_cluster_ca_certificate = data.terraform_remote_state.infra.outputs.cluster_ca_certificate
23+
k8s_client_token = data.terraform_remote_state.infra.outputs.token
24+
kubelet_identity = data.terraform_remote_state.infra.outputs.kubelet_identity
25+
resource_group_name = data.terraform_remote_state.infra.outputs.resource_group_name
26+
resource_group_id = data.terraform_remote_state.infra.outputs.resource_group_id
27+
tags = local.default_tags
28+
dns_zone = var.external_dns_azure_dns_zone
29+
sources = var.external_dns_sources
30+
annotation_filter = var.external_dns_annotation_filter
31+
label_filter = var.external_dns_label_filter
32+
interval = var.external_dns_interval
33+
output_path = var.output_path
34+
}

0 commit comments

Comments
 (0)