diff --git a/infra/fridge/__main__.py b/infra/fridge/__main__.py index f527515a..323287ae 100644 --- a/infra/fridge/__main__.py +++ b/infra/fridge/__main__.py @@ -102,7 +102,7 @@ def patch_namespace(name: str, pss: PodSecurityStandard) -> NamespacePatch: for namespace in standard_namespaces: patch_namespace(namespace, PodSecurityStandard.RESTRICTED) -# Minio +# MinIO minio = components.ObjectStorage( "minio", args=components.ObjectStorageArgs( @@ -119,6 +119,23 @@ def patch_namespace(name: str, pss: PodSecurityStandard) -> NamespacePatch: ), ) +# MinIO configuration +minio_config = components.MinioConfigJob( + name=f"{stack_name}-minio-config-job", + args=components.MinioConfigArgs( + minio_cluster_url=minio.minio_cluster_url, + minio_credentials={ + "minio_root_user": config.require_secret("minio_root_user"), + "minio_root_password": config.require_secret("minio_root_password"), + }, + minio_tenant_ns=minio.minio_tenant_ns, + minio_tenant=minio.minio_tenant, + ), + opts=ResourceOptions( + depends_on=[minio], + ), +) + # Argo Workflows enable_sso = k8s_environment is not K8sEnvironment.K3S diff --git a/infra/fridge/components/__init__.py b/infra/fridge/components/__init__.py index 731e3e16..f45103e3 100644 --- a/infra/fridge/components/__init__.py +++ b/infra/fridge/components/__init__.py @@ -2,6 +2,7 @@ from .cert_manager import CertManager, CertManagerArgs from .container_registry import ContainerRegistry, ContainerRegistryArgs from .ingress import Ingress, IngressArgs +from .minio_config import MinioConfigJob, MinioConfigArgs from .network_policies import NetworkPolicies from .object_storage import ObjectStorage, ObjectStorageArgs from .storage_classes import StorageClasses, StorageClassesArgs diff --git a/infra/fridge/components/minio_config.py b/infra/fridge/components/minio_config.py new file mode 100644 index 00000000..2d60bd0c --- /dev/null +++ b/infra/fridge/components/minio_config.py @@ -0,0 +1,152 @@ +from pulumi import ComponentResource, Output, ResourceOptions +from pulumi_kubernetes.batch.v1 import ( + Job, + JobSpecArgs, +) +from pulumi_kubernetes.core.v1 import ( + ConfigMap, + ConfigMapVolumeSourceArgs, + ContainerArgs, + EnvVarArgs, + Namespace, + PodSpecArgs, + PodTemplateSpecArgs, + SecurityContextArgs, + VolumeMountArgs, + VolumeArgs, +) +from pulumi_kubernetes.helm.v4 import Chart +from pulumi_kubernetes.meta.v1 import ObjectMetaArgs + + +class MinioConfigArgs: + def __init__( + self, + minio_tenant_ns: Namespace, + minio_tenant: Chart, + minio_credentials: dict, + minio_cluster_url: Output[str], + ): + self.minio_cluster_url = minio_cluster_url + self.minio_credentials = minio_credentials + self.minio_tenant_ns = minio_tenant_ns + self.minio_tenant = minio_tenant + + +class MinioConfigJob(ComponentResource): + def __init__( + self, name: str, args: MinioConfigArgs, opts: ResourceOptions | None = None + ) -> None: + super().__init__("fridge:k8s:MinioConfigJob", name, {}, opts) + child_opts = ResourceOptions.merge(opts, ResourceOptions(parent=self)) + + minio_setup_sh = """ + #!/bin/sh + mc --insecure alias set "$MINIO_ALIAS" "$MINIO_URL" "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" + echo "Configuring ingress and egress buckets with anonymous S3 policies" + mc anonymous set upload "$MINIO_ALIAS/egress" + mc anonymous set download "$MINIO_ALIAS/ingress" + """ + + # Create a ConfigMap for MinIO configuration + minio_config_map = ConfigMap( + "minio-configuration", + metadata=ObjectMetaArgs( + name="minio-configuration", + namespace=args.minio_tenant_ns.metadata.name, + ), + data={ + "setup.sh": minio_setup_sh, + }, + opts=child_opts, + ) + + # Create a Job to configure MinIO + Job( + "minio-config-job", + metadata=ObjectMetaArgs( + name="minio-config-job", + namespace=args.minio_tenant_ns.metadata.name, + labels={"app": "minio-config-job"}, + ), + spec=JobSpecArgs( + backoff_limit=1, + template=PodTemplateSpecArgs( + spec=PodSpecArgs( + containers=[ + ContainerArgs( + name="minio-config-job", + image="minio/mc:latest", + command=[ + "/bin/sh", + "-c", + ], + args=[ + "/tmp/scripts/setup.sh", + ], + resources={ + "requests": { + "cpu": "100m", + "memory": "128Mi", + }, + "limits": { + "cpu": "100m", + "memory": "128Mi", + }, + }, + env=[ + EnvVarArgs(name="MC_CONFIG_DIR", value="/tmp/.mc"), + EnvVarArgs( + name="MINIO_ALIAS", + value="argoartifacts", + ), + EnvVarArgs( + name="MINIO_URL", + value=Output.concat( + "http://", args.minio_cluster_url, ":80" + ), + ), + EnvVarArgs( + name="MINIO_ROOT_USER", + value=args.minio_credentials.get( + "minio_root_user", "" + ), + ), + EnvVarArgs( + name="MINIO_ROOT_PASSWORD", + value=args.minio_credentials.get( + "minio_root_password", "" + ), + ), + ], + security_context=SecurityContextArgs( + allow_privilege_escalation=False, + capabilities={"drop": ["ALL"]}, + run_as_group=1000, + run_as_non_root=True, + run_as_user=1000, + seccomp_profile={"type": "RuntimeDefault"}, + ), + volume_mounts=[ + VolumeMountArgs( + name="minio-config-volume", + mount_path="/tmp/scripts/", + ) + ], + ) + ], + volumes=[ + VolumeArgs( + name="minio-config-volume", + config_map=ConfigMapVolumeSourceArgs( + name=minio_config_map.metadata.name, + default_mode=0o777, + ), + ) + ], + restart_policy="Never", + ), + ), + ), + opts=child_opts, + ) diff --git a/infra/fridge/components/object_storage.py b/infra/fridge/components/object_storage.py index e684d7d3..6dddcfe0 100644 --- a/infra/fridge/components/object_storage.py +++ b/infra/fridge/components/object_storage.py @@ -35,7 +35,7 @@ def __init__( super().__init__("fridge:ObjectStorage", name, {}, opts) child_opts = ResourceOptions.merge(opts, ResourceOptions(parent=self)) - minio_operator_ns = Namespace( + self.minio_operator_ns = Namespace( "minio-operator-ns", metadata=ObjectMetaArgs( name="minio-operator", @@ -44,7 +44,7 @@ def __init__( opts=child_opts, ) - minio_tenant_ns = Namespace( + self.minio_tenant_ns = Namespace( "minio-tenant-ns", metadata=ObjectMetaArgs( name="argo-artifacts", @@ -53,9 +53,9 @@ def __init__( opts=child_opts, ) - minio_operator = Chart( + self.minio_operator = Chart( "minio-operator", - namespace=minio_operator_ns.metadata.name, + namespace=self.minio_operator_ns.metadata.name, chart="operator", repository_opts=RepositoryOptsArgs( repo="https://operator.min.io", @@ -63,19 +63,19 @@ def __init__( version="7.1.1", opts=ResourceOptions.merge( child_opts, - ResourceOptions(depends_on=[minio_operator_ns]), + ResourceOptions(depends_on=[self.minio_operator_ns]), ), ) - minio_fqdn = ".".join( + self.minio_fqdn = ".".join( ( args.config.require("minio_fqdn_prefix"), args.config.require("base_fqdn"), ) ) - minio_cluster_url = pulumi.Output.concat( - "minio.", minio_tenant_ns.metadata.name, ".svc.cluster.local" + self.minio_cluster_url = pulumi.Output.concat( + "minio.", self.minio_tenant_ns.metadata.name, ".svc.cluster.local" ) minio_config_env = Output.format( @@ -85,8 +85,8 @@ def __init__( "export MINIO_ROOT_USER={2}\n" "export MINIO_ROOT_PASSWORD={3}" ), - minio_fqdn, - minio_cluster_url, + self.minio_fqdn, + self.minio_cluster_url, args.config.require_secret("minio_root_user"), args.config.require_secret("minio_root_password"), ) @@ -95,7 +95,7 @@ def __init__( "minio-env-secret", metadata=ObjectMetaArgs( name="argo-artifacts-env-configuration", - namespace=minio_tenant_ns.metadata.name, + namespace=self.minio_tenant_ns.metadata.name, ), type="Opaque", string_data={ @@ -103,13 +103,13 @@ def __init__( }, opts=ResourceOptions.merge( child_opts, - ResourceOptions(depends_on=[minio_tenant_ns]), + ResourceOptions(depends_on=[self.minio_tenant_ns]), ), ) - minio_tenant = Chart( + self.minio_tenant = Chart( "minio-tenant", - namespace=minio_tenant_ns.metadata.name, + namespace=self.minio_tenant_ns.metadata.name, chart="tenant", name="argo-artifacts", version="7.1.1", @@ -121,6 +121,8 @@ def __init__( "name": "argo-artifacts", "buckets": [ {"name": "argo-artifacts"}, + {"name": "ingress"}, + {"name": "egress"}, ], "certificate": { "requestAutoCert": "false", @@ -136,10 +138,10 @@ def __init__( }, "features": { "domains": { - "console": minio_fqdn, + "console": self.minio_fqdn, "minio": [ - Output.concat(minio_fqdn, "/api"), - minio_cluster_url, + Output.concat(self.minio_fqdn, "/api"), + self.minio_cluster_url, ], } }, @@ -169,18 +171,18 @@ def __init__( ResourceOptions( depends_on=[ minio_env_secret, - minio_operator, - minio_tenant_ns, + self.minio_operator, + self.minio_tenant_ns, ] ), ), ) - minio_ingress = Ingress( + self.minio_ingress = Ingress( "minio-ingress", metadata=ObjectMetaArgs( name="minio-ingress", - namespace=minio_tenant_ns.metadata.name, + namespace=self.minio_tenant_ns.metadata.name, annotations={ "nginx.ingress.kubernetes.io/proxy-body-size": "0", "nginx.ingress.kubernetes.io/force-ssl-redirect": "true", @@ -193,7 +195,7 @@ def __init__( ingress_class_name="nginx", rules=[ IngressRuleArgs( - host=minio_fqdn, + host=self.minio_fqdn, http={ "paths": [ { @@ -212,26 +214,24 @@ def __init__( ], tls=[ IngressTLSArgs( - hosts=[minio_fqdn], + hosts=[self.minio_fqdn], secret_name="argo-artifacts-tls", ) ], ), opts=ResourceOptions.merge( child_opts, - ResourceOptions(depends_on=[minio_tenant]), + ResourceOptions(depends_on=[self.minio_tenant]), ), ) - self.minio_fqdn = minio_fqdn - self.minio_cluster_url = minio_cluster_url self.register_outputs( { - "minio_ingress": minio_ingress, - "minio_tenant": minio_tenant, - "minio_operator": minio_operator, + "minio_ingress": self.minio_ingress, + "minio_tenant": self.minio_tenant, + "minio_operator": self.minio_operator, "minio_env_secret": minio_env_secret, - "minio_tenant_ns": minio_tenant_ns, - "minio_operator_ns": minio_operator_ns, + "minio_tenant_ns": self.minio_tenant_ns, + "minio_operator_ns": self.minio_operator_ns, } ) diff --git a/infra/fridge/k8s/cilium/minio-tenant.yaml b/infra/fridge/k8s/cilium/minio-tenant.yaml index 300332d3..06248bef 100644 --- a/infra/fridge/k8s/cilium/minio-tenant.yaml +++ b/infra/fridge/k8s/cilium/minio-tenant.yaml @@ -58,6 +58,13 @@ spec: - ports: - port: "9000" protocol: TCP + - fromEndpoints: + - matchLabels: + job-name: minio-config-job + toPorts: + - ports: + - port: "9000" + protocol: TCP egress: # Outbound to k8s DNS to look up IPs - toEndpoints: @@ -93,3 +100,39 @@ spec: - ports: - port: "8089" protocol: TCP +--- +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: minio-config-jobs + namespace: argo-artifacts +spec: + endpointSelector: + matchLabels: + job-name: minio-config-job + egress: + # Outbound to k8s DNS to look up IPs + - toEndpoints: + - matchLabels: + io.kubernetes.pod.namespace: kube-system + k8s-app: kube-dns + toPorts: + - ports: + - port: "53" + protocol: ANY + rules: + dns: + - matchName: "minio.argo-artifacts.svc.cluster.local" + - toFQDNs: + - matchName: minio.argo-artifacts.svc.cluster.local + toPorts: + - ports: + - port: "80" + # Allows the MinIO config job to connect to the MinIO tenant for configuration + - toEndpoints: + - matchLabels: + v1.min.io/tenant: argo-artifacts + toPorts: + - ports: + - port: "9000" + protocol: TCP