Some Kubernetes cases

[zc-devs]

Environment

K3s 1.33, Traefik 3.3.6

Pass-through mode

Dual-stack network.

kube-system namespace

traefik.helmchartconfig.yaml

apiVersion: helm.cattle.io/v1
kind: HelmChartConfig
metadata:
  name: traefik
  namespace: kube-system
spec:
  valuesContent: |-
    ...
    namespaceOverride: "traefik"
    ports:
      anubis:
        expose:
          default: true
        exposedPort: 81
        port: 8081
      web:
        hostPort: 80
        redirections:
          entryPoint:
            scheme: "https"
            to: "websecure"
      websecure:
        hostPort: 443
        http3:
          enabled: true
    providers:
      kubernetesCRD:
        allowCrossNamespace: true
      kubernetesIngress:
        publishedService:
          pathOverride: "traefik/traefik"
    service:
      ipFamilyPolicy: "PreferDualStack"
      type: "ClusterIP"

anubis namespace

anubis.namespace.yaml

apiVersion: v1
kind: Namespace
metadata:
  labels:
    pod-security.kubernetes.io/audit: restricted
    pod-security.kubernetes.io/enforce: restricted
    pod-security.kubernetes.io/warn: restricted
  name: anubis
  namespace: anubis

anubis-env.configmap.yaml

apiVersion: v1
data:
  DIFFICULTY: "4"
  POLICY_FNAME: /etc/anubis/botPolicies.yaml
  TARGET: http://traefik.traefik.svc.cluster.local:81
kind: ConfigMap
metadata:
  labels:
    app.kubernetes.io/component: dos-protection
    app.kubernetes.io/name: anubis
    app.kubernetes.io/part-of: anubis
  name: anubis-env
  namespace: anubis

anubis-file.configmap.yaml

apiVersion: v1
data:
  botPolicies.yaml: |-
    bots:
    - import: "(data)/bots/_deny-pathological.yaml"
    - import: "(data)/meta/ai-block-aggressive.yaml"
    - import: "(data)/crawlers/_allow-good.yaml"
    - import: "(data)/bots/aggressive-brazilian-scrapers.yaml"
    - import: "(data)/common/keep-internet-working.yaml"
    - action: "CHALLENGE"
      name: "generic-browser"
      user_agent_regex: "Mozilla|Opera"
    dnsbl: false
    status_codes:
      CHALLENGE: 200
      DENY: 200
kind: ConfigMap
metadata:
  labels:
    app.kubernetes.io/component: dos-protection
    app.kubernetes.io/name: anubis
    app.kubernetes.io/part-of: anubis
  name: anubis-file
  namespace: anubis

anubis.deployment.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app.kubernetes.io/component: dos-protection
    app.kubernetes.io/name: anubis
    app.kubernetes.io/part-of: anubis
    app.kubernetes.io/version: v1.21.3
  name: anubis
  namespace: anubis
spec:
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: anubis
      name: anubis
  template:
    metadata:
      annotations:
        anubis-file-hash: 2f0d28cad70ff9a5ae4e9cb4e7c121fb
      labels:
        app.kubernetes.io/component: dos-protection
        app.kubernetes.io/name: anubis
        app.kubernetes.io/part-of: anubis
        app.kubernetes.io/version: v1.21.3
        name: anubis
    spec:
      containers:
      - envFrom:
        - configMapRef:
            name: anubis-env
        - secretRef:
            name: anubis-env
        image: ghcr.io/techarohq/anubis:v1.21.3
        livenessProbe:
          failureThreshold: 4
          httpGet:
            path: /healthz
            port: metrics
          initialDelaySeconds: 10
          periodSeconds: 15
          successThreshold: 1
          timeoutSeconds: 1
        name: anubis
        ports:
        - containerPort: 8923
          name: http
        - containerPort: 9090
          name: metrics
        readinessProbe:
          failureThreshold: 3
          httpGet:
            path: /healthz
            port: metrics
          initialDelaySeconds: 5
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 1
        resources:
          limits:
            cpu: 1000m
            memory: 64Mi
          requests:
            cpu: 50m
            memory: 32Mi
        securityContext:
          allowPrivilegeEscalation: false
          capabilities:
            drop:
            - ALL
          readOnlyRootFilesystem: true
          seccompProfile:
            type: RuntimeDefault
        startupProbe:
          failureThreshold: 8
          httpGet:
            path: /healthz
            port: metrics
          initialDelaySeconds: 10
          periodSeconds: 15
          successThreshold: 1
          timeoutSeconds: 3
        volumeMounts:
        - mountPath: /etc/anubis
          name: anubis-file
      enableServiceLinks: false
      securityContext:
        runAsGroup: 1000
        runAsNonRoot: true
        runAsUser: 1000
        seccompProfile:
          type: RuntimeDefault
      volumes:
      - configMap:
          name: anubis-file
        name: anubis-file

anubis.service.yaml

apiVersion: v1
kind: Service
metadata:
  labels:
    app.kubernetes.io/component: dos-protection
    app.kubernetes.io/name: anubis
    app.kubernetes.io/part-of: anubis
  name: anubis
  namespace: anubis
spec:
  ipFamilyPolicy: PreferDualStack
  ports:
  - name: http
    port: 8923
    targetPort: http
  selector:
    app.kubernetes.io/name: anubis
    name: anubis

default.networkpolicy.yaml

apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: default
  namespace: anubis
spec:
  egress:
  - ports:
    - port: 53
      protocol: UDP
    - port: 53
      protocol: TCP
    to:
    - namespaceSelector:
        matchLabels:
          kubernetes.io/metadata.name: kube-system
      podSelector:
        matchLabels:
          k8s-app: kube-dns
  ingress: []
  podSelector:
    matchLabels: {}
  policyTypes:
  - Ingress
  - Egress

anubis.networkpolicy.yaml

apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  labels:
    app.kubernetes.io/component: dos-protection
    app.kubernetes.io/name: anubis
    app.kubernetes.io/part-of: anubis
  name: anubis
  namespace: anubis
spec:
  egress:
  - ports:
    - port: 8081
    to:
    - namespaceSelector:
        matchLabels:
          kubernetes.io/metadata.name: traefik
      podSelector:
        matchLabels:
          app.kubernetes.io/name: traefik
  ingress:
  - from:
    - namespaceSelector:
        matchLabels:
          kubernetes.io/metadata.name: traefik
      podSelector:
        matchLabels:
          app.kubernetes.io/name: traefik
    ports:
    - port: 8923
  podSelector:
    matchLabels:
      app.kubernetes.io/name: anubis
      name: anubis
  policyTypes:
  - Ingress
  - Egress

devops namespace

gitea-anubis.ingressroute.yaml

apiVersion: traefik.io/v1alpha1
kind: IngressRoute
metadata:
  labels:
    app.kubernetes.io/component: scm
    app.kubernetes.io/name: gitea
    app.kubernetes.io/part-of: devops
  name: gitea-anubis
  namespace: devops
spec:
  entryPoints:
  - websecure
  routes:
  - kind: Rule
    match: Host(`gitea.example.org`)
    services:
    - name: anubis
      namespace: anubis
      port: http

gitea.ingressroute.yaml

apiVersion: traefik.io/v1alpha1
kind: IngressRoute
metadata:
  labels:
    app.kubernetes.io/component: scm
    app.kubernetes.io/name: gitea
    app.kubernetes.io/part-of: devops
  name: gitea
  namespace: devops
spec:
  entryPoints:
  - anubis
  routes:
  - kind: Rule
    match: Host(`gitea.example.org`)
    services:
    - name: gitea
      port: http

gitea.service.yaml

apiVersion: v1
kind: Service
metadata:
  labels:
    app.kubernetes.io/component: scm
    app.kubernetes.io/name: gitea
    app.kubernetes.io/part-of: devops
  name: gitea
  namespace: devops
spec:
  ipFamilyPolicy: PreferDualStack
  ports:
  - name: http
    port: 3000
    targetPort: http
  selector:
    app.kubernetes.io/name: gitea
    name: gitea

Forward Auth mode

IPv6-only network.

kube-system namespace

traefik.helmchartconfig.yaml

apiVersion: helm.cattle.io/v1
kind: HelmChartConfig
metadata:
  name: traefik
  namespace: kube-system
spec:
  valuesContent: |-
    ...
    experimental:
      plugins:
        staticresponse:
          moduleName: "github.com/jdel/staticresponse"
          version: "v0.0.1"
    namespaceOverride: "traefik"
    ports:
      web:
        hostPort: 80
        redirections:
          entryPoint:
            scheme: "https"
            to: "websecure"
      websecure:
        hostPort: 443
        http3:
          enabled: true
    providers:
      kubernetesCRD:
        allowCrossNamespace: true
      kubernetesIngress:
        publishedService:
          pathOverride: "traefik/traefik"
    service:
      ipFamilyPolicy: "PreferDualStack"
      type: "ClusterIP"

anubis namespace

anubis.namespace.yaml

apiVersion: v1
kind: Namespace
metadata:
  labels:
    pod-security.kubernetes.io/audit: restricted
    pod-security.kubernetes.io/enforce: restricted
    pod-security.kubernetes.io/warn: restricted
  name: anubis
  namespace: anubis

anubis-env.configmap.yaml

apiVersion: v1
data:
  COOKIE_DOMAIN: test.example.org
  DIFFICULTY: "4"
  POLICY_FNAME: /etc/anubis/botPolicies.yaml
  PUBLIC_URL: https://anubis.test.example.org
  REDIRECT_DOMAINS: '*.test.example.org'
  SLOG_LEVEL: DEBUG
  TARGET: ' '
kind: ConfigMap
metadata:
  labels:
    app.kubernetes.io/component: dos-protection
    app.kubernetes.io/name: anubis
    app.kubernetes.io/part-of: anubis
  name: anubis-env
  namespace: anubis

anubis-file.configmap.yaml

apiVersion: v1
data:
  botPolicies.yaml: |-
    bots:
    - import: "(data)/bots/_deny-pathological.yaml"
    - import: "(data)/meta/ai-block-aggressive.yaml"
    - import: "(data)/crawlers/_allow-good.yaml"
    - import: "(data)/bots/aggressive-brazilian-scrapers.yaml"
    - import: "(data)/common/keep-internet-working.yaml"
    - action: "CHALLENGE"
      name: "generic-browser"
      user_agent_regex: "Mozilla|Opera"
    dnsbl: false
    status_codes:
      CHALLENGE: 200
      DENY: 403
kind: ConfigMap
metadata:
  labels:
    app.kubernetes.io/component: dos-protection
    app.kubernetes.io/name: anubis
    app.kubernetes.io/part-of: anubis
  name: anubis-file
  namespace: anubis

anubis.deployment.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app.kubernetes.io/component: dos-protection
    app.kubernetes.io/name: anubis
    app.kubernetes.io/part-of: anubis
    app.kubernetes.io/version: v1.22.0
  name: anubis
  namespace: anubis
spec:
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: anubis
      name: anubis
  template:
    metadata:
      annotations:
        anubis-file-hash: e17df128b552c8f6fd09c7ba6e21ad68
      labels:
        app.kubernetes.io/component: dos-protection
        app.kubernetes.io/name: anubis
        app.kubernetes.io/part-of: anubis
        app.kubernetes.io/version: v1.22.0
        name: anubis
    spec:
      containers:
      - envFrom:
        - configMapRef:
            name: anubis-env
        - secretRef:
            name: anubis-env
        image: ghcr.io/techarohq/anubis:v1.22.0
        livenessProbe:
          failureThreshold: 4
          httpGet:
            path: /healthz
            port: metrics
          initialDelaySeconds: 10
          periodSeconds: 15
          successThreshold: 1
          timeoutSeconds: 1
        name: anubis
        ports:
        - containerPort: 8923
          name: http
        - containerPort: 9090
          name: metrics
        readinessProbe:
          failureThreshold: 3
          httpGet:
            path: /healthz
            port: metrics
          initialDelaySeconds: 5
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 1
        resources:
          limits:
            cpu: 1000m
            memory: 256Mi
          requests:
            cpu: 50m
            memory: 64Mi
        securityContext:
          allowPrivilegeEscalation: false
          capabilities:
            drop:
            - ALL
          readOnlyRootFilesystem: true
          seccompProfile:
            type: RuntimeDefault
        startupProbe:
          failureThreshold: 8
          httpGet:
            path: /healthz
            port: metrics
          initialDelaySeconds: 10
          periodSeconds: 15
          successThreshold: 1
          timeoutSeconds: 3
        volumeMounts:
        - mountPath: /etc/anubis
          name: anubis-file
      enableServiceLinks: false
      securityContext:
        runAsGroup: 1000
        runAsNonRoot: true
        runAsUser: 1000
        seccompProfile:
          type: RuntimeDefault
      volumes:
      - configMap:
          name: anubis-file
        name: anubis-file

anubis.service.yaml

apiVersion: v1
kind: Service
metadata:
  labels:
    app.kubernetes.io/component: dos-protection
    app.kubernetes.io/name: anubis
    app.kubernetes.io/part-of: anubis
  name: anubis
  namespace: anubis
spec:
  ipFamilyPolicy: SingleStack
  ports:
  - name: http
    port: 8923
    targetPort: http
  selector:
    app.kubernetes.io/name: anubis
    name: anubis

anubis.ingressroute.yaml

apiVersion: traefik.io/v1alpha1
kind: IngressRoute
metadata:
  labels:
    app.kubernetes.io/component: dos-protection
    app.kubernetes.io/name: anubis
    app.kubernetes.io/part-of: anubis
  name: anubis
  namespace: anubis
spec:
  entryPoints:
  - websecure
  routes:
  - kind: Rule
    match: Host(`anubis.test.example.org`)
    services:
    - name: anubis
      port: http

www namespace

www.namespace.yaml

apiVersion: v1
kind: Namespace
metadata:
  labels:
    pod-security.kubernetes.io/audit: restricted
    pod-security.kubernetes.io/enforce: restricted
    pod-security.kubernetes.io/warn: restricted
  name: www
  namespace: www

anubis-forward-auth.middleware.yaml

apiVersion: traefik.io/v1alpha1
kind: Middleware
metadata:
  name: anubis-forward-auth
  namespace: www
spec:
  forwardAuth:
    address: http://anubis.anubis.svc.cluster.local:8923/.within.website/x/cmd/anubis/api/check

anubis-test.middleware.yaml

apiVersion: traefik.io/v1alpha1
kind: Middleware
metadata:
  labels:
    app.kubernetes.io/name: anubis-test
  name: anubis-test
  namespace: www
spec:
  plugin:
    staticresponse:
      Body: This is fine
      StatusCode: 200

anubis-test.ingressroute.yaml

apiVersion: traefik.io/v1alpha1
kind: IngressRoute
metadata:
  labels:
    app.kubernetes.io/name: anubis-test
  name: anubis-test
  namespace: www
spec:
  entryPoints:
  - websecure
  routes:
  - kind: Rule
    match: Host(`a.test.example.org`)
    middlewares:
    - name: anubis-forward-auth
    - name: anubis-test
    services:
    - kind: TraefikService
      name: noop@internal

[drewdomi]

When it says "Forward Auth mode IPv6-only network." this means that I can't use in dual stack??

[zc-devs]

No-no. It is just my test environment, from which took the configs. I noted this because I use ipFamilyPolicy: SingleStack there. Similarly for the first example with Dual-stack and ipFamilyPolicy: PreferDualStack.
Today I rolled out Forward Auth mode to prod with Dual-stack. It works like a charm.

[drewdomi]

Ahhh ok, I understand now.

[zc-devs]

There have to be anubis-forward-auth in anubis-test IngressRoute of course. I've updated the original post.

[zc-devs]

In Forward Auth mode at least DENY status code need to be 4xx. I've updated the original post with DENY: 403.

In pass-through mode it might be 2xx.

---

From Forward Auth docs:

If the service answers with a 2XX code, access is granted, and the original request is performed.

So, if we have

    status_codes:
      CHALLENGE: 200
      DENY: 200

and perform, for example

curl -v --user-agent "Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)" https://defended-service.example.org

Anubis "explicit deny", but returns HTTP 200, then Traefik passes request to the service, which is wrong.