Merge pull request #17 from sayar/logs

Add Kafka logs in file - Updating sample manifests to utilize helm charts

Author: edaena

component.yaml

@@ -16,8 +16,13 @@ subcomponents:
     source: "https://github.com/microsoft/fabrikate-definitions.git"
     path: "definitions/fabrikate-prometheus-grafana"
     method: "git"
+  - name: "kafka-cluster"
+    type: "helm"
+    source: ""
+    path: "/helm-chart/kafka-cluster"
+    method: "local"
   # - name: "portworx"
   #   type: helm
   #   source: https://github.com/portworx/helm
   #   method: git
-  #   path: charts/portworx
+  #   path: charts/portworx

helm-chart/kafka-cluster/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+appVersion: "1.0"
+description: Fabrikate-Kafka is stateful workload example with backed virtual storage on a single Kubernetes cluster.
+name: kafka-cluster
+version: 0.1.0

manifests/kafka-cluster.yaml renamed to helm-chart/kafka-cluster/templates/kafka-cluster.yaml

@@ -7,6 +7,27 @@ spec:
   kafka:
     version: 2.2.1
     replicas: 3
+    logging:
+      type: inline
+      loggers:
+        kafka.root.logger.level: INFO, kafkaAppender
+        log4j.logger.org.I0Itec.zkclient.ZkClient: INFO
+        log4j.logger.org.apache.zookeeper: INFO
+        log4j.logger.kafka: INFO
+        log4j.logger.org.apache.kafka: INFO
+        log4j.logger.kafka.request.logger: INFO
+        log4j.logger.kafka.network.Processor: INFO
+        log4j.logger.kafka.server.KafkaApis: INFO
+        log4j.logger.kafka.controller: INFO
+        log4j.logger.kafka.log.LogCleaner: INFO
+        log4j.logger.state.change.logger: INFO
+        log4j.logger.kafka.authorizer.logger: INFO
+        # log4j server.log
+        log4j.appender.kafkaAppender: org.apache.log4j.DailyRollingFileAppender
+        log4j.appender.kafkaAppender.DatePattern: "'.'yyyy-MM-dd-HH"
+        log4j.appender.kafkaAppender.File: {{ .Values.kafkaLogDirs }}/server.log
+        log4j.appender.kafkaAppender.layout: org.apache.log4j.PatternLayout
+        log4j.appender.kafkaAppender.layout.ConversionPattern: '[%d] %p %m (%c)%n'
     listeners:
       external:
         type: loadbalancer

manifests/kafka-connect-user.yaml renamed to helm-chart/kafka-cluster/templates/kafka-connect-user.yaml

@@ -51,4 +51,19 @@ spec:
   - from:
     - podSelector:
         matchLabels:
-          app: prometheus
+          app: prometheus
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: allow-logstash-kafka
+  namespace: kafka
+spec:
+  podSelector:
+    matchLabels:
+      strimzi.io/name: kconnect-cluster
+  ingress:
+  - from:
+    - podSelector:
+        matchLabels:
+          app: logstash

manifests/kafka-connect.yaml renamed to helm-chart/kafka-cluster/templates/kafka-connect.yaml

@@ -0,0 +1,5 @@
+# Default values for fabrikate-kafka.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+kafkaLogDirs: /var/lib/kafka/data-0