MLH-1378 Atlas Helm Package + Version + Publish - Master

.github/workflows/maven.yml

@@ -34,268 +34,352 @@
       - mlh-1240-improve-cm-refresh-master
 
 jobs:
-  build:
+  helm-lint:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          # Application charts
+          - chart: atlas
+            path: helm/atlas
+            requires_app_version: true
+          - chart: atlas-read
+            path: helm/atlas-read
+            requires_app_version: true
+          # Atlas infrastructure charts
+          - chart: cassandra
+            path: helm/atlas/charts/cassandra
+            requires_app_version: false
+          - chart: elasticsearch
+            path: helm/atlas/charts/elasticsearch
+            requires_app_version: false
+          - chart: logstash
+            path: helm/atlas/charts/logstash
+            requires_app_version: false
+          # Atlas-Read infrastructure charts
+          - chart: cassandra-online-dc
+            path: helm/atlas-read/charts/cassandra-online-dc
+            requires_app_version: false
+          - chart: elasticsearch-read
+            path: helm/atlas-read/charts/elasticsearch-read
+            requires_app_version: false
+          - chart: elasticsearch-exporter-read
+            path: helm/atlas-read/charts/elasticsearch-exporter-read
+            requires_app_version: false
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Install Helm
+        uses: azure/setup-helm@v3
+        with:
+          version: '3.12.0'
+
+      - name: Update helm dependencies
+        if: matrix.chart == 'atlas' || matrix.chart == 'atlas-read'
+        run: |
+          cd ${{ matrix.path }}
+          helm dependency update
+
+          echo "Chart dependencies:"
+          ls -la charts/
+
+      - name: Lint helm chart
+        run: |
+          helm lint ${{ matrix.path }}/
+          echo "✅ ${{ matrix.chart }} chart lint passed!"
+
+      - name: Validate Chart.yaml
+        run: |
+          # Check for required fields
+          if ! grep -q "^version:" ${{ matrix.path }}/Chart.yaml; then
+            echo "❌ Error: version field missing in Chart.yaml"
+            exit 1
+          fi
+
+          # appVersion is only required for application charts (atlas, atlas-read)
+          if [[ "${{ matrix.requires_app_version }}" == "true" ]]; then
+            if ! grep -q "^appVersion:" ${{ matrix.path }}/Chart.yaml; then
+              echo "❌ Error: appVersion field missing in Chart.yaml for application chart"
+              exit 1
+            fi
+          fi
+
+          echo "✅ Chart.yaml validation passed for ${{ matrix.chart }}!"
 
+  build:
+    needs: helm-lint
     runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v3
 
       # Set up Docker
       - name: Set up Docker
         uses: docker/setup-buildx-action@v2
         with:
           driver-opts: image=moby/buildkit:master
           install: true
 
       - name: Set up JDK 17
         uses: actions/setup-java@v1
         with:
           java-version: 17
 
       - name: Print JDK version
         run: java -version
 
       # Verify Docker is available
       - name: Verify Docker
         run: |
          docker --version
          docker info
 
       - name: Cache Maven packages
         uses: actions/cache@v3
         with:
           path: ~/.m2
           key: ${{ runner.os }}-m2-${{ hashFiles('**/build.sh') }}
           restore-keys: ${{ runner.os }}-m2
 
       - name: Get branch name
         run: |
           echo "BRANCH_NAME=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV
           echo BRANCH_NAME=${GITHUB_REF#refs/heads/}
 
       - name: Create Maven Settings
         uses: s4u/[email protected]
         with:
           servers: |
             [{
                 "id": "github",
                 "username": "atlan-ci",
                 "password": "${{ secrets.ORG_PAT_GITHUB }}"
             }]
 
       - name: Build with Maven
         run: |
           echo "build without dashboard"
           chmod +x ./build.sh && ./build.sh
 
       - name: Check disk space before tests
         id: check_disk
         run: |
           echo "=========================================="
           echo "DISK SPACE CHECK"
           echo "=========================================="
           df -h / | grep -E '^/dev/' || df -h / | tail -1
           echo ""
 
           # Get disk usage percentage (remove % sign)
           DISK_USAGE=$(df / | tail -1 | awk '{print $5}' | sed 's/%//')
           echo "Current disk usage: ${DISK_USAGE}%"
           echo "disk_usage=$DISK_USAGE" >> $GITHUB_OUTPUT
 
           if [ "$DISK_USAGE" -gt 70 ]; then
             echo "⚠️  Disk usage is high (${DISK_USAGE}%), cleanup will run"
           else
             echo "✅ Disk space is adequate (${DISK_USAGE}%), skipping cleanup"
           fi
 
       - name: Free up disk space for tests
         if: steps.check_disk.outputs.disk_usage > 70
         run: |
           echo "=========================================="
           echo "CLEANING UP DISK SPACE"
           echo "=========================================="
 
           # Clean Docker system
           echo "Cleaning Docker system..."
           docker system prune -af --volumes || true
 
           # Clean apt cache
           echo "Cleaning apt cache..."
           sudo apt-get clean || true
           sudo rm -rf /var/cache/apt/archives/* || true
 
           # Clean temp files
           echo "Cleaning temp files..."
           sudo rm -rf /tmp/* || true
 
           # Clean old GitHub Actions logs
           echo "Cleaning GitHub Actions logs..."
           sudo rm -rf /home/runner/work/_temp/_runner_file_commands/* || true
 
           # Clean hostedtoolcache if needed (keep essentials)
           echo "Cleaning hostedtoolcache (non-essential tools)..."
           sudo rm -rf /opt/hostedtoolcache/CodeQL || true
           sudo rm -rf /opt/hostedtoolcache/go || true
           sudo rm -rf /opt/hostedtoolcache/PyPy || true
           sudo rm -rf /opt/hostedtoolcache/node || true
           sudo rm -rf /opt/hostedtoolcache/Ruby || true
 
           echo ""
           echo "Disk space after cleanup:"
           df -h / | grep -E '^/dev/' || df -h / | tail -1
 
       - name: Verify sufficient disk space
         run: |
           echo "=========================================="
           echo "VERIFYING DISK SPACE"
           echo "=========================================="
 
           DISK_USAGE=$(df / | tail -1 | awk '{print $5}' | sed 's/%//')
           AVAILABLE_GB=$(df -h / | tail -1 | awk '{print $4}')
 
           echo "Current disk usage: ${DISK_USAGE}%"
           echo "Available space: ${AVAILABLE_GB}"
 
           # Fail if disk usage is still above 85%
           if [ "$DISK_USAGE" -gt 85 ]; then
             echo "ERROR: Insufficient disk space (${DISK_USAGE}% used)"
             echo "Tests require at least 15% free space to run reliably"
             echo "Elasticsearch will fail with high disk watermark errors at 90%+"
             exit 1
           else
             echo "Sufficient disk space available (${DISK_USAGE}% used)"
           fi
 
       - name: Run Integration Tests
         id: integration_tests
         continue-on-error: true
         env:
           # Configure Testcontainers for GitHub Actions
           TESTCONTAINERS_RYUK_DISABLED: true
           TESTCONTAINERS_CHECKS_DISABLE: true
           DOCKER_HOST: unix:///var/run/docker.sock
         run: |
           echo "Running integration tests..."
           chmod +x ./run-integration-tests.sh && ./run-integration-tests.sh
 
       - name: Upload container logs as artifact
         if: always()  # Upload logs even if tests pass (for debugging)
         uses: actions/upload-artifact@v4
         with:
           name: container-logs-${{ github.run_id }}
           path: target/test-logs/
           retention-days: 5
 
       - name: Setup tmate session on test failure
         if: steps.integration_tests.outcome == 'failure'
         uses: mxschmitt/action-tmate@v3
         timeout-minutes: 30
         with:
           detached: true
           limit-access-to-actor: false
 
       - name: Fail the workflow if tests failed
         if: steps.integration_tests.outcome == 'failure'
         run: exit 1
 
+
       - name: Clean up after integration tests
+        if: always()
         run: |
-          echo "=========================================="
-          echo "CLEANING UP AFTER INTEGRATION TESTS"
-          echo "=========================================="
-
-          # Remove test containers and images
-          echo "Removing test containers and images..."
+          echo "Cleaning up test environment..."
+          docker ps -a -q | xargs -r docker rm -f || true
+          docker system prune -af || true
+          echo ""
+          echo "Removing other test containers and images..."
           docker system prune -af --volumes || true
 
           # Clean Maven artifacts to free up space
           echo "Cleaning Maven artifacts..."
           rm -rf ~/.m2/repository/org/apache/atlas/ || true
 
           # Clean test artifacts
           echo "Cleaning test artifacts..."
           rm -rf webapp/target/surefire-reports/ || true
           rm -rf test-debug-logs/ || true
 
           # Clean temp files
           echo "Cleaning temp files..."
           sudo rm -rf /tmp/* || true
 
           echo ""
           echo "Disk space after cleanup:"
           df -h / | tail -1
 
       - name: Get Repository Name
         run:   echo "REPOSITORY_NAME=`echo "$GITHUB_REPOSITORY" | awk -F / '{print $2}' | sed -e "s/:refs//"`" >> $GITHUB_ENV
         shell: bash
 
       - name: Get version tag
         # run: echo "##[set-output name=version;]$(echo `git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ env.BRANCH_NAME }} | awk '{ print $1}' | cut -c1-7`)abcd"
         run: |
           echo "VERSION=$(git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ env.BRANCH_NAME }} | awk '{ print $1}' | cut -c1-7 | head -n 1)abcd"
           echo "VERSION=$(git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ env.BRANCH_NAME }} | awk '{ print $1}' | cut -c1-7 | tr -d '[:space:]')abcd"
           echo "VERSION=$(git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ env.BRANCH_NAME }} | awk '{ print $1}' | cut -c1-7 | tr -d '[:space:]')abcd" >> $GITHUB_ENV
 
       - name: Get commit ID
         run: echo "COMMIT_ID=$(echo ${GITHUB_SHA} | cut -c1-7)abcd" >> $GITHUB_ENV
 
       # QEMU is required to build arm from a non-arm build machine
       - name: Set up QEMU
         id: qemu
         uses: docker/setup-qemu-action@v3
         with:
           image: tonistiigi/binfmt:qemu-v7.0.0-28
           platforms: arm64
 
       - name: Set up Buildx
         id: buildx
         uses: docker/setup-buildx-action@v1
 
       - name: Login to GitHub Registry
         uses: docker/login-action@v1
         with:
           registry: ghcr.io
           username: $GITHUB_ACTOR
           password: ${{ secrets.ORG_PAT_GITHUB }}
 
       - name: Build and push
         id: docker_build
         uses: docker/build-push-action@v3
         with:
           platforms: linux/amd64,linux/arm64
           context: .
           file: ./Dockerfile
           no-cache: true
           sbom: true
           provenance: true
           push: true
           tags: |
             ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ env.BRANCH_NAME }}:latest
             ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ env.BRANCH_NAME }}:${{ env.COMMIT_ID }}
 
       - name: Check Image Manifest
         run: docker buildx imagetools inspect --raw ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ env.BRANCH_NAME }}:${{ env.COMMIT_ID }}
 
       - name: Scan Image
         uses: aquasecurity/trivy-action@master
         with:
           image-ref: 'ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ env.BRANCH_NAME }}:${{ env.COMMIT_ID }}'
           vuln-type: 'os,library'
           format: 'sarif'
           output: 'trivy-image-results.sarif'
 
       - name: Upload Trivy scan results to GitHub Security tab
         uses: github/codeql-action/[email protected]
         with:
           sarif_file: 'trivy-image-results.sarif'
 
   # Smoke test on vclusters (parallel with single VPN)
 
   smoke-test:
     name: Multi-Cloud Smoke Test
     needs: build
     runs-on: ubuntu-latest
+    # Only run smoke tests on protected branches (beta, staging, master)
+    # Feature branches skip smoke tests but can still publish OCI charts for manual testing
+    if: github.ref_name == 'beta' || github.ref_name == 'staging' || github.ref_name == 'master'
+
+    # Concurrency control: Only one smoke test at a time across all branches
+    # Shared vClusters cannot handle concurrent deployments
+    concurrency:
+      group: smoke-test-shared-vclusters
+      cancel-in-progress: false  # Don't cancel running tests, queue instead
 
     env:
       VCLUSTER_AWS_NAME: ${{ vars.VCLUSTER_AWS_NAME }}
@@ -430,4 +514,181 @@
         uses: actions/upload-artifact@v4
         with:
           name: smoke-test-logs-${{ github.run_id }}
-          path: smoke-test-logs/
+          path: smoke-test-logs/
+
+  helm-publish:
+    needs: [build, smoke-test]
+    runs-on: ubuntu-latest
+    # Run if smoke-test passed (protected branches) OR was skipped (feature branches)
+    if: always() && needs.build.result == 'success' && (needs.smoke-test.result == 'success' || needs.smoke-test.result == 'skipped')
+    strategy:
+      matrix:
+        include:
+          # Application charts
+          - chart: atlas
+            path: helm/atlas
+            base_version: "1.0.0"
+          - chart: atlas-read
+            path: helm/atlas-read
+            base_version: "1.0.0"
+          # Atlas infrastructure charts
+          - chart: cassandra
+            path: helm/atlas/charts/cassandra
+            base_version: "0.14.4"
+          - chart: elasticsearch
+            path: helm/atlas/charts/elasticsearch
+            base_version: "7.6.1"
+          - chart: logstash
+            path: helm/atlas/charts/logstash
+            base_version: "9.1.2"
+          # Atlas-Read infrastructure charts
+          - chart: cassandra-online-dc
+            path: helm/atlas-read/charts/cassandra-online-dc
+            base_version: "0.14.4"
+          - chart: elasticsearch-read
+            path: helm/atlas-read/charts/elasticsearch-read
+            base_version: "7.6.1"
+          - chart: elasticsearch-exporter-read
+            path: helm/atlas-read/charts/elasticsearch-exporter-read
+            base_version: "3.3.0"
+      max-parallel: 1  # Publish sequentially to avoid race conditions
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Get branch name
+        id: branch
+        run: |
+          echo "name=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT
+
+      - name: Get commit ID
+        id: commit
+        run: |
+          echo "id=$(echo ${GITHUB_SHA} | cut -c1-7)abcd" >> $GITHUB_OUTPUT
+
+      - name: Generate chart version
+        id: version
+        run: |
+          # Semantic version: {base_version}-{branch}.{commit}
+          # Replace underscores with hyphens for semver compliance
+          BRANCH_NAME_NORMALIZED=$(echo "${{ steps.branch.outputs.name }}" | tr '_' '-')
+          CHART_VERSION="${{ matrix.base_version }}-${BRANCH_NAME_NORMALIZED}.${{ steps.commit.outputs.id }}"
+          echo "chart=${CHART_VERSION}" >> $GITHUB_OUTPUT
+          echo "Generated chart version for ${{ matrix.chart }}: ${CHART_VERSION}"
+
+      - name: Install Helm
+        uses: azure/setup-helm@v3
+        with:
+          version: '3.12.0'
+
+      - name: Update Chart.yaml with version
+        run: |
+          sed -i "s/^version: .*/version: ${{ steps.version.outputs.chart }}/" ${{ matrix.path }}/Chart.yaml
+
+          # Only update appVersion for application charts (atlas, atlas-read)
+          if [[ "${{ matrix.chart }}" == "atlas" ]] || [[ "${{ matrix.chart }}" == "atlas-read" ]]; then
+            sed -i "s/^appVersion: .*/appVersion: \"${{ steps.commit.outputs.id }}\"/" ${{ matrix.path }}/Chart.yaml
+            echo "Updated appVersion for ${{ matrix.chart }}"
+          fi
+
+          echo "Updated ${{ matrix.chart }}/Chart.yaml:"
+          cat ${{ matrix.path }}/Chart.yaml | head -10
+
+      - name: Update values.yaml with image tags (application charts only)
+        if: matrix.chart == 'atlas' || matrix.chart == 'atlas-read'
+        run: |
+          # Replace placeholders with actual values (only for atlas/atlas-read)
+          sed -i "s/ATLAS_LATEST_IMAGE_TAG/${{ steps.commit.outputs.id }}/g" ${{ matrix.path }}/values.yaml
+          sed -i "s/ATLAS_BRANCH_NAME/${{ steps.branch.outputs.name }}/g" ${{ matrix.path }}/values.yaml
+
+          echo "Image configuration in ${{ matrix.chart }}/values.yaml:"
+          grep -A 3 "image:" ${{ matrix.path }}/values.yaml | head -10
+
+      - name: Update helm dependencies
+        if: matrix.chart == 'atlas' || matrix.chart == 'atlas-read'
+        run: |
+          cd ${{ matrix.path }}
+          helm dependency update
+
+          echo "Chart dependencies:"
+          ls -la charts/
+
+      - name: Package helm chart
+        run: |
+          mkdir -p helm-packages
+          helm package ${{ matrix.path }}/ --destination ./helm-packages/
+
+          echo "Packaged chart: ${{ matrix.chart }}"
+          ls -lh helm-packages/${{ matrix.chart }}-*
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: $GITHUB_ACTOR
+          password: ${{ secrets.ORG_PAT_GITHUB }}
+
+      - name: Push chart to GHCR (OCI Registry)
+        run: |
+          CHART_FILE=$(ls helm-packages/${{ matrix.chart }}-*.tgz)
+          echo "Pushing chart: ${CHART_FILE}"
+
+          helm push ${CHART_FILE} oci://ghcr.io/atlanhq/helm-charts
+
+          echo "✅ Chart published successfully!"
+          echo "📦 Chart: ${{ matrix.chart }}"
+          echo "📌 Version: ${{ steps.version.outputs.chart }}"
+          echo "🏷️  Registry: oci://ghcr.io/atlanhq/helm-charts/${{ matrix.chart }}"
+
+      - name: Create GitHub Release
+        uses: ncipollo/release-action@v1
+        with:
+          tag: helm-${{ matrix.chart }}-v${{ steps.version.outputs.chart }}
+          name: "${{ matrix.chart }} Helm Chart v${{ steps.version.outputs.chart }}"
+          body: |
+            ## 📦 ${{ matrix.chart }} Helm Chart Release
+
+            **Chart**: `${{ matrix.chart }}`  
+            **Chart Version**: `${{ steps.version.outputs.chart }}`  
+            **Base Version**: `${{ matrix.base_version }}`  
+            **Branch**: `${{ steps.branch.outputs.name }}`  
+            **Commit**: `${{ steps.commit.outputs.id }}`
+
+            ### 📥 Installation
+
+            **Via OCI Registry (Recommended):**
+            ```bash
+            helm install ${{ matrix.chart }} oci://ghcr.io/atlanhq/helm-charts/${{ matrix.chart }} \
+              --version ${{ steps.version.outputs.chart }}
+            ```
+
+            **With custom values:**
+            ```bash
+            helm install ${{ matrix.chart }} oci://ghcr.io/atlanhq/helm-charts/${{ matrix.chart }} \
+              --version ${{ steps.version.outputs.chart }} \
+              -f custom-values.yaml
+            ```
+
+            ### 🔗 Links
+
+            - **Registry**: `oci://ghcr.io/atlanhq/helm-charts/${{ matrix.chart }}`
+            - **Repository**: [atlanhq/atlas-metastore](https://github.com/atlanhq/atlas-metastore)
+            - **Path**: `${{ matrix.path }}`
+          artifacts: "./helm-packages/${{ matrix.chart }}-*.tgz"
+          token: ${{ secrets.ORG_PAT_GITHUB }}
+          allowUpdates: true
+          makeLatest: false
+
+      - name: Chart publish summary
+        run: |
+          echo "## 🎉 Helm Chart Published Successfully!" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "**Chart**: ${{ matrix.chart }}" >> $GITHUB_STEP_SUMMARY
+          echo "**Version**: ${{ steps.version.outputs.chart }}" >> $GITHUB_STEP_SUMMARY
+          echo "**Registry**: oci://ghcr.io/atlanhq/helm-charts/${{ matrix.chart }}" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "### Installation Command" >> $GITHUB_STEP_SUMMARY
+          echo '```bash' >> $GITHUB_STEP_SUMMARY
+          echo "helm install ${{ matrix.chart }} oci://ghcr.io/atlanhq/helm-charts/${{ matrix.chart }} --version ${{ steps.version.outputs.chart }}" >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY

helm/atlas-read/Chart.yaml

@@ -0,0 +1,34 @@
+apiVersion: v2
+name: atlas-read
+description: Apache Atlas Read Replica for Metadata Management
+type: application
+version: 1.0.0
+appVersion: "3.0.0"  # Will be updated by CI with commit ID
+maintainers:
+  - name: Atlan Engineering
+    email: [email protected]
+keywords:
+  - atlas
+  - atlas-read
+  - metadata
+  - read-replica
+  - apache-atlas
+sources:
+  - https://github.com/atlanhq/atlas-metastore
+home: https://github.com/atlanhq/atlas-metastore
+dependencies:
+  # Infrastructure charts - disabled by default (published separately as OCI)
+  # Charts exist in charts/ subdirectory for individual OCI publishing
+  # Consumed as peers in parent atlan chart, not as subcharts here
+  - name: cassandra-online-dc
+    repository: file://./charts/cassandra-online-dc
+    version: 0.x.x
+    condition: cassandra-online-dc.enabled  # Disabled in values.yaml
+  - name: elasticsearch-read
+    repository: file://./charts/elasticsearch-read
+    version: 7.x.x
+    condition: elasticsearch-read.enabled  # Disabled in values.yaml
+  - name: elasticsearch-exporter-read
+    repository: file://./charts/elasticsearch-exporter-read
+    version: 3.3.0
+    condition: elasticsearch-exporter-read.enabled  # Disabled in values.yaml

helm/atlas-read/README.md

@@ -0,0 +1,2 @@
+# atlas
+This chart will install the apache atlas which use elasticsearch and cassandra.

helm/atlas-read/charts/cassandra-online-dc/.helmignore

@@ -0,0 +1,17 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+OWNERS

helm/atlas-read/charts/cassandra-online-dc/Chart.yaml

@@ -0,0 +1,19 @@
+apiVersion: v2
+appVersion: 3.11.5
+description: Apache Cassandra is a free and open-source distributed database management
+  system designed to handle large amounts of data across many commodity servers, providing
+  high availability with no single point of failure.
+engine: gotpl
+home: http://cassandra.apache.org
+icon: https://upload.wikimedia.org/wikipedia/commons/thumb/5/5e/Cassandra_logo.svg/330px-Cassandra_logo.svg.png
+keywords:
+- cassandra
+- database
+- nosql
+maintainers:
+- email: [email protected]
+  name: KongZ
+- email: [email protected]
+  name: maorfr
+name: cassandra-online-dc
+version: 0.14.4