perf(turbo): optimize build pipeline with better caching (#636) #118
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Token Cost | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| checks: write | |
| jobs: | |
| measure-tokens: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: "20" | |
| # pnpm/action-setup@v4 | |
| - uses: pnpm/action-setup@a7487c7e89a18df4991f7f222e4898a00d66ddda | |
| name: Install pnpm | |
| with: | |
| run_install: false | |
| - name: Get pnpm store directory | |
| shell: bash | |
| run: | | |
| echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV | |
| - uses: actions/cache@v4 | |
| name: Setup pnpm cache | |
| with: | |
| path: ${{ env.STORE_PATH }} | |
| key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pnpm-store- | |
| - name: Install dependencies | |
| run: pnpm install --no-frozen-lockfile | |
| - name: Build tool definitions | |
| run: pnpm -w run build | |
| - name: Measure token cost | |
| id: measure | |
| working-directory: packages/mcp-server | |
| run: | | |
| # Run token counter with JSON output to file | |
| pnpm run measure-tokens -- -o token-stats.json | |
| # Extract key metrics from JSON for GitHub outputs | |
| TOTAL_TOKENS=$(jq -r '.total_tokens' token-stats.json) | |
| TOOL_COUNT=$(jq -r '.tool_count' token-stats.json) | |
| AVG_TOKENS=$(jq -r '.avg_tokens_per_tool' token-stats.json) | |
| # Save for later steps | |
| echo "total_tokens=$TOTAL_TOKENS" >> $GITHUB_OUTPUT | |
| echo "tool_count=$TOOL_COUNT" >> $GITHUB_OUTPUT | |
| echo "avg_tokens=$AVG_TOKENS" >> $GITHUB_OUTPUT | |
| - name: Generate detailed report | |
| id: report | |
| working-directory: packages/mcp-server | |
| run: | | |
| # Build markdown table from JSON | |
| cat > token-report.md <<REPORT_EOF | |
| ## 📊 MCP Server Token Cost Report | |
| **Summary** | |
| - **Total Tokens:** ${{ steps.measure.outputs.total_tokens }} tokens | |
| - **Tool Count:** ${{ steps.measure.outputs.tool_count }} tools | |
| - **Average:** ${{ steps.measure.outputs.avg_tokens }} tokens/tool | |
| ### Per-Tool Breakdown | |
| | Tool | Tokens | % of Total | | |
| |------|--------|------------| | |
| REPORT_EOF | |
| # Add each tool as a table row | |
| jq -r '.tools[] | "| `\(.name)` | \(.tokens) | \(.percentage)% |"' token-stats.json >> token-report.md | |
| cat >> token-report.md <<REPORT_EOF | |
| --- | |
| **Note:** This measures the static overhead of tool definitions sent to LLM clients with every request. Lower is better. The \`use_sentry\` tool is excluded as it's only available in agent mode. | |
| REPORT_EOF | |
| # Display report | |
| cat token-report.md | |
| # Save for summary | |
| cat token-report.md >> $GITHUB_STEP_SUMMARY | |
| - name: Download main branch token stats | |
| if: github.event_name == 'pull_request' | |
| uses: dawidd6/action-download-artifact@v6 | |
| continue-on-error: true | |
| with: | |
| workflow: token-cost.yml | |
| branch: main | |
| name_is_regexp: true | |
| name: 'token-stats-.*' | |
| path: main-stats | |
| search_artifacts: true | |
| - name: Compare with main branch | |
| id: compare | |
| if: github.event_name == 'pull_request' | |
| working-directory: packages/mcp-server | |
| run: | | |
| # Check if we got main's stats | |
| if [ -f ../../main-stats/token-stats.json ]; then | |
| MAIN_TOKENS=$(jq -r '.total_tokens' ../../main-stats/token-stats.json) | |
| CURRENT_TOKENS=${{ steps.measure.outputs.total_tokens }} | |
| DELTA=$((CURRENT_TOKENS - MAIN_TOKENS)) | |
| echo "has_comparison=true" >> $GITHUB_OUTPUT | |
| echo "main_tokens=$MAIN_TOKENS" >> $GITHUB_OUTPUT | |
| echo "delta=$DELTA" >> $GITHUB_OUTPUT | |
| if [ $DELTA -gt 0 ]; then | |
| echo "delta_direction=increased" >> $GITHUB_OUTPUT | |
| echo "delta_symbol=📈" >> $GITHUB_OUTPUT | |
| elif [ $DELTA -lt 0 ]; then | |
| echo "delta_direction=decreased" >> $GITHUB_OUTPUT | |
| echo "delta_symbol=📉" >> $GITHUB_OUTPUT | |
| else | |
| echo "delta_direction=unchanged" >> $GITHUB_OUTPUT | |
| echo "delta_symbol=➡️" >> $GITHUB_OUTPUT | |
| fi | |
| else | |
| echo "has_comparison=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Add comparison to report | |
| if: github.event_name == 'pull_request' && steps.compare.outputs.has_comparison == 'true' | |
| working-directory: packages/mcp-server | |
| run: | | |
| # Add comparison section at the top of the report | |
| cat > comparison-header.md <<COMPARISON_EOF | |
| ### ${{ steps.compare.outputs.delta_symbol }} Comparison with Main Branch | |
| - **Current (PR):** ${{ steps.measure.outputs.total_tokens }} tokens | |
| - **Main branch:** ${{ steps.compare.outputs.main_tokens }} tokens | |
| - **Change:** ${{ steps.compare.outputs.delta }} tokens (${{ steps.compare.outputs.delta_direction }}) | |
| --- | |
| COMPARISON_EOF | |
| # Insert comparison at the beginning (after the title) | |
| sed -i '2r comparison-header.md' token-report.md | |
| # Update job summary with new report | |
| cat token-report.md >> $GITHUB_STEP_SUMMARY | |
| - name: Create check run | |
| uses: actions/github-script@v7 | |
| if: always() | |
| with: | |
| script: | | |
| const measureSucceeded = '${{ steps.measure.outcome }}' === 'success'; | |
| const totalTokens = '${{ steps.measure.outputs.total_tokens }}'; | |
| const toolCount = '${{ steps.measure.outputs.tool_count }}'; | |
| const avgTokens = '${{ steps.measure.outputs.avg_tokens }}'; | |
| const hasComparison = '${{ steps.compare.outputs.has_comparison }}' === 'true'; | |
| const delta = '${{ steps.compare.outputs.delta }}'; | |
| const deltaSymbol = '${{ steps.compare.outputs.delta_symbol }}' || '📊'; | |
| let title; | |
| let summary; | |
| let conclusion; | |
| if (measureSucceeded && totalTokens && toolCount && avgTokens) { | |
| // Build title | |
| title = `${totalTokens} tokens (${toolCount} tools, avg ${avgTokens})`; | |
| // Build summary | |
| summary = `**Total Tokens:** ${totalTokens}\n`; | |
| summary += `**Tool Count:** ${toolCount}\n`; | |
| summary += `**Average:** ${avgTokens} tokens/tool\n`; | |
| if (hasComparison && delta) { | |
| const deltaPrefix = parseInt(delta) >= 0 ? '+' : ''; | |
| title += ` ${deltaSymbol}`; | |
| summary += `\n**Change from main:** ${deltaPrefix}${delta} tokens ${deltaSymbol}`; | |
| } | |
| conclusion = 'success'; | |
| } else { | |
| // Measurement step failed or outputs are missing | |
| title = 'Token cost measurement failed'; | |
| summary = 'The token cost measurement step failed. Check the workflow logs for details.'; | |
| conclusion = 'failure'; | |
| } | |
| await github.rest.checks.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| name: 'Token Cost', | |
| head_sha: context.sha, | |
| status: 'completed', | |
| conclusion: conclusion, | |
| output: { | |
| title: title, | |
| summary: summary, | |
| text: measureSucceeded ? 'See job summary for detailed per-tool breakdown' : 'Token measurement failed - check workflow logs' | |
| }, | |
| details_url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` | |
| }); | |
| - name: Comment on PR if token count changed | |
| uses: actions/github-script@v7 | |
| if: github.event_name == 'pull_request' && steps.compare.outputs.has_comparison == 'true' && steps.compare.outputs.delta != '0' | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const report = fs.readFileSync('packages/mcp-server/token-report.md', 'utf8'); | |
| // Find existing comment | |
| const comments = await github.rest.issues.listComments({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| }); | |
| const botComment = comments.data.find(comment => | |
| comment.user.type === 'Bot' && | |
| comment.body.includes('📊 MCP Server Token Cost Report') | |
| ); | |
| // Update or create comment | |
| if (botComment) { | |
| await github.rest.issues.updateComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: botComment.id, | |
| body: report | |
| }); | |
| } else { | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| body: report | |
| }); | |
| } | |
| - name: Upload token stats artifact | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: token-stats-${{ github.sha }} | |
| path: packages/mcp-server/token-stats.json | |
| retention-days: 90 |