Experimental support for GitHub token authentication

infinisil · infinisil · commit ada887c3b2a6 · 2025-05-21T01:00:31.000+02:00
This comes from the desire to be more fine-grained in the permissions
given to the github management automation.

E.g. With GitHub Apps, there's no way to give both the "manage teams"
permission without also the "remove org owners" permission. Similarly,
the "manage repos" permission also comes with the "remove repos"
permission. only what's required.

In comparison, with a personal access token of a user, you can give that
user specific access levels on the repos it needs access to, or only add
it to specific teams it needs to manage.

For now, personal access tokens are limited to team member management,
because unless you make the user an org admin, you run into problems for
pretty much any other functionality.
diff --git a/.github/workflows/apply.yml b/.github/workflows/apply.yml
@@ -32,6 +32,7 @@ jobs:
       - name: Find sha for plan
         id: sha
         env:
+          GITHUB_TOKEN: ${{ secrets.RW_GITHUB_TOKEN }}
           GITHUB_APP_ID: ${{ secrets.RW_GITHUB_APP_ID }}
           GITHUB_APP_INSTALLATION_ID: ${{ secrets[format('RW_GITHUB_APP_INSTALLATION_ID_{0}', matrix.workspace)] || secrets.RW_GITHUB_APP_INSTALLATION_ID }}
           GITHUB_APP_PEM_FILE: ${{ secrets.RW_GITHUB_APP_PEM_FILE }}
@@ -55,6 +56,7 @@ jobs:
       TF_WORKSPACE: ${{ matrix.workspace }}
       AWS_ACCESS_KEY_ID: ${{ secrets.RW_AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.RW_AWS_SECRET_ACCESS_KEY }}
+      GITHUB_TOKEN: ${{ secrets.RW_GITHUB_TOKEN }}
       GITHUB_APP_ID: ${{ secrets.RW_GITHUB_APP_ID }}
       GITHUB_APP_INSTALLATION_ID: ${{ secrets[format('RW_GITHUB_APP_INSTALLATION_ID_{0}', matrix.workspace)] || secrets.RW_GITHUB_APP_INSTALLATION_ID }}
       GITHUB_APP_PEM_FILE: ${{ secrets.RW_GITHUB_APP_PEM_FILE }}
diff --git a/.github/workflows/clean.yml b/.github/workflows/clean.yml
@@ -60,6 +60,7 @@ jobs:
       TF_WORKSPACE_OPT: ${{ matrix.workspace }}
       AWS_ACCESS_KEY_ID: ${{ secrets.RW_AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.RW_AWS_SECRET_ACCESS_KEY }}
+      GITHUB_TOKEN: ${{ secrets.RW_GITHUB_TOKEN }}
       GITHUB_APP_ID: ${{ secrets.RW_GITHUB_APP_ID }}
       GITHUB_APP_INSTALLATION_ID: ${{ secrets[format('RW_GITHUB_APP_INSTALLATION_ID_{0}', matrix.workspace)] || secrets.RW_GITHUB_APP_INSTALLATION_ID }}
       GITHUB_APP_PEM_FILE: ${{ secrets.RW_GITHUB_APP_PEM_FILE }}
diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml
@@ -31,6 +31,7 @@ jobs:
     name: Clean Up
     runs-on: ubuntu-latest
     env:
+      GITHUB_TOKEN: ${{ secrets.RO_GITHUB_TOKEN }}
       GITHUB_APP_ID: ${{ secrets.RO_GITHUB_APP_ID }}
       GITHUB_APP_INSTALLATION_ID: ${{ secrets[format('RO_GITHUB_APP_INSTALLATION_ID_{0}', github.repository_owner)] || secrets.RO_GITHUB_APP_INSTALLATION_ID }}
       GITHUB_APP_PEM_FILE: ${{ secrets.RO_GITHUB_APP_PEM_FILE }}
diff --git a/.github/workflows/fix.yml b/.github/workflows/fix.yml
@@ -135,6 +135,9 @@ jobs:
     steps:
       - name: Generate app token
         id: token
+        env:
+          GITHUB_TOKEN: ${{ secrets.RW_GITHUB_TOKEN }}
+        if: ${{ ! env.GITHUB_TOKEN }}
         uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0
         with:
           app_id: ${{ secrets.RW_GITHUB_APP_ID }}
@@ -146,7 +149,7 @@ jobs:
         with:
           repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
           ref: ${{ github.event.pull_request.head.sha || github.sha }}
-          token: ${{ steps.token.outputs.token }}
+          token: ${{ secrets.RW_GITHUB_TOKEN || steps.token.outputs.token }}
           path: head
       - name: Checkout
         uses: actions/checkout@v4
@@ -184,7 +187,7 @@ jobs:
       - if: steps.github-modified.outputs.this == 'true' && github.event_name != 'pull_request_target'
         uses: ./base/.github/actions/git-push
         env:
-          GITHUB_TOKEN: ${{ steps.token.outputs.token }}
+          GITHUB_TOKEN: ${{ secrets.RW_GITHUB_TOKEN || steps.token.outputs.token }}
         with:
           suffix: fix
           working-directory: head
diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml
@@ -29,6 +29,7 @@ jobs:
     name: Sync
     runs-on: ubuntu-latest
     env:
+      GITHUB_TOKEN: ${{ secrets.RW_GITHUB_TOKEN }}
       GITHUB_APP_ID: ${{ secrets.RW_GITHUB_APP_ID }}
       GITHUB_APP_INSTALLATION_ID: ${{ secrets[format('RW_GITHUB_APP_INSTALLATION_ID_{0}', github.repository_owner)] || secrets.RW_GITHUB_APP_INSTALLATION_ID }}
       GITHUB_APP_PEM_FILE: ${{ secrets.RW_GITHUB_APP_PEM_FILE }}
diff --git a/.github/workflows/plan.yml b/.github/workflows/plan.yml
@@ -64,6 +64,7 @@ jobs:
       TF_WORKSPACE: ${{ matrix.workspace }}
       AWS_ACCESS_KEY_ID: ${{ secrets.RO_AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.RO_AWS_SECRET_ACCESS_KEY }}
+      GITHUB_TOKEN: ${{ secrets.RO_GITHUB_TOKEN }}
       GITHUB_APP_ID: ${{ secrets.RO_GITHUB_APP_ID }}
       GITHUB_APP_INSTALLATION_ID: ${{ secrets[format('RO_GITHUB_APP_INSTALLATION_ID_{0}', matrix.workspace)] || secrets.RO_GITHUB_APP_INSTALLATION_ID }}
       GITHUB_APP_PEM_FILE: ${{ secrets.RO_GITHUB_APP_PEM_FILE }}
diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml
@@ -54,6 +54,7 @@ jobs:
       TF_WORKSPACE_OPT: ${{ matrix.workspace }}
       AWS_ACCESS_KEY_ID: ${{ secrets.RW_AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.RW_AWS_SECRET_ACCESS_KEY }}
+      GITHUB_TOKEN: ${{ secrets.RW_GITHUB_TOKEN }}
       GITHUB_APP_ID: ${{ secrets.RW_GITHUB_APP_ID }}
       GITHUB_APP_INSTALLATION_ID: ${{ secrets[format('RW_GITHUB_APP_INSTALLATION_ID_{0}', matrix.workspace)] || secrets.RW_GITHUB_APP_INSTALLATION_ID }}
       GITHUB_APP_PEM_FILE: ${{ secrets.RW_GITHUB_APP_PEM_FILE }}
@@ -107,6 +108,9 @@ jobs:
     steps:
       - name: Generate app token
         id: token
+        env:
+          GITHUB_TOKEN: ${{ secrets.RW_GITHUB_TOKEN }}
+        if: ${{ ! env.GITHUB_TOKEN }}
         uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0
         with:
           app_id: ${{ secrets.RW_GITHUB_APP_ID }}
@@ -116,7 +120,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
         with:
-          token: ${{ steps.token.outputs.token }}
+          token: ${{ secrets.RW_GITHUB_TOKEN || steps.token.outputs.token }}
       - uses: ./.github/actions/git-config-user
       - env:
           WORKSPACES: ${{ needs.prepare.outputs.workspaces }}
diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml
@@ -25,6 +25,7 @@ jobs:
         working-directory: scripts
       - name: Update PRs
         env:
+          GITHUB_TOKEN: ${{ secrets.RW_GITHUB_TOKEN }}
           GITHUB_APP_ID: ${{ secrets.RW_GITHUB_APP_ID }}
           GITHUB_APP_INSTALLATION_ID: ${{ secrets[format('RW_GITHUB_APP_INSTALLATION_ID_{0}', matrix.workspace)] || secrets.RW_GITHUB_APP_INSTALLATION_ID }}
           GITHUB_APP_PEM_FILE: ${{ secrets.RW_GITHUB_APP_PEM_FILE }}
diff --git a/.github/workflows/upgrade.yml b/.github/workflows/upgrade.yml
@@ -14,6 +14,7 @@ jobs:
     with:
       ref: inputs.ref
     secrets:
+      RW_GITHUB_TOKEN: ${{ secrets.RW_GITHUB_TOKEN }}
       GITHUB_APP_ID: ${{ secrets.RW_GITHUB_APP_ID }}
       GITHUB_APP_INSTALLATION_ID: ${{ secrets[format('RW_GITHUB_APP_INSTALLATION_ID_{0}', github.repository_owner)] || secrets.RW_GITHUB_APP_INSTALLATION_ID }}
       GITHUB_APP_PEM_FILE: ${{ secrets.RW_GITHUB_APP_PEM_FILE }}
diff --git a/.github/workflows/upgrade_reusable.yml b/.github/workflows/upgrade_reusable.yml
@@ -9,12 +9,14 @@ on:
         description: The github-mgmt-template ref to upgrade to
         default: master
     secrets:
+      RW_GITHUB_TOKEN:
+        required: false
       GITHUB_APP_ID:
-        required: true
+        required: false
       GITHUB_APP_INSTALLATION_ID:
-        required: true
+        required: false
       GITHUB_APP_PEM_FILE:
-        required: true
+        required: false
 
 jobs:
   upgrade:
@@ -26,6 +28,9 @@ jobs:
     steps:
       - name: Generate app token
         id: token
+        env:
+          GITHUB_TOKEN: ${{ secrets.RW_GITHUB_TOKEN }}
+        if: ${{ ! env.GITHUB_TOKEN }}
         uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0
         with:
           app_id: ${{ secrets.GITHUB_APP_ID }}
@@ -42,7 +47,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           path: github-mgmt
-          token: ${{ steps.token.outputs.token }}
+          token: ${{ secrets.RW_GITHUB_TOKEN || steps.token.outputs.token }}
       - name: Copy files from the template
         run: |
           for file in $(git ls-files ':!:github/*.yml' ':!:scripts/src/actions/fix-yaml-config.ts' ':!:terraform/*_override.tf' ':!:.github/workflows/*_reusable.yml' ':!:README.md'); do
@@ -57,7 +62,7 @@ jobs:
         working-directory: github-mgmt
       - uses: ./github-mgmt-template/.github/actions/git-push
         env:
-          GITHUB_TOKEN: ${{ steps.token.outputs.token }}
+          GITHUB_TOKEN: ${{ secrets.RW_GITHUB_TOKEN || steps.token.outputs.token }}
         with:
           suffix: upgrade
           working-directory: github-mgmt
diff --git a/docs/SETUP.md b/docs/SETUP.md
@@ -87,7 +87,15 @@
     - [ ] one with read & write policy attached
 - [ ] Modify [terraform/terraform_override.tf](terraform/terraform_override.tf) to reflect your AWS setup
 
-## GitHub App
+## GitHub API access
+
+There are two possible ways for GitHub API access:
+- With GitHub Apps, which has the benefit of not being tied to a GitHub user
+- [experimental] With personal access tokens for a GitHub user, which has the benefit of more granular permissions, but is limited in functionality and requires more manual work:
+  - Only teams and team memberships are supported right now
+  - The GitHub user must be a team maintainer for any teams it should manage
+
+### GitHub App
 
 *NOTE*: If you already have a GitHub App with required permissions you can skip the app creation step.
 
@@ -114,18 +122,50 @@
     </details>
 - [ ] [Install the GitHub Apps](https://docs.github.com/en/developers/apps/managing-github-apps/installing-github-apps) in the GitHub organization for `All repositories`
 
+### Personal access token
+
+- [ ] Create a separate dedicated GitHub account for GitHub Management. It is not recommended to use your personal account.
+- [ ] [Create two fine-grained personal access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#creating-a-fine-grained-personal-access-token) for the dedicated GitHub account - *they are going to be used by terraform and GitHub Actions to authenticate with GitHub*:
+  - Resource owner: The GitHub Organization
+  - Expiration: 366 days (you can also [remove the limit](https://docs.github.com/en/organizations/managing-programmatic-access-to-your-organization/setting-a-personal-access-token-policy-for-your-organization#enforcing-a-maximum-lifetime-policy-for-personal-access-tokens)
+  - Repository access:
+    - Only select repositories: Select the GitHub Management repository
+  - Permissions:
+    <details><summary>read-only</summary>
+
+    - `Repository permissions`
+        - `Contents`: `Read-only`
+        - `Metadata`: `Read-only`
+    - `Organization permissions`
+        - `Members`: `Read-only`
+    </details>
+    <details><summary>read & write</summary>
+
+    - `Repository permissions`
+        - `Contents`: `Read & Write`
+        - `Metadata`: `Read-only`
+    - `Organization permissions`
+        - `Members`: `Read & Write`
+    </details>
+- [ ] Switch to an organization owner account and approve the tokens in the organizations settings, under "Personal access tokens > Pending requests"
+- [ ] Give the dedicated GitHub account write access to the GitHub Management Repository
+
 ## GitHub Repository Secrets
 
 - [ ] [Create encrypted secrets](https://docs.github.com/en/actions/security-guides/encrypted-secrets#creating-encrypted-secrets-for-an-organization) for the GitHub organization and allow the repository to access them (\*replace `$GITHUB_ORGANIZATION_NAME` with the GitHub organization name) - *these secrets are read by the GitHub Action workflows*
-    - [ ] Go to `https://github.com/organizations/$GITHUB_ORGANIZATION_NAME/settings/apps/$GITHUB_APP_NAME` and copy the `App ID`
-       - [ ] `RO_GITHUB_APP_ID`
-       - [ ] `RW_GITHUB_APP_ID`
-    - [ ] Go to `https://github.com/organizations/$GITHUB_ORGANIZATION_NAME/settings/installations`, click `Configure` next to the `$GITHUB_APP_NAME` and copy the numeric suffix from the URL
-       - [ ] `RO_GITHUB_APP_INSTALLATION_ID` (or `RO_GITHUB_APP_INSTALLATION_ID_$GITHUB_ORGANIZATION_NAME` for organizations other than the repository owner)
-       - [ ] `RW_GITHUB_APP_INSTALLATION_ID` (or `RW_GITHUB_APP_INSTALLATION_ID_$GITHUB_ORGANIZATION_NAME` for organizations other than the repository owner)
-    - [ ] Go to `https://github.com/organizations/$GITHUB_ORGANIZATION_NAME/settings/apps/$GITHUB_APP_NAME`, click `Generate a private key` and copy the contents of the downloaded PEM file
-       - [ ] `RO_GITHUB_APP_PEM_FILE`
-       - [ ] `RW_GITHUB_APP_PEM_FILE`
+    - If you use a GitHub App:
+      - [ ] Go to `https://github.com/organizations/$GITHUB_ORGANIZATION_NAME/settings/apps/$GITHUB_APP_NAME` and copy the `App ID`
+         - [ ] `RO_GITHUB_APP_ID`
+         - [ ] `RW_GITHUB_APP_ID`
+      - [ ] Go to `https://github.com/organizations/$GITHUB_ORGANIZATION_NAME/settings/installations`, click `Configure` next to the `$GITHUB_APP_NAME` and copy the numeric suffix from the URL
+         - [ ] `RO_GITHUB_APP_INSTALLATION_ID` (or `RO_GITHUB_APP_INSTALLATION_ID_$GITHUB_ORGANIZATION_NAME` for organizations other than the repository owner)
+         - [ ] `RW_GITHUB_APP_INSTALLATION_ID` (or `RW_GITHUB_APP_INSTALLATION_ID_$GITHUB_ORGANIZATION_NAME` for organizations other than the repository owner)
+      - [ ] Go to `https://github.com/organizations/$GITHUB_ORGANIZATION_NAME/settings/apps/$GITHUB_APP_NAME`, click `Generate a private key` and copy the contents of the downloaded PEM file
+         - [ ] `RO_GITHUB_APP_PEM_FILE`
+         - [ ] `RW_GITHUB_APP_PEM_FILE`
+    - If you use personal access tokens
+      - [ ] `RO_GITHUB_TOKEN`
+      - [ ] `RW_GITHUB_TOKEN`
     - [ ] Use the values generated during [AWS](#aws) setup
        - [ ] `RO_AWS_ACCESS_KEY_ID`
        - [ ] `RW_AWS_ACCESS_KEY_ID`
diff --git a/scripts/src/env.ts b/scripts/src/env.ts
@@ -4,6 +4,7 @@ export default {
   TF_WORKING_DIR: process.env.TF_WORKING_DIR || '../terraform',
   FILES_DIR: process.env.FILES_DIR || '../files',
   GITHUB_DIR: process.env.GITHUB_DIR || '../github',
+  GITHUB_TOKEN: process.env.GITHUB_TOKEN || '',
   GITHUB_APP_ID: process.env.GITHUB_APP_ID || '',
   GITHUB_APP_INSTALLATION_ID: process.env.GITHUB_APP_INSTALLATION_ID || '',
   GITHUB_APP_PEM_FILE: process.env.GITHUB_APP_PEM_FILE || '',
diff --git a/scripts/src/github.ts b/scripts/src/github.ts
@@ -102,16 +102,21 @@ export class GitHub {
       // NOTE: We import these dynamically so that they can be mocked
       const {createAppAuth} = await import('@octokit/auth-app')
       const {Octokit} = await import('@octokit/rest')
-      const auth = createAppAuth({
-        appId: env.GITHUB_APP_ID,
-        privateKey: env.GITHUB_APP_PEM_FILE
-      })
-      const installationAuth = await auth({
-        type: 'installation',
-        installationId: env.GITHUB_APP_INSTALLATION_ID
-      })
+      let token = env.GITHUB_TOKEN;
+      if (token == '') {
+        const auth = createAppAuth({
+          appId: env.GITHUB_APP_ID,
+          privateKey: env.GITHUB_APP_PEM_FILE
+        })
+        const installationAuth = await auth({
+          type: 'installation',
+          installationId: env.GITHUB_APP_INSTALLATION_ID
+        })
+        token = installationAuth.token;
+      }
+
       const client = new (Octokit.plugin(retry, throttling))({
-        auth: installationAuth.token,
+        auth: token,
         throttle: {
           onRateLimit: (
             retryAfter: number,
