Skip to content

Commit b1429d5

Browse files
cameron-dunn-sublimeemonti-ltc
cameron-dunn-sublime
and
emonti-ltc
authored
Fix template injection and script expansion in github workflows (#627)
Co-authored-by: Eric Monti <[email protected]>
1 parent a4ef060 commit b1429d5

File tree

5 files changed

+63
-44
lines changed

5 files changed

+63
-44
lines changed

‎.github/workflows/ensure-sorted.yml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ jobs:
3333
3434
- name: Commit & Push Results, if needed
3535
id: push_results
36+
env:
37+
HEAD_REF: ${{ github.head_ref }}
3638
run: |
3739
if [ -z "$(git status --porcelain)" ]; then
3840
echo "No files changed, nothing to do"
@@ -43,4 +45,5 @@ jobs:
4345
git config user.email '[email protected]'
4446
git add '*.txt'
4547
git commit -m "Auto sorting static files"
46-
git push origin HEAD:${{ github.head_ref }}
48+
49+
git push origin HEAD:${HEAD_REF}

‎.github/workflows/update-and-process-tranco.yml‎

Lines changed: 35 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,8 @@ jobs:
2424
- name: Set up date variables
2525
id: date
2626
run: |
27-
echo "today=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
28-
echo "timestamp=$(date +'%Y%m%d%H%M%S')" >> $GITHUB_OUTPUT
27+
echo "today=$(date +'%Y-%m-%d')" >> "$GITHUB_OUTPUT"
28+
echo "timestamp=$(date +'%Y%m%d%H%M%S')" >> "$GITHUB_OUTPUT"
2929
3030
- name: Fetch Tranco list ID
3131
id: tranco-id
@@ -55,7 +55,7 @@ jobs:
5555
5656
# Check if we got a valid ID (non-empty and contains alphanumeric characters)
5757
if [[ ! -z "$TRANCO_ID" && "$TRANCO_ID" =~ ^[A-Za-z0-9]+$ ]]; then
58-
echo "id=$TRANCO_ID" >> $GITHUB_OUTPUT
58+
echo "id=$TRANCO_ID" >> "$GITHUB_OUTPUT"
5959
echo "Successfully fetched Tranco list ID: $TRANCO_ID"
6060
SUCCESS=true
6161
else
@@ -85,28 +85,30 @@ jobs:
8585
8686
- name: Download Tranco list
8787
id: download
88+
env:
89+
STEPS_TRANCO_ID_OUTPUTS_ID: ${{ steps.tranco-id.outputs.id }}
8890
run: |
8991
# Maximum retry count
9092
MAX_RETRIES=5
9193
RETRY_COUNT=0
9294
SUCCESS=false
9395
9496
while [ $RETRY_COUNT -lt $MAX_RETRIES ] && [ "$SUCCESS" = "false" ]; do
95-
echo "Attempt $(($RETRY_COUNT + 1)) of $MAX_RETRIES: Downloading Tranco list ${{ steps.tranco-id.outputs.id }}..."
97+
echo "Attempt $(($RETRY_COUNT + 1)) of $MAX_RETRIES: Downloading Tranco list ${STEPS_TRANCO_ID_OUTPUTS_ID}..."
9698
9799
# Use -w to capture HTTP status code
98100
HTTP_STATUS=$(curl -s -L -o tranco.zip -w "%{http_code}" \
99101
--retry 3 --retry-delay 10 --retry-max-time 300 \
100102
--connect-timeout 15 --max-time 300 \
101-
"https://tranco-list.eu/download_daily/${{ steps.tranco-id.outputs.id }}")
103+
"https://tranco-list.eu/download_daily/${STEPS_TRANCO_ID_OUTPUTS_ID}")
102104
103105
echo "HTTP Status Code: $HTTP_STATUS"
104106
105107
# Check if HTTP status code is 200 (OK)
106108
if [ "$HTTP_STATUS" -eq 200 ]; then
107109
# Check if file was actually downloaded and has content
108110
if [ -s tranco.zip ]; then
109-
echo "Successfully downloaded Tranco list ${{ steps.tranco-id.outputs.id }}"
111+
echo "Successfully downloaded Tranco list ${STEPS_TRANCO_ID_OUTPUTS_ID}"
110112
SUCCESS=true
111113
else
112114
echo "Downloaded file is empty despite HTTP 200"
@@ -182,7 +184,7 @@ jobs:
182184
run: |
183185
# Configuration is defined here
184186
CONFIG='[{"count": 10000, "filename": "tranco_top_10k.csv"}, {"count": 50000, "filename": "tranco_top_50k.csv"}]'
185-
echo "CONFIG=$CONFIG" >> $GITHUB_ENV
187+
echo "CONFIG=$CONFIG" >> "$GITHUB_ENV"
186188
echo "Using configuration: $CONFIG"
187189
188190
- name: Validate manifest.json
@@ -198,26 +200,26 @@ jobs:
198200
TEMP_FILE=$(mktemp)
199201
200202
# Check each output file in the configuration
201-
echo $CONFIG | jq -c '.[]' | while read -r config; do
202-
filename=$(echo $config | jq -r '.filename')
203+
echo "$CONFIG" | jq -c '.[]' | while read -r config; do
204+
filename=$(echo "$config" | jq -r '.filename')
203205
204206
# Check if the filename is in manifest.json
205207
if ! grep -q "\"file\": \"$filename\"" manifest.json; then
206208
echo "Error: $filename is not defined in manifest.json"
207-
echo "VALIDATION_FAILED=true" >> $TEMP_FILE
209+
echo "VALIDATION_FAILED=true" >> "$TEMP_FILE"
208210
else
209211
echo "✓ $filename is defined in manifest.json"
210212
fi
211213
done
212214
213215
# Exit if any file is not defined in manifest.json
214-
if grep -q "VALIDATION_FAILED=true" $TEMP_FILE; then
216+
if grep -q "VALIDATION_FAILED=true" "$TEMP_FILE"; then
215217
echo "One or more output files are not defined in manifest.json. Please update manifest.json first."
216-
rm $TEMP_FILE
218+
rm "$TEMP_FILE"
217219
exit 1
218220
fi
219221
220-
rm $TEMP_FILE
222+
rm "$TEMP_FILE"
221223
222224
- name: Process Tranco CSV
223225
id: process
@@ -231,17 +233,17 @@ jobs:
231233
fi
232234
233235
# Parse the JSON configuration and process each output
234-
echo $CONFIG | jq -c '.[]' | while read -r config; do
235-
count=$(echo $config | jq -r '.count')
236-
filename=$(echo $config | jq -r '.filename')
236+
echo "$CONFIG" | jq -c '.[]' | while read -r config; do
237+
count=$(echo "$config" | jq -r '.count')
238+
filename=$(echo "$config" | jq -r '.filename')
237239
238240
if [ -z "$count" ] || [ -z "$filename" ]; then
239241
echo "Skipping invalid configuration: $config"
240242
continue
241243
fi
242244
243245
# Get exactly the requested number of lines from the file
244-
head -n $count tranco.csv > "$filename"
246+
head -n "$count" tranco.csv > "$filename"
245247
246248
lines=$(wc -l < "$filename")
247249
echo "Successfully created $filename with $lines rows"
@@ -254,35 +256,41 @@ jobs:
254256
255257
- name: Create and push branch
256258
id: create-branch
259+
env:
260+
STEPS_DATE_OUTPUTS_TODAY: ${{ steps.date.outputs.today }}
261+
STEPS_DATE_OUTPUTS_TIMESTAMP: ${{ steps.date.outputs.timestamp }}
262+
STEPS_TRANCO_ID_OUTPUTS_ID: ${{ steps.tranco-id.outputs.id }}
257263
run: |
258264
# Create a unique branch name with timestamp
259-
BRANCH_NAME="tranco_update-${{ steps.date.outputs.today }}-${{ steps.date.outputs.timestamp }}"
260-
echo "branch_name=$BRANCH_NAME" >> $GITHUB_OUTPUT
265+
BRANCH_NAME="tranco_update-${STEPS_DATE_OUTPUTS_TODAY}-${STEPS_DATE_OUTPUTS_TIMESTAMP}"
266+
echo "branch_name=$BRANCH_NAME" >> "$GITHUB_OUTPUT"
261267
262268
git checkout -b "$BRANCH_NAME"
263269
git add tranco.csv
264270
265271
# Parse the configuration to get the filenames
266-
echo $CONFIG | jq -c '.[]' | while read -r config; do
267-
filename=$(echo $config | jq -r '.filename')
272+
echo "$CONFIG" | jq -c '.[]' | while read -r config; do
273+
filename=$(echo "$config" | jq -r '.filename')
268274
# Add each generated file individually
269275
git add "$filename"
270276
done
271277
272-
git commit -m "Update Tranco list for ${{ steps.date.outputs.today }} (ID: ${{ steps.tranco-id.outputs.id }})"
278+
git commit -m "Update Tranco list for ${STEPS_DATE_OUTPUTS_TODAY} (ID: ${STEPS_TRANCO_ID_OUTPUTS_ID})"
273279
git push origin "$BRANCH_NAME"
274-
280+
275281
- name: Create Pull Request
276282
env:
277283
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
284+
STEPS_DATE_OUTPUTS_TODAY: ${{ steps.date.outputs.today }}
285+
STEPS_TRANCO_ID_OUTPUTS_ID: ${{ steps.tranco-id.outputs.id }}
278286
run: |
279287
gh pr create \
280-
--title "Update Tranco list and derived files - ${{ steps.date.outputs.today }}" \
288+
--title "Update Tranco list and derived files - ${STEPS_DATE_OUTPUTS_TODAY}" \
281289
--body "This PR updates the Tranco top 1 million domains list and all derived files.
282290
283-
- Date: ${{ steps.date.outputs.today }}
284-
- Tranco List ID: ${{ steps.tranco-id.outputs.id }}
285-
- List URL: https://tranco-list.eu/list/${{ steps.tranco-id.outputs.id }}
291+
- Date: ${STEPS_DATE_OUTPUTS_TODAY}
292+
- Tranco List ID: ${STEPS_TRANCO_ID_OUTPUTS_ID}
293+
- List URL: https://tranco-list.eu/list/${STEPS_TRANCO_ID_OUTPUTS_ID}
286294
- Automated update via GitHub Actions" \
287295
--head "$BRANCH_NAME" \
288296
--base "master"

‎.github/workflows/update-majestic.yml‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,8 @@ jobs:
2424
- name: Set up date variables
2525
id: date
2626
run: |
27-
echo "today=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
28-
echo "timestamp=$(date +'%Y%m%d%H%M%S')" >> $GITHUB_OUTPUT
27+
echo "today=$(date +'%Y-%m-%d')" >> "$GITHUB_OUTPUT"
28+
echo "timestamp=$(date +'%Y%m%d%H%M%S')" >> "$GITHUB_OUTPUT"
2929
3030
- name: Download Majestic Million list
3131
id: download-majestic
@@ -114,26 +114,30 @@ jobs:
114114
115115
- name: Create and push branch
116116
id: create-branch
117+
env:
118+
STEPS_DATE_OUTPUTS_TODAY: ${{ steps.date.outputs.today }}
119+
STEPS_DATE_OUTPUTS_TIMESTAMP: ${{ steps.date.outputs.timestamp }}
117120
run: |
118121
# Create a unique branch name with timestamp
119-
BRANCH_NAME="majestic_update-${{ steps.date.outputs.today }}-${{ steps.date.outputs.timestamp }}"
120-
echo "branch_name=$BRANCH_NAME" >> $GITHUB_OUTPUT
122+
BRANCH_NAME="majestic_update-${STEPS_DATE_OUTPUTS_TODAY}-${STEPS_DATE_OUTPUTS_TIMESTAMP}"
123+
echo "branch_name=$BRANCH_NAME" >> "$GITHUB_OUTPUT"
121124
122125
git checkout -b "$BRANCH_NAME"
123126
git add majestic_million.csv
124127
125-
git commit -m "Update Majestic Million list for ${{ steps.date.outputs.today }}"
128+
git commit -m "Update Majestic Million list for ${STEPS_DATE_OUTPUTS_TODAY}"
126129
git push origin "$BRANCH_NAME"
127130
128131
- name: Create Pull Request
129132
env:
130133
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
134+
STEPS_DATE_OUTPUTS_TODAY: ${{ steps.date.outputs.today }}
131135
run: |
132136
gh pr create \
133-
--title "Update Majestic Million list - ${{ steps.date.outputs.today }}" \
137+
--title "Update Majestic Million list - ${STEPS_DATE_OUTPUTS_TODAY}" \
134138
--body "This PR updates the Majestic Million domains list.
135139
136-
- Date: ${{ steps.date.outputs.today }}
140+
- Date: ${STEPS_DATE_OUTPUTS_TODAY}
137141
- Files updated:
138142
- majestic_million.csv
139143
- Automated update via GitHub Actions" \

‎.github/workflows/update-umbrella.yml‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,8 @@ jobs:
2424
- name: Set up date variables
2525
id: date
2626
run: |
27-
echo "today=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
28-
echo "timestamp=$(date +'%Y%m%d%H%M%S')" >> $GITHUB_OUTPUT
27+
echo "today=$(date +'%Y-%m-%d')" >> "$GITHUB_OUTPUT"
28+
echo "timestamp=$(date +'%Y%m%d%H%M%S')" >> "$GITHUB_OUTPUT"
2929
3030
- name: Download Umbrella top 1 million list
3131
id: download-top-1m
@@ -215,27 +215,31 @@ jobs:
215215
216216
- name: Create and push branch
217217
id: create-branch
218+
env:
219+
STEPS_DATE_OUTPUTS_TODAY: ${{ steps.date.outputs.today }}
220+
STEPS_DATE_OUTPUTS_TIMESTAMP: ${{ steps.date.outputs.timestamp }}
218221
run: |
219222
# Create a unique branch name with timestamp
220-
BRANCH_NAME="umbrella_update-${{ steps.date.outputs.today }}-${{ steps.date.outputs.timestamp }}"
221-
echo "branch_name=$BRANCH_NAME" >> $GITHUB_OUTPUT
223+
BRANCH_NAME="umbrella_update-${STEPS_DATE_OUTPUTS_TODAY}-${STEPS_DATE_OUTPUTS_TIMESTAMP}"
224+
echo "branch_name=$BRANCH_NAME" >> "$GITHUB_OUTPUT"
222225
223226
git checkout -b "$BRANCH_NAME"
224227
git add umbrella_top_1m.csv
225228
git add umbrella_top_1m_tld.csv
226229
227-
git commit -m "Update Umbrella lists for ${{ steps.date.outputs.today }}"
230+
git commit -m "Update Umbrella lists for ${STEPS_DATE_OUTPUTS_TODAY}"
228231
git push origin "$BRANCH_NAME"
229232
230233
- name: Create Pull Request
231234
env:
232235
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
236+
STEPS_DATE_OUTPUTS_TODAY: ${{ steps.date.outputs.today }}
233237
run: |
234238
gh pr create \
235-
--title "Update Umbrella lists - ${{ steps.date.outputs.today }}" \
239+
--title "Update Umbrella lists - ${STEPS_DATE_OUTPUTS_TODAY}" \
236240
--body "This PR updates the Umbrella top 1 million domains list and top TLDs list.
237241
238-
- Date: ${{ steps.date.outputs.today }}
242+
- Date: ${STEPS_DATE_OUTPUTS_TODAY}
239243
- Files updated:
240244
- umbrella_top_1m.csv
241245
- umbrella_top_1m_tld.csv

‎.github/workflows/validate-manifest.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ jobs:
1717
run: |
1818
echo "Validating manifest files"
1919
for item in $(jq -c -r '.lists.[].file | select( . != null )' manifest.json); do
20-
if test -f $item; then
20+
if test -f "$item"; then
2121
echo "$item file is listed in the manifest, and exists in the repository"
2222
else
2323
echo "::error file=manifest.json,title=Invalid-Manifest::$item file is listed in the manifest, but the file does not exist"
@@ -28,7 +28,7 @@ jobs:
2828
echo "Validating manifest urls"
2929
for item in $(jq -c -r '.lists.[].url | select( . != null )' manifest.json); do
3030
urlstatus=$(curl -H 'Cache-Control: no-cache' -o /dev/null --silent --head --write-out "$URL %{http_code}" "$item")
31-
if [ $urlstatus -ne 200 ]; then
31+
if [ "$urlstatus" -ne 200 ]; then
3232
echo "::error file=manifest.json,title=Invalid-Manifest::$item URL is listed in the manifest, but the received HTTP status of $urlstatus"
3333
exit 1
3434
fi

0 commit comments

Comments
 (0)