Skip to content

Commit a2ef5e9

Browse files
almahmoudalmahmoud
authored
Separate arch builds and use arm64 native runner
1 parent ff52ffb commit a2ef5e9

File tree

1 file changed

+185
-53
lines changed

1 file changed

+185
-53
lines changed

‎.github/workflows/build_containers.yaml‎

Lines changed: 185 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -9,34 +9,27 @@ on:
99
- cron: '0 18 * * 5'
1010

1111
jobs:
12-
build:
12+
build-amd64:
13+
runs-on: ubuntu-latest
1314
strategy:
1415
fail-fast: false
1516
matrix:
1617
base:
17-
- {image: 'ghcr.io/bioconductor/rocker-r-ver', amdtag: 'devel-amd64', armtag: 'devel-arm64', outname: 'r-ver'}
18-
- {image: 'ghcr.io/bioconductor/rocker-rstudio', amdtag: 'devel-amd64', armtag: 'devel-arm64', outname: 'bioconductor_docker'}
19-
- {image: 'rocker/tidyverse', amdtag: 'devel', armtag: 'N/A', outname: 'tidyverse'}
20-
- {image: 'ghcr.io/bioconductor/rocker-cuda', amdtag: 'devel-amd64', armtag: 'N/A', outname: 'cuda'}
21-
- {image: 'ghcr.io/bioconductor/rocker-ml', amdtag: 'devel-amd64', armtag: 'N/A', outname: 'ml'}
22-
- {image: 'ghcr.io/bioconductor/rocker-ml-verse', amdtag: 'devel', armtag: 'N/A', outname: 'ml-verse'}
23-
- {image: 'rocker/shiny', amdtag: 'devel', armtag: 'N/A', outname: 'shiny'}
24-
25-
name: Build branch images
26-
runs-on: ubuntu-latest
18+
- {image: 'ghcr.io/bioconductor/rocker-r-ver', amdtag: 'devel-amd64', outname: 'r-ver'}
19+
- {image: 'ghcr.io/bioconductor/rocker-rstudio', amdtag: 'devel-amd64', outname: 'bioconductor_docker'}
20+
- {image: 'rocker/tidyverse', amdtag: 'devel', outname: 'tidyverse'}
21+
- {image: 'ghcr.io/bioconductor/rocker-cuda', amdtag: 'devel-amd64', outname: 'cuda'}
22+
- {image: 'ghcr.io/bioconductor/rocker-ml', amdtag: 'devel-amd64', outname: 'ml'}
23+
- {image: 'ghcr.io/bioconductor/rocker-ml-verse', amdtag: 'devel', outname: 'ml-verse'}
24+
- {image: 'rocker/shiny', amdtag: 'devel', outname: 'shiny'}
2725
steps:
2826
- uses: actions/checkout@v3
29-
27+
3028
- name: Free root space
3129
uses: almahmoud/free-root-space@main
3230
with:
3331
verbose: true
3432

35-
- name: Set up QEMU
36-
uses: docker/setup-qemu-action@v2
37-
with:
38-
platforms: arm64
39-
4033
- name: Set up Docker Buildx
4134
uses: docker/setup-buildx-action@v2
4235

@@ -53,62 +46,201 @@ jobs:
5346
username: ${{ secrets.DOCKER_USERNAME }}
5447
password: ${{ secrets.DOCKER_PASSWORD }}
5548

56-
- name: Extract metadata for container image
49+
- name: Extract metadata
5750
id: meta
5851
uses: docker/metadata-action@v4
5952
with:
6053
images: ${{ github.repository_owner }}/${{ matrix.base.outname }}
6154
tags: |
6255
type=raw,value={{branch}}
6356
64-
- name: Set comma-separated list with all repository names
65-
id: images
57+
- name: Build and push AMD64 by digest
58+
id: build
59+
uses: docker/build-push-action@v3
60+
with:
61+
build-args: |
62+
BASE_IMAGE=${{ matrix.base.image }}
63+
amd64_tag=${{ matrix.base.amdtag }}
64+
file: Dockerfile
65+
platforms: linux/amd64
66+
labels: ${{ steps.meta.outputs.labels }}
67+
outputs: type=image,name=${{ github.repository_owner }}/${{ matrix.base.outname }},push-by-digest=true,name-canonical=true,push=true
68+
69+
- name: Export digest
6670
run: |
67-
IMG=${{ steps.meta.outputs.tags }}
68-
REPOLIST="docker.io/$IMG"
69-
REPOLIST="ghcr.io/$IMG,$REPOLIST"
70-
# Add tag with tag 3.xx as well as RELEASE_3_xx
71-
if [[ $IMG = *"RELEASE_"* ]]; then
72-
REPOLIST="$(echo $REPOLIST | sed 's/:RELEASE_3_/:3./g'),$REPOLIST"
73-
fi
74-
# Add tag with R version (based on amd64)
75-
docker pull ${{ matrix.base.image }}:${{ matrix.base.amdtag }}
76-
RVER=$(docker inspect ${{ matrix.base.image }}:${{ matrix.base.amdtag }} | jq -r '.[].Config.Env[]|select(match("^R_VERSION"))|.[index("=")+1:]')
77-
REPOLIST="$(echo $REPOLIST | sed "s/,/-R-$RVER,/g")-R-$RVER,$REPOLIST"
78-
# Also add alternative without _docker when in name
79-
if [[ $REPOLIST == *"_docker"* ]]; then
80-
REPOLIST="$(echo $REPOLIST | sed 's/_docker//g'),$REPOLIST"
81-
fi
82-
echo list=$REPOLIST >> $GITHUB_OUTPUT
71+
mkdir -p /tmp/digests
72+
digest="${{ steps.build.outputs.digest }}"
73+
touch "/tmp/digests/${digest#sha256:}"
74+
75+
- name: Upload digest
76+
uses: actions/upload-artifact@v4
77+
with:
78+
name: digests-${{ matrix.base.outname }}-amd64
79+
path: /tmp/digests/*
80+
if-no-files-found: error
81+
retention-days: 1
82+
83+
build-arm64:
84+
runs-on: ubuntu-latest-arm64
85+
strategy:
86+
fail-fast: false
87+
matrix:
88+
base:
89+
- {image: 'ghcr.io/bioconductor/rocker-r-ver', armtag: 'devel-arm64', outname: 'r-ver'}
90+
- {image: 'ghcr.io/bioconductor/rocker-rstudio', armtag: 'devel-arm64', outname: 'bioconductor_docker'}
91+
steps:
92+
- uses: actions/checkout@v3
93+
94+
- name: Free root space
95+
uses: almahmoud/free-root-space@main
96+
with:
97+
verbose: true
8398

8499
- name: Set up Docker Buildx
85100
uses: docker/setup-buildx-action@v2
101+
102+
- name: Login to GHCR
103+
uses: docker/login-action@v2
86104
with:
87-
platforms: linux/amd64,linux/arm64
105+
registry: ghcr.io
106+
username: ${{ github.actor }}
107+
password: ${{ secrets.GITHUB_TOKEN }}
88108

89-
- name: Build and push container image to all repos both arm64 and amd64
109+
- name: Login to Dockerhub
110+
uses: docker/login-action@v2
111+
with:
112+
username: ${{ secrets.DOCKER_USERNAME }}
113+
password: ${{ secrets.DOCKER_PASSWORD }}
114+
115+
- name: Extract metadata
116+
id: meta
117+
uses: docker/metadata-action@v4
118+
with:
119+
images: ${{ github.repository_owner }}/${{ matrix.base.outname }}
120+
tags: |
121+
type=raw,value={{branch}}
122+
123+
- name: Build and push ARM64 by digest
124+
id: build
90125
uses: docker/build-push-action@v3
91126
with:
92127
build-args: |
93128
BASE_IMAGE=${{ matrix.base.image }}
94129
arm64_tag=${{ matrix.base.armtag }}
95-
amd64_tag=${{ matrix.base.amdtag }}
96130
file: Dockerfile
97-
push: true
98-
tags: ${{ steps.images.outputs.list }}
131+
platforms: linux/arm64
99132
labels: ${{ steps.meta.outputs.labels }}
100-
platforms: linux/amd64,linux/arm64
133+
outputs: type=image,name=${{ github.repository_owner }}/${{ matrix.base.outname }},push-by-digest=true,name-canonical=true,push=true
134+
135+
- name: Export digest
136+
run: |
137+
mkdir -p /tmp/digests
138+
digest="${{ steps.build.outputs.digest }}"
139+
touch "/tmp/digests/${digest#sha256:}"
140+
141+
- name: Upload digest
142+
uses: actions/upload-artifact@v4
143+
with:
144+
name: digests-${{ matrix.base.outname }}-arm64
145+
path: /tmp/digests/*
146+
if-no-files-found: error
147+
retention-days: 1
148+
149+
merge:
150+
needs: [build-amd64, build-arm64]
151+
runs-on: ubuntu-latest
152+
strategy:
153+
matrix:
154+
base:
155+
- {image: 'ghcr.io/bioconductor/rocker-r-ver', armtag: 'devel-arm64', outname: 'r-ver'}
156+
- {image: 'ghcr.io/bioconductor/rocker-rstudio', armtag: 'devel-arm64', outname: 'bioconductor_docker'}
157+
- {image: 'rocker/tidyverse', armtag: 'N/A', outname: 'tidyverse'}
158+
- {image: 'ghcr.io/bioconductor/rocker-cuda', armtag: 'N/A', outname: 'cuda'}
159+
- {image: 'ghcr.io/bioconductor/rocker-ml', armtag: 'N/A', outname: 'ml'}
160+
- {image: 'ghcr.io/bioconductor/rocker-ml-verse', armtag: 'N/A', outname: 'ml-verse'}
161+
- {image: 'rocker/shiny', armtag: 'N/A', outname: 'shiny'}
162+
steps:
163+
- name: Set up Docker Buildx
164+
uses: docker/setup-buildx-action@v2
165+
166+
- name: Login to GHCR
167+
uses: docker/login-action@v2
168+
with:
169+
registry: ghcr.io
170+
username: ${{ github.actor }}
171+
password: ${{ secrets.GITHUB_TOKEN }}
172+
173+
- name: Login to Dockerhub
174+
uses: docker/login-action@v2
175+
with:
176+
username: ${{ secrets.DOCKER_USERNAME }}
177+
password: ${{ secrets.DOCKER_PASSWORD }}
178+
179+
- name: Download AMD64 digests
180+
uses: actions/download-artifact@v4
181+
with:
182+
name: digests-${{ matrix.base.outname }}-amd64
183+
path: /tmp/digests/amd64
184+
185+
- name: Download ARM64 digests
101186
if: matrix.base.armtag != 'N/A'
187+
uses: actions/download-artifact@v4
188+
with:
189+
name: digests-${{ matrix.base.outname }}-arm64
190+
path: /tmp/digests/arm64
102191

103-
- name: Build and push container image to all repos amd64 only
104-
uses: docker/build-push-action@v3
192+
- name: Set image tags
193+
id: meta
194+
uses: docker/metadata-action@v4
105195
with:
106-
build-args: |
107-
BASE_IMAGE=${{ matrix.base.image }}
108-
amd64_tag=${{ matrix.base.amdtag }}
109-
file: Dockerfile
110-
push: true
111-
tags: ${{ steps.images.outputs.list }}
112-
labels: ${{ steps.meta.outputs.labels }}
113-
platforms: linux/amd64
114-
if: matrix.base.armtag == 'N/A'
196+
images: |
197+
docker.io/${{ github.repository_owner }}/${{ matrix.base.outname }}
198+
ghcr.io/${{ github.repository_owner }}/${{ matrix.base.outname }}
199+
tags: |
200+
type=raw,value={{branch}}
201+
202+
- name: Create manifest list and push
203+
run: |
204+
# Prepare tags
205+
TAGS=$(echo '${{ steps.meta.outputs.tags }}' | tr '\n' ' ' | sed 's/ *$//')
206+
TAG_ARGS=""
207+
for tag in $TAGS; do
208+
TAG_ARGS="$TAG_ARGS -t $tag"
209+
done
210+
211+
# Add R version tags for ghcr.io images
212+
if [[ "${{ matrix.base.image }}" == *"ghcr.io"* ]]; then
213+
R_VER=$(docker pull ${{ matrix.base.image }}:${{ matrix.base.amdtag }} 2>/dev/null && \
214+
docker inspect ${{ matrix.base.image }}:${{ matrix.base.amdtag }} | \
215+
jq -r '.[].Config.Env[]|select(match("^R_VERSION"))|.[index("=")+1:]')
216+
if [ ! -z "$R_VER" ]; then
217+
for tag in $TAGS; do
218+
TAG_ARGS="$TAG_ARGS -t ${tag}-R-${R_VER}"
219+
done
220+
fi
221+
fi
222+
223+
# Add alternative tags without _docker in name
224+
if [[ "${{ matrix.base.outname }}" == *"_docker"* ]]; then
225+
for tag in $TAGS; do
226+
ALT_TAG=$(echo $tag | sed 's/_docker//')
227+
TAG_ARGS="$TAG_ARGS -t $ALT_TAG"
228+
done
229+
fi
230+
231+
# Create manifest list
232+
if [ "${{ matrix.base.armtag }}" != "N/A" ]; then
233+
# Both AMD64 and ARM64
234+
DIGESTS=$(cd /tmp/digests && find . -type f -exec echo "${{ github.repository_owner }}/${{ matrix.base.outname }}@sha256:{}" \; | sed 's/\.\//:/')
235+
else
236+
# AMD64 only
237+
DIGESTS=$(cd /tmp/digests/amd64 && find . -type f -exec echo "${{ github.repository_owner }}/${{ matrix.base.outname }}@sha256:{}" \; | sed 's/\.\//:/')
238+
fi
239+
240+
docker buildx imagetools create $TAG_ARGS $DIGESTS
241+
242+
- name: Inspect image
243+
run: |
244+
for tag in ${{ steps.meta.outputs.tags }}; do
245+
docker buildx imagetools inspect $tag
246+
done

0 commit comments

Comments
 (0)