Skip to content

Commit b16c950

Browse files
mathis-marcottemathis-marcotte
committed
feat: add docker-bake to build process (#135)
Adds docker bake as the solution for building our images.
1 parent 4aaaa93 commit b16c950

File tree

14 files changed

+296
-190
lines changed

14 files changed

+296
-190
lines changed

‎.github/workflows/docker-build-upload.yaml‎

Lines changed: 15 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,6 @@ env:
66
on:
77
workflow_call:
88
inputs:
9-
parent-image:
10-
description: Parent image name
11-
required: true
12-
type: string
139
directory:
1410
description: The directory of the image files
1511
required: true
@@ -78,39 +74,29 @@ jobs:
7874
username: ${{ secrets.REGISTRY_USERNAME }}
7975
password: ${{ secrets.REGISTRY_PASSWORD }}
8076

81-
- name: Pull parent image
82-
id: pull-parent
83-
if: inputs.parent-image != ''
84-
run: make pull/${{ inputs.parent-image }} REPO=${{ inputs.registry-name }}
85-
86-
- name: Set BASE_IMAGE variable
87-
run: |
88-
if [ "${{ inputs.base-image }}" == "" ]; then
89-
echo "BASE_IMAGE=${{ steps.pull-parent.outputs.image_name }}" >> $GITHUB_ENV
90-
else
91-
echo "BASE_IMAGE=${{ inputs.base-image }}" >> $GITHUB_ENV
92-
fi
93-
94-
- name: Set FROM and as in Dockerfile
77+
- name: Set short SHA
9578
run: |
96-
sed -i '1i FROM ${{ env.BASE_IMAGE}} as ${{ inputs.image }}' ./images/${{ inputs.directory }}/Dockerfile
79+
shortSha=$(git rev-parse --short ${{ github.sha}})
80+
echo "COMMIT_SHORT_SHA=$shortSha"
81+
echo "COMMIT_SHORT_SHA=$shortSha" >> $GITHUB_ENV
9782
98-
# make build emits full_image_name, image_tag, and image_repo outputs
9983
- name: Build image
10084
id: build-image
101-
run: make build/${{ inputs.image }} REPO=${{ inputs.registry-name }} DIRECTORY=${{ inputs.directory }} DARGS="--build-arg ACCOUNT_NAME=${{ secrets.SAS_SA_NAME }} --build-arg SRC_ACCOUNT_KEY=${{ secrets.SAS_SA_KEY_1 }}"
85+
run: |
86+
tags="${{inputs.branch-name}} ${{env.COMMIT_SHORT_SHA}} ${{github.sha}}"
87+
88+
# adds tags for master
89+
if [ "${{ inputs.branch-name }}" == 'master' ]; then
90+
tags="${tags} v2 latest"
91+
fi
92+
93+
make bake/${{ inputs.image }} BASE_IMAGE=${{ inputs.base-image }} REPO=${{ inputs.registry-name }} \
94+
TAGS="${tags}" DARGS="--set ${{ inputs.image }}.args.ACCOUNT_NAME=${{ secrets.SAS_SA_NAME }} \
95+
--set ${{ inputs.image }}.args.SRC_ACCOUNT_KEY=${{ secrets.SAS_SA_KEY_1 }}"
10296
10397
- name: Echo disk usage after build completion
10498
run: ./.github/scripts/echo_usage.sh
10599

106-
- name: Add standard tags (short sha, sha, and branch) and any other post-build activity
107-
if: ${{ inputs.branch-name != 'master' }}
108-
run: make post-build/${{ inputs.image }} REPO=${{ inputs.registry-name }}
109-
110-
- name: Add master and standard tags (v2, latest; short sha, sha, and branch) and any other post-build activity
111-
if: ${{ inputs.branch-name == 'master' }}
112-
run: make post-build/${{ inputs.image }} REPO=${{ inputs.registry-name }} IMAGE_VERSION=v2 IS_LATEST=true
113-
114100
- name: Push image to registry (default pushes all tags)
115101
run: make push/${{ inputs.image }} REPO=${{ inputs.registry-name }}
116102

‎.github/workflows/docker-pull-test.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ jobs:
5959

6060
- name: Pull existing image
6161
id: pull-existing
62-
run: make pull/${{ inputs.image }} REPO=${{ inputs.registry-name }} TAG=${{ inputs.tag }}
62+
run: docker pull ${{ inputs.registry-name }}/${{ inputs.image }}:${{ inputs.tag }}
6363

6464
- name: Set Up Python for Test Suite
6565
uses: actions/setup-python@v4

‎.github/workflows/docker-pull-upload.yaml‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -48,12 +48,15 @@ jobs:
4848
- name: Pull existing image
4949
id: pull-existing
5050
run: |
51+
image_name=${{ inputs.registry-name }}/${{ inputs.image }}:v2
5152
if [ "${{ github.event.pull_request.base.ref }}" == "beta" ]; then
52-
make pull/${{ inputs.image }} REPO=${{ inputs.registry-name }} TAG=beta
53-
else
54-
make pull/${{ inputs.image }} REPO=${{ inputs.registry-name }} TAG=v2
53+
image_name=${{ inputs.registry-name }}/${{ inputs.image }}:beta
5554
fi
5655
56+
docker pull ${image_name}
57+
58+
echo "image_name=$image_name" >> $GITHUB_OUTPUT
59+
5760
- name: Retag existing image
5861
run: make post-build/${{ inputs.image }} REPO=${{ inputs.registry-name }} SOURCE_FULL_IMAGE_NAME=${{ steps.pull-existing.outputs.image_name }}
5962

‎.github/workflows/docker-steps.yaml‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -58,24 +58,37 @@ jobs:
5858
parent-image-is-diff: ${{ inputs.parent-image-is-diff }}
5959
branch-name: ${{ inputs.branch-name }}
6060

61-
print-variables:
61+
set-and-print-variables:
6262
needs: [check-diff]
6363
runs-on: ubuntu-latest
64+
outputs:
65+
base-image: ${{ steps.set-base.outputs.base-image }}
6466
steps:
6567
- name: print inputs
6668
run: echo "${{ toJSON(inputs) }}"
6769
- name: print check-diff
6870
run: echo "${{ toJSON(needs.check-diff) }}"
71+
- name: set BASE_IMAGE env
72+
id: set-base
73+
# if base image is given, then it is from upstream
74+
# otherwise, set the base image to use our registry
75+
run: |
76+
if [ "${{ inputs.base-image }}" != "" ]; then
77+
echo "BASE_IMAGE=${{ inputs.base-image }}"
78+
echo "base-image=${{ inputs.base-image }}" >> $GITHUB_OUTPUT
79+
else
80+
echo "BASE_IMAGE=${{ inputs.registry-name }}/${{ inputs.parent-image }}:${{ inputs.branch-name }}"
81+
echo "base-image=${{ inputs.registry-name }}/${{ inputs.parent-image }}:${{ inputs.branch-name }}" >> $GITHUB_OUTPUT
82+
fi
6983
7084
build-upload:
71-
needs: [check-diff]
85+
needs: [check-diff, set-and-print-variables]
7286
if: ${{ needs.check-diff.outputs.is-diff == 'true' }}
7387
uses: ./.github/workflows/docker-build-upload.yaml
7488
with:
75-
parent-image: ${{ inputs.parent-image }}
7689
directory: ${{ inputs.directory }}
7790
image: ${{ inputs.image }}
78-
base-image: ${{ inputs.base-image }}
91+
base-image: ${{ needs.set-and-print-variables.outputs.base-image }}
7992
registry-name: ${{ inputs.registry-name }}
8093
branch-name: ${{ inputs.branch-name }}
8194
secrets:

‎Makefile‎

Lines changed: 9 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,13 @@ pytorch-CUDA := 11.8.0
1818

1919
# Misc Directories
2020
TESTS_DIR := ./tests
21-
MAKE_HELPERS := ./make_helpers/
21+
MAKE_HELPERS := ./make_helpers
2222
PYTHON_VENV := .venv
2323

2424
# Executables
2525
PYTHON := $(PYTHON_VENV)/bin/python
2626
POST_BUILD_HOOK := post-build-hook.sh
27+
BAKE_BUILD_EXEC := bake-build.sh
2728

2829
# Default labels
2930
DEFAULT_REPO := k8scc01covidacr.azurecr.io
@@ -41,34 +42,13 @@ DEFAULT_NB_PREFIX := /notebook/username/notebookname
4142
###### Docker helpers ######
4243
###################################
4344

44-
pull/%: GITHUB_OUTPUT ?= .tmp/github_output.log
45-
pull/%: DARGS?=
46-
pull/%: REPO?=$(DEFAULT_REPO)
47-
pull/%: TAG?=$(DEFAULT_TAG)
48-
pull/%:
49-
# End repo with a single slash and start tag with a single colon, if they exist
50-
REPO=$$(echo "$(REPO)" | sed 's:/*$$:/:' | sed 's:^\s*/*\s*$$::') &&\
51-
TAG=$$(echo "$(TAG)" | sed 's~^:*~:~' | sed 's~^\s*:*\s*$$~~') &&\
52-
IMAGE_NAME="$${REPO}$(notdir $@):$(TAG)" && \
53-
echo "Pulling $$IMAGE_NAME" &&\
54-
docker pull $(DARGS) $$IMAGE_NAME &&\
55-
echo "image_name=$$IMAGE_NAME" >> $(GITHUB_OUTPUT)
56-
57-
build/%: GITHUB_OUTPUT ?= .tmp/github_output.log
58-
build/%: DIRECTORY?=
59-
build/%: DARGS?=
60-
build/%: REPO?=$(DEFAULT_REPO)
61-
build/%: TAG?=$(DEFAULT_TAG)
62-
build/%: ## build the latest image
63-
# End repo with exactly one trailing slash, unless it is empty
64-
REPO=$$(echo "$(REPO)" | sed 's:/*$$:/:' | sed 's:^\s*/*\s*$$::') && \
65-
IMAGE_NAME="$${REPO}$(notdir $@):$(TAG)" && \
66-
DOCKER_BUILDKIT=1 docker build $(DARGS) --rm --force-rm -t $$IMAGE_NAME ./images/$(DIRECTORY) && \
67-
echo -n "Built image $$IMAGE_NAME of size: " && \
68-
docker images $$IMAGE_NAME --format "{{.Size}}" && \
69-
echo "full_image_name=$$IMAGE_NAME" >> $(GITHUB_OUTPUT) && \
70-
echo "image_tag=$(TAG)" >> $(GITHUB_OUTPUT) && \
71-
echo "image_repo=$${REPO}" >> $(GITHUB_OUTPUT)
45+
bake/%: export DARGS?=
46+
bake/%: export BASE_IMAGE?=
47+
bake/%: export REPO?=$(DEFAULT_REPO)
48+
bake/%: export TAGS?=
49+
bake/%: ## build the desired image with docker bake
50+
IMAGE_NAME="$(notdir $@)" \
51+
bash "$(MAKE_HELPERS)/$(BAKE_BUILD_EXEC)"
7252

7353
post-build/%: export REPO?=$(DEFAULT_REPO)
7454
post-build/%: export TAG?=$(DEFAULT_TAG)

0 commit comments

Comments
 (0)