feat: support for custom node selector (#12)

* allow node type to be null
* feat: support for custom node selector
* feat: support for node selector and arm build

we want jobs to be able to select different node types

Signed-off-by: vsoch <[email protected]>

Author: vsoch

.github/workflows/build-deploy.yaml

@@ -9,6 +9,34 @@ on:
     - main
 
 jobs:
+  # Only build arm image on merge, takes too long otherwise
+  build-arm:
+    if: (github.event_name != 'pull_request')
+    runs-on: ubuntu-latest
+    name: make and build arm
+    steps:
+    - name: Checkout Repository
+      uses: actions/checkout@v4
+    - uses: actions/setup-go@v3
+      with:
+        go-version: ^1.22
+    - name: GHCR Login
+      if: (github.event_name != 'pull_request')
+      uses: docker/login-action@v2
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Add custom buildx ARM builder
+      run: |
+       docker buildx create --name armbuilder
+       docker buildx use armbuilder
+       docker buildx inspect --bootstrap
+
+    - name: Deploy Container
+      run: make arm-deploy
+
   build-manager:
     permissions:
       packages: write

Makefile

@@ -41,6 +41,7 @@ endif
 OPERATOR_SDK_VERSION ?= v1.38.0
 # Image URL to use all building/pushing image targets
 IMG ?= ghcr.io/converged-computing/state-machine-operator:latest
+ARMIMG ?= ghcr.io/converged-computing/state-machine-operator:arm
 DEVIMG ?= ghcr.io/converged-computing/state-machine-operator:test
 MANAGER_IMG ?= ghcr.io/converged-computing/state-machine-operator:manager
 
@@ -276,6 +277,16 @@ golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary.
 $(GOLANGCI_LINT): $(LOCALBIN)
 	$(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint,$(GOLANGCI_LINT_VERSION))
 
+.PHONY: arm-build
+arm-build: test ## Build docker image with the manager.
+	docker buildx build --platform linux/arm64 -t ${ARMIMG} .
+
+.PHONY: arm-deploy
+arm-deploy: manifests kustomize
+	docker buildx build --platform linux/arm64 --push -t ${ARMIMG} .
+	cd config/manager && $(KUSTOMIZE) edit set image controller=${ARMIMG}
+	$(KUSTOMIZE) build config/default > examples/dist/state-machine-operator-arm.yaml
+
 # go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist
 # $1 - target path with name of binary
 # $2 - package url which can be installed

README.md

@@ -90,6 +90,8 @@ For each job script section, the following environment variables are provided fo
 - registry: the registry where your artifact will be pushed
   - pull_tag: the pull tag to use (if the workflow is pulling)
   - push_tag: the push tag to use (if the workflow is pushing)
+- properties:
+  - node-selector: key value pair to be added as node selectors for the job (Kubernetes only). E.g., `node.kubernetes.io/instance-type: c7a.4xlarge`
 
 Take a look at the simple example [examples/state-machine.yaml](examples/state-machine.yaml) to see how push/pull is defined between steps. Given that these are found (with a tag) your artifact will be named `<registry>:<jobid>:<tag>` to be moved between steps.
 

api/v1alpha1/statemachine_types.go

@@ -96,6 +96,10 @@ type JobStep struct {
 	// + optional
 	AppConfig string `json:"appConfig,omitempty"`
 
+	// Properties for the tracker, etc.
+	// +optional
+	Properties map[string]string `json:"properties,omitempty"`
+
 	// Namespace is inherited from StateMachine Spec
 	// This is the continer image (required)
 	Image string `json:"image,omitempty"`

api/v1alpha1/zz_generated.deepcopy.go

@@ -117,6 +117,11 @@ spec:
                     name:
                       description: Name is the name of the job (required)
                       type: string
+                    properties:
+                      additionalProperties:
+                        type: string
+                      description: Properties for the tracker, etc.
+                      type: object
                     registry:
                       description: Configuration for the job registry
                       properties:

config/crd/bases/state-machine.converged-computing.org_statemachines.yaml

@@ -125,6 +125,11 @@ spec:
                     name:
                       description: Name is the name of the job (required)
                       type: string
+                    properties:
+                      additionalProperties:
+                        type: string
+                      description: Properties for the tracker, etc.
+                      type: object
                     registry:
                       description: Configuration for the job registry
                       properties:

examples/dist/state-machine-operator-dev.yaml

@@ -6,7 +6,9 @@ name: {{ .Job.Name }}
 image: {{ .Job.Image }}
 namespace: {{ if .StateMachine.Namespace }}{{ .StateMachine.Namespace }}{{ else }}default{{ end }}
 workdir:  {{ if .Job.Workdir }}{{ .Job.Workdir }}{{ else }}/tmp/out{{ end }}
-
+{{ if .Job.Properties }}properties: {{range $key, $value := .Job.Properties }}
+ {{ $key }}: {{ $value }}
+{{ end }}{{ end }}
 
 {{ if .Job.HasRegistry }}
 registry:

internal/controller/manager/jobs/templates/job.yaml

@@ -81,6 +81,7 @@
         "script": {"type": "string"},
         "image": {"type": "string"},
         "registry": {"$ref": "#/definitions/registry"},
+        "properties": {"type": ["object", "null"]},
         "workdir": {"type": "string", "default": "/tmp/out"},
         "additionalProperties": False,
     },

python/state_machine_operator/schema.py

@@ -90,6 +90,18 @@ def delete_configmap(self, name):
             except Exception as e:
                 LOGGER.warning(f"Issue deleting configmap {name}: {e}")
 
+    def get_node_selector(self):
+        """
+        Node selector is in properties -> node-selector
+        """
+        # Properties can be provided as a string to json load
+        props = self.job_desc.get("properties", {})
+        if isinstance(props, str):
+            props = json.loads(props)
+        if not props:
+            return props
+        return props.get("node-selector")
+
     def generate_batch_job(self, step, jobid):
         """
         Generate the job CRD assuming the config map entrypoint.
@@ -186,6 +198,12 @@ def generate_batch_job(self, step, jobid):
             },
         }
 
+        # Add node selectors? E.g.,
+        # node.kubernetes.io/instance-type: c7a.4xlarge
+        node_selector = self.get_node_selector()
+        if node_selector is not None:
+            template["spec"]["nodeSelector"] = node_selector
+
         # Only add walltime if it's > 0 and not None
         if walltime:
             template["spec"]["activeDeadlineSeconds"] = int(walltime)