Merge branch 'kubeflow:master' into support_kai

Author: Harshal292004

.github/workflows/template-publish-image/action.yaml

@@ -81,6 +81,7 @@ runs:
       with:
         images: ${{ inputs.image }}
         tags: |
+          type=ref,event=tag
           type=raw,latest
           type=sha
 

.github/workflows/test-e2e.yaml

@@ -42,7 +42,7 @@ jobs:
           pip install papermill==2.6.0 jupyter==1.1.1 ipykernel==6.29.5
 
           echo "Install Kubeflow SDK"
-          pip install ./sdk
+          pip install git+https://github.com/kubeflow/sdk.git@main#subdirectory=python
 
       - name: Setup cluster
         run: |

.gitignore

@@ -20,9 +20,6 @@ artifacts
 __pycache__/
 *.egg-info/
 
-# OpenAPI Generator CLI JAR file
-hack/python-sdk/openapi-generator-cli.jar
-
 # Coverage
 cover.out
 

.pre-commit-config.yaml

@@ -26,9 +26,5 @@ exclude: |
   (?x)^(
     docs/images/.*|
     pkg/client/.*|
-    sdk/kubeflow/trainer/__init__.py|
-    sdk/kubeflow/trainer/api/__init__.py|
-    sdk/kubeflow/trainer/models/.*|
     api/python_api/kubeflow_trainer_api/models/.*|
-    sdk/docs/.*
   )$

CHANGELOG.md

@@ -21,7 +21,7 @@ Note for Lima the link is to the Adopters, which supports several different cont
 The Kubeflow Trainer project includes a Makefile with several helpful commands to streamline your development workflow:
 
 ```sh
-# Generate manifests, APIs and SDK
+# Generate manifests and APIs.
 make generate
 ```
 
@@ -215,24 +215,6 @@ kubectl logs -n kubeflow -l training.kubeflow.org/job-name=pytorch-simple --foll
 
 ### SDK Development
 
-To generate Python SDK for the operator, run:
+Changes to the Kubeflow Trainer Python SDK can be made in the https://github.com/kubeflow/sdk repo.
 
-```sh
-./hack/python-sdk/gen-sdk.sh
-```
-
-This command will re-generate the api and model files together with the documentation and model tests.
-The following files/folders in `sdk/python` are auto-generated and should not be modified directly:
-
-```
-sdk/python/docs
-sdk/python/kubeflow/training/models
-sdk/python/kubeflow/training/*.py
-sdk/python/test/*.py
-```
-
-The Training Operator client and public APIs are located here:
-
-```
-sdk/python/kubeflow/training/api
-```
+The Trainer SDK can be found at https://github.com/kubeflow/sdk/tree/main/python/kubeflow/trainer.

CONTRIBUTING.md

@@ -29,6 +29,9 @@ HELM_CHART_TESTING_VERSION ?= v3.12.0
 HELM_DOCS_VERSION ?= v1.14.2
 YQ_VERSION ?= v4.45.1
 
+# Container runtime (docker or podman)
+CONTAINER_RUNTIME ?=
+
 # Tool binaries
 GINKGO ?= $(LOCALBIN)/ginkgo
 ENVTEST ?= $(LOCALBIN)/setup-envtest
@@ -117,13 +120,11 @@ manifests: controller-gen ## Generate manifests.
 		output:rbac:artifacts:config=manifests/base/rbac \
 		output:webhook:artifacts:config=manifests/base/webhook
 
-## TODO (kramaranya): Remove gen-sdk.sh when moving SDK
 .PHONY: generate
-generate: go-mod-download manifests ## Generate APIs and SDK.
+generate: go-mod-download manifests ## Generate APIs.
 	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate/boilerplate.go.txt" paths="./pkg/apis/..."
 	hack/update-codegen.sh
-	hack/python-api/gen-api.sh
-	hack/python-sdk/gen-sdk.sh
+	CONTAINER_RUNTIME=$(CONTAINER_RUNTIME) hack/python-api/gen-api.sh
 
 .PHONY: go-mod-download
 go-mod-download: ## Run go mod download to download modules.
@@ -160,7 +161,6 @@ test-integration: ginkgo envtest jobset-operator-crd scheduler-plugins-crd ## Ru
 test-python: ## Run Python unit test.
 	pip install pytest
 	pip install -r ./cmd/initializers/dataset/requirements.txt
-	pip install ./sdk
 
 	PYTHONPATH=$(PROJECT_DIR) pytest ./pkg/initializers/dataset
 	PYTHONPATH=$(PROJECT_DIR) pytest ./pkg/initializers/model

Makefile

@@ -1,12 +1,13 @@
 approvers:
   - andreyvelich
+  - Electronic-Waste
   - gaocegege
   - Jeffwan
   - johnugeorge
   - tenzen-y
   - terrytangyuan
 reviewers:
-  - Electronic-Waste
+  - astefanutti
   - jinchihe
   - kuizhiqing
 emeritus_approvers:

OWNERS

@@ -3,6 +3,7 @@
 [![Build Status](https://github.com/kubeflow/trainer/actions/workflows/test-go.yaml/badge.svg?branch=master)](https://github.com/kubeflow/trainer/actions/workflows/test-go.yaml?branch=master)
 [![Coverage Status](https://coveralls.io/repos/github/kubeflow/trainer/badge.svg?branch=master)](https://coveralls.io/github/kubeflow/trainer?branch=master)
 [![Go Report Card](https://goreportcard.com/badge/github.com/kubeflow/trainer)](https://goreportcard.com/report/github.com/kubeflow/trainer)
+[![OpenSSF Best Practices](https://www.bestpractices.dev/projects/10435/badge)](https://www.bestpractices.dev/projects/10435)
 
 <h1 align="center">
     <img src="./docs/images/trainer-logo.svg" alt="logo" width="200">
@@ -19,8 +20,9 @@ You can integrate other ML libraries such as [HuggingFace](https://huggingface.c
 [DeepSpeed](https://github.com/microsoft/DeepSpeed), or [Megatron-LM](https://github.com/NVIDIA/Megatron-LM)
 with Kubeflow Training to orchestrate their ML training on Kubernetes.
 
-Kubeflow Trainer allows you effortlessly develop your LLMs with the Kubeflow Python SDK and
-build Kubernetes-native Training Runtimes with Kubernetes Custom Resources APIs.
+Kubeflow Trainer enables you to effortlessly develop your LLMs with the
+[Kubeflow Python SDK](https://github.com/kubeflow/sdk/), and build Kubernetes-native Training
+Runtimes using Kubernetes Custom Resource APIs.
 
 <h1 align="center">
     <img src="./docs/images/trainer-tech-stack.drawio.svg" alt="logo" width="500">

README.md

@@ -13155,32 +13155,14 @@
         }
       },
       "trainer.v1alpha1.ContainerOverride": {
-        "description": "ContainerOverride represents parameters that can be overridden using PodSpecOverrides. Parameters from the Trainer, DatasetConfig, and ModelConfig will take precedence.",
+        "description": "ContainerOverride represents parameters that can be overridden using PodSpecOverrides.",
         "type": "object",
         "required": [
           "name"
         ],
         "properties": {
-          "args": {
-            "description": "Arguments to the entrypoint for the training container.",
-            "type": "array",
-            "items": {
-              "type": "string",
-              "default": ""
-            },
-            "x-kubernetes-list-type": "atomic"
-          },
-          "command": {
-            "description": "Entrypoint commands for the training container.",
-            "type": "array",
-            "items": {
-              "type": "string",
-              "default": ""
-            },
-            "x-kubernetes-list-type": "atomic"
-          },
           "env": {
-            "description": "List of environment variables to set in the container. These values will be merged with the TrainingRuntime's environments.",
+            "description": "List of environment variables to set in the container. These values will be merged with the TrainingRuntime's environments. These values can't be set for container with the name: `node`, `dataset-initializer`, or `model-initializer`. For those containers the envs can only be set via Trainer or Initializer APIs.",
             "type": "array",
             "items": {
               "default": {},
@@ -13195,19 +13177,6 @@
             ],
             "x-kubernetes-list-type": "map"
           },
-          "envFrom": {
-            "description": "List of sources to populate environment variables in the container. These   values will be merged with the TrainingRuntime's environments.",
-            "type": "array",
-            "items": {
-              "default": {},
-              "allOf": [
-                {
-                  "$ref": "#/components/schemas/io.k8s.api.core.v1.EnvFromSource"
-                }
-              ]
-            },
-            "x-kubernetes-list-type": "atomic"
-          },
           "name": {
             "description": "Name for the container. TrainingRuntime must have this container.",
             "type": "string",
@@ -13544,7 +13513,7 @@
             "x-kubernetes-list-type": "map"
           },
           "nodeSelector": {
-            "description": "Override for the node selector to place Pod on the specific mode.",
+            "description": "Override for the node selector to place Pod on the specific node.",
             "type": "object",
             "additionalProperties": {
               "type": "string",
@@ -13582,7 +13551,7 @@
             "x-kubernetes-list-type": "atomic"
           },
           "volumes": {
-            "description": "Overrides for the Pod volume configuration.",
+            "description": "Overrides for the Pod volume configurations.",
             "type": "array",
             "items": {
               "default": {},