[Misc] Batch API envoy integration fix, E2E verification, and document update

build/container/Dockerfile.metadata

@@ -45,6 +45,6 @@ RUN apt-get update \
 # Expose metadata service port
 EXPOSE 8090
 
-# Run metadata server
-CMD ["python", "-m", "aibrix.metadata.app", "--host", "0.0.0.0", "--port", "8090"]
+# Set entrypoint for Metadata service
+ENTRYPOINT ["aibrix_metadata", "--host", "0.0.0.0"]
 

config/gateway/gateway-plugin/gateway-plugin.yaml

@@ -134,7 +134,7 @@ spec:
 apiVersion: gateway.networking.k8s.io/v1
 kind: HTTPRoute
 metadata:
-  name: reserved-router-models-endpoint
+  name: reserved-router-metadata-endpoint
   namespace: system
 spec:
   parentRefs:
@@ -144,6 +144,12 @@ spec:
         - path:
             type: PathPrefix
             value: /v1/models
+        - path:
+            type: PathPrefix
+            value: /v1/files
+        - path:
+            type: PathPrefix
+            value: /v1/batches
       backendRefs:
         - name: aibrix-metadata-service
           port: 8090
@@ -157,7 +163,7 @@ spec:
   targetRef:
     group: gateway.networking.k8s.io
     kind: HTTPRoute
-    name: aibrix-reserved-router-models-endpoint
+    name: aibrix-reserved-router-metadata-endpoint
 ---
 # this is a dummy route for incoming request and,
 # then request is routed to httproute using model name OR

config/job/kustomization.yaml

@@ -0,0 +1,4 @@
+kind: Kustomization
+
+resources:
+- k8s_job_rbac.yaml

config/metadata/job_template_patch.yaml

@@ -0,0 +1,24 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: batch-job-template
+  namespace: default
+spec:
+  parallelism: 1 # Customizable. The number of parallel workers.
+  completions: 1 # Customizable. Must equal to the parallelism.
+  backoffLimit: 2 # Customizable, but usually no need to change.
+  template:
+    spec:
+      containers:
+      - name: batch-worker
+        image: aibrix/runtime:nightly # Customizable, batch job worker image
+      - name: llm-engine
+        image: aibrix/vllm-mock:nightly # Customizable, customize your LLM engine image
+        # command: ["/bin/sh", "-c"] # Customization is not recommended. Know what you are doing.
+        args: # Customizable in the format of "WORKER_VICTIM=1 [your command] || true"
+          - |
+            # Run llm engine. 
+            # 'WORKER_VICTIM=1' helps the batch-worker to identify llm-engine process.
+            # '|| true' at the end ensures the container llm-engine never fails.
+            WORKER_VICTIM=1 python app.py || true
+        readinessProbe: # Customizable, customize your readinessProbe

config/metadata/kustomization.yaml

@@ -2,6 +2,18 @@ resources:
 - metadata.yaml
 - redis.yaml
 
+configMapGenerator:
+- name: metadata-config
+  namespace: aibrix-system
+  files:
+  - job_template_patch.yaml
+
+patches:
+# Uncomment the following lines to enable S3 as the object store
+# - path: s3-env-patch.yaml
+# Uncomment the following lines to enable TOS as the object store
+# - path: tos-env-patch.yaml
+
 labels:
   - pairs:
       app.kubernetes.io/component: aibrix-metadata-service

config/metadata/metadata.yaml

@@ -30,6 +30,20 @@ rules:
   - apiGroups: ["model.aibrix.ai"]
     resources: ["modeladapters"]
     verbs: ["get", "list"]
+  # For batch job watching
+  - apiGroups: ["batch"] 
+    resources: ["jobs"]
+    verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+  # For kopf high availability
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+  - apiGroups: ["apiextensions.k8s.io"] # required by kopf
+    resources: ["customresourcedefinitions"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""] # required by kopf
+    resources: ["namespaces"]
+    verbs: ["list", "watch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -64,14 +78,21 @@ spec:
         - name: init-redis
           image: busybox
           command: ['sh', '-c', 'until echo "ping" | nc aibrix-redis-master 6379 -w 1 | grep -c PONG; do echo waiting for redis; sleep 2; done']
+      volumes:
+        - name: config-volume
+          configMap:
+            name: metadata-config
       containers:
         - name: metadata-service
           image: metadata-service:latest
           imagePullPolicy: IfNotPresent
-          command: ["python", "-m", "aibrix.metadata.app"]
-          args: ["--host=0.0.0.0", "--port=8090"]
+          # Enable S3 or TOS to enable-k8s-job, default disabled.
           ports:
             - containerPort: 8090
+          volumeMounts:
+            - name: config-volume
+              mountPath: /app/config
+              readOnly: true
           resources:
             limits:
               cpu: 500m

config/metadata/s3-env-patch.yaml

@@ -0,0 +1,40 @@
+# This patch contains the S3 object store configuration for the metadata service
+# !!Important: Please make sure aibrix-s3-credentials secret exists in both aibrix-system and default namespaces.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: metadata-service
+  namespace: aibrix-system
+spec:
+  template:
+    spec:
+      containers:
+      - name: metadata-service
+        command: 
+          - aibrix_metadata
+          - --host
+          - "0.0.0.0"
+          - --enable-k8s-job
+          - --k8s-job-patch
+          - /app/config/job_template_patch.yaml
+        env:
+          - name: STORAGE_AWS_ACCESS_KEY_ID
+            valueFrom:
+              secretKeyRef:
+                name: aibrix-s3-credentials
+                key: access-key-id
+          - name: STORAGE_AWS_SECRET_ACCESS_KEY
+            valueFrom:
+              secretKeyRef:
+                name: aibrix-s3-credentials
+                key: secret-access-key
+          - name: STORAGE_AWS_REGION
+            valueFrom:
+              secretKeyRef:
+                name: aibrix-s3-credentials
+                key: region
+          - name: STORAGE_AWS_BUCKET
+            valueFrom:
+              secretKeyRef:
+                name: aibrix-s3-credentials
+                key: bucket-name

config/metadata/tos-env-patch.yaml

@@ -0,0 +1,45 @@
+# This patch contains the TOS object store configuration for the metadata service
+# !!Important: Please make sure aibrix-tos-credentials secret exists in both aibrix-system and default namespaces.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: metadata-service
+  namespace: aibrix-system
+spec:
+  template:
+    spec:
+      containers:
+      - name: metadata-service
+        command: 
+          - aibrix_metadata
+          - --host
+          - "0.0.0.0"
+          - --enable-k8s-job
+          - --k8s-job-patch
+          - /app/config/job_template_patch.yaml
+        env:
+          - name: STORAGE_TOS_ACCESS_KEY
+            valueFrom:
+            secretKeyRef:
+              name: aibrix-tos-credentials
+              key: access-key
+          - name: STORAGE_TOS_SECRET_KEY
+            valueFrom:
+              secretKeyRef:
+                name: aibrix-tos-credentials
+                key: secret-key
+          - name: STORAGE_TOS_ENDPOINT
+            valueFrom:
+              secretKeyRef:
+                name: aibrix-tos-credentials
+                key: endpoint
+          - name: STORAGE_TOS_REGION
+            valueFrom:
+              secretKeyRef:
+                name: aibrix-tos-credentials
+                key: region
+          - name: STORAGE_TOS_BUCKET
+            valueFrom:
+              secretKeyRef:
+                name: aibrix-tos-credentials
+                key: bucket-name

dist/chart/templates/gateway-plugin/envoy_extension_policy.yaml

@@ -35,4 +35,4 @@ spec:
   targetRef:
     group: gateway.networking.k8s.io
     kind: HTTPRoute
-    name: aibrix-reserved-router-models-endpoint
+    name: aibrix-reserved-router-metadata-endpoint

dist/chart/templates/gateway-plugin/httproute.yaml

@@ -33,7 +33,7 @@ spec:
 apiVersion: gateway.networking.k8s.io/v1
 kind: HTTPRoute
 metadata:
-  name: aibrix-reserved-router-models-endpoint
+  name: aibrix-reserved-router-metadata-endpoint
   namespace: {{ .Release.Namespace }}
   labels:
     {{- include "chart.labels" . | nindent 4 }}
@@ -46,6 +46,12 @@ spec:
         - path:
             type: PathPrefix
             value: /v1/models
+        - path:
+            type: PathPrefix
+            value: /v1/files
+        - path:
+            type: PathPrefix
+            value: /v1/batches
       backendRefs:
         - name: aibrix-metadata-service
           port: 8090

dist/chart/templates/metadata-service/deployment.yaml

@@ -31,8 +31,12 @@ spec:
         - name: metadata-service
           image: {{ .Values.metadata.service.container.image.repository }}:{{ .Values.metadata.service.container.image.tag }}
           imagePullPolicy: {{ .Values.metadata.service.container.image.imagePullPolicy | default "IfNotPresent" }}
-          command: ["python", "-m", "aibrix.metadata.app"]
-          args: ["--host=0.0.0.0", "--port=8090"]
+          command:
+            - python
+            - -m
+            - aibrix.metadata.app
+            - --host
+            - "0.0.0.0"
           ports:
             - containerPort: 8090
           resources: {{ toYaml .Values.metadata.service.container.resources | nindent 12 }}

dist/chart/templates/metadata-service/rbac.yaml

@@ -24,6 +24,20 @@ rules:
   - apiGroups: ["model.aibrix.ai"]
     resources: ["modeladapters"]
     verbs: ["get", "list"]
+  # For batch job watching
+  - apiGroups: ["batch"] 
+    resources: ["jobs"]
+    verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+  # For kopf high availability
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+  - apiGroups: ["apiextensions.k8s.io"] # required by kopf
+    resources: ["customresourcedefinitions"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""] # required by kopf
+    resources: ["namespaces"]
+    verbs: ["list", "watch"]
 ---
 
 apiVersion: rbac.authorization.k8s.io/v1