Add monitor flag to common DO service deployment

docker/README.md

@@ -658,6 +658,8 @@ services:
 | `FIFTYONE_DELEGATED_OPERATION_RUN_LINK_PATH` | Full path to a network-mounted file system or a cloud storage path to use for storing logs generated by delegated operation runs, one file per job. The default `null` means log upload is disabled.                                                                           | No                        |
 | `FIFTYONE_DEFAULT_APP_ADDRESS`               | The host address that `fiftyone-app` should bind to; `127.0.0.1` is appropriate in most cases                                                                                                                                                                                  | Yes                       |
 | `FIFTYONE_DEFAULT_APP_PORT`                  | The host port that `fiftyone-app` should bind to; the default is `5151`                                                                                                                                                                                                        | Yes                       |
+| `FIFTYONE_DO_EXPIRATION_DAYS`                | The amount of time in days that a delegated operation can run before being automatically terminated. Default is 1 day.                                                                                                                                                         | No                        |
+| `FIFTYONE_DO_EXPIRATION_MINUTES`             | The amount of time in minutes that a delegated operation can run before being automatically terminated. Default is `FIFTYONE_DO_EXPIRATION_DAYS` converted to minutes. If this field is provided it will override `FIFTYONE_DO_EXPIRATION_DAYS`.                               | No                        |
 | `FIFTYONE_ENCRYPTION_KEY`                    | Used to encrypt storage credentials in MongoDB                                                                                                                                                                                                                                 | Yes                       |
 | `FIFTYONE_ENV`                               | GraphQL verbosity for the `fiftyone-teams-api` service; `production` will not log every GraphQL query, any other value will                                                                                                                                                    | No                        |
 | `FIFTYONE_LOGGING_FORMAT`                    | The format to use for log messages; `json` or `text`. The default is `text`.                                                                                                                                                                                                   | No                        |

docker/common-services.yaml

@@ -160,7 +160,7 @@ services:
     deploy:
       replicas: ${FIFTYONE_DELEGATED_OPERATOR_WORKER_REPLICAS:-3}
     command: >
-      /bin/sh -c "fiftyone delegated launch -t remote"
+      /bin/sh -c "fiftyone delegated launch -t remote -m"
     environment:
       API_URL: ${API_URL}
       FIFTYONE_DATABASE_ADMIN: false

docker/docs/configuring-gpu-workloads.md

@@ -67,7 +67,7 @@ services:
   teams-do-with-gpu:
     image: voxel51/fiftyone-teams-cv-full:v2.12.0
     command: >
-      /bin/sh -c "fiftyone delegated launch -t remote  -n 'teams-do-with-gpu'"
+      /bin/sh -c "fiftyone delegated launch -t remote  -n 'teams-do-with-gpu' -m"
     environment:
       API_URL: ${API_URL}
       FIFTYONE_DATABASE_ADMIN: false

helm/fiftyone-teams-app/README.md

@@ -481,6 +481,8 @@ follow
 | apiSettings.affinity | object | `{}` | Affinity and anti-affinity for `teams-api`. [Reference][affinity]. |
 | apiSettings.deploymentAnnotations | object | `{}` | Annotations for the `teams-api` deployment. [Reference][annotations]. |
 | apiSettings.dnsName | string | `""` | Controls whether `teams-api` is added to the chart's ingress. When an empty string, a rule for `teams-api` is not added to the chart managed ingress. When not an empty string, becomes the value to the `host` in the ingress' rule and set `ingress.api` too. Additionally, the `apiSettings.dnsName` is used by `teams-api` to generate the `API_EXTERNAL_URL` environment variable for configuring external orchestrators. |
+| apiSettings.env.FIFTYONE_DO_EXPIRATION_DAYS | int | `1` | The amount of time in days that a delegated operation can run before being automatically terminated. Overridden by "FIFTYONE_DO_EXPIRATION_MINUTES" if provided. |
+| apiSettings.env.FIFTYONE_DO_EXPIRATION_MINUTES | int | `nil` | The amount of time in minutes that a delegated operation can run before being automatically terminated. Overrides "FIFTYONE_DO_EXPIRATION_DAYS" if provided. |
 | apiSettings.env.FIFTYONE_ENV | string | `"production"` | Controls FiftyOne GraphQL verbosity. When "production", debug mode is disabled and the default logging level is "INFO". When "development", debug mode is enabled and the default logging level is "DEBUG". Can be overridden by setting `apiSettings.env.LOGGING_LEVEL`. |
 | apiSettings.env.FIFTYONE_INTERNAL_SERVICE | bool | `true` | Whether the SDK is running in an internal service context. When running in FiftyOne Enterprise, set to `true`. |
 | apiSettings.env.FIFTYONE_LOGGING_FORMAT | string | `"text"` | The format to use for log messages. Can be one of `json` or `text`. |

helm/fiftyone-teams-app/templates/delegated-operator-instance-deployment.yaml

@@ -63,6 +63,7 @@ spec:
             {{- if ($v.description | default $defaultDescription) }}
             - -d
             - {{ ($v.description | default $defaultDescription) | quote }}
+            - -m
             {{- end }}
           securityContext:
             {{- toYaml (merge (dict) ($v.securityContext | default dict) ($baseTpl.securityContext)) | nindent 12 }}

helm/fiftyone-teams-app/values.schema.json

@@ -26,6 +26,23 @@
         "env": {
           "description": "Environment Variables are passed to the `teams-api` containers",
           "properties": {
+            "FIFTYONE_DO_EXPIRATION_DAYS": {
+              "default": 1,
+              "description": "The amount of time in days that a delegated operation can run before being automatically terminated.\nOverridden by \"FIFTYONE_DO_EXPIRATION_MINUTES\" if provided.",
+              "required": [],
+              "title": "FIFTYONE_DO_EXPIRATION_DAYS",
+              "type": "integer"
+            },
+            "FIFTYONE_DO_EXPIRATION_MINUTES": {
+              "default": "",
+              "description": "(int) The amount of time in minutes that a delegated operation can run before being automatically terminated.\nOverrides \"FIFTYONE_DO_EXPIRATION_DAYS\" if provided.",
+              "required": [],
+              "title": "FIFTYONE_DO_EXPIRATION_MINUTES",
+              "type": [
+                "integer",
+                "null"
+              ]
+            },
             "FIFTYONE_ENV": {
               "default": "production",
               "description": "Controls FiftyOne GraphQL verbosity.\nWhen \"production\", debug mode is disabled and the default logging level is \"INFO\".\nWhen \"development\", debug mode is enabled and the default logging level is \"DEBUG\".\nCan be overridden by setting `apiSettings.env.LOGGING_LEVEL`.",

helm/fiftyone-teams-app/values.yaml

@@ -36,6 +36,15 @@ apiSettings:
     # -- The format to use for log messages.
     # Can be one of `json` or `text`.
     FIFTYONE_LOGGING_FORMAT: text
+    # -- The amount of time in days that a delegated operation can run before being automatically terminated.
+    # Overridden by "FIFTYONE_DO_EXPIRATION_MINUTES" if provided.
+    FIFTYONE_DO_EXPIRATION_DAYS: 1
+    # @schema
+    # type: ["integer", "null"]
+    # @schema
+    # -- (int) The amount of time in minutes that a delegated operation can run before being automatically terminated.
+    # Overrides "FIFTYONE_DO_EXPIRATION_DAYS" if provided.
+    FIFTYONE_DO_EXPIRATION_MINUTES:
 
   # -- Secret variables to be passed to the `teams-api` containers.
   secretEnv: {}

tests/unit/helm/api-deployment_test.go

@@ -486,6 +486,14 @@ func (s *deploymentApiTemplateTest) TestContainerEnv() {
               }
             }
           },
+		  {
+			"name": "FIFTYONE_DO_EXPIRATION_DAYS",
+			"value": "1"
+		  },
+		  {
+			"name": "FIFTYONE_DO_EXPIRATION_MINUTES",
+			"value": ""
+		  },
           {
             "name": "FIFTYONE_ENV",
             "value": "production"
@@ -575,6 +583,14 @@ func (s *deploymentApiTemplateTest) TestContainerEnv() {
               }
             }
           },
+		  {
+			"name": "FIFTYONE_DO_EXPIRATION_DAYS",
+			"value": "1"
+		  },
+		  {
+			"name": "FIFTYONE_DO_EXPIRATION_MINUTES",
+			"value": ""
+		  },
           {
             "name": "FIFTYONE_ENV",
             "value": "production"
@@ -678,6 +694,14 @@ func (s *deploymentApiTemplateTest) TestContainerEnv() {
               }
             }
           },
+		  {
+			"name": "FIFTYONE_DO_EXPIRATION_DAYS",
+			"value": "1"
+		  },
+		  {
+			"name": "FIFTYONE_DO_EXPIRATION_MINUTES",
+			"value": ""
+		  },
           {
             "name": "FIFTYONE_ENV",
             "value": "production"
@@ -778,6 +802,14 @@ func (s *deploymentApiTemplateTest) TestContainerEnv() {
               }
             }
           },
+		  {
+			"name": "FIFTYONE_DO_EXPIRATION_DAYS",
+			"value": "1"
+		  },
+		  {
+			"name": "FIFTYONE_DO_EXPIRATION_MINUTES",
+			"value": ""
+		  },
           {
             "name": "FIFTYONE_ENV",
             "value": "production"
@@ -866,6 +898,14 @@ func (s *deploymentApiTemplateTest) TestContainerEnv() {
               }
             }
           },
+		  {
+			"name": "FIFTYONE_DO_EXPIRATION_DAYS",
+			"value": "1"
+		  },
+		  {
+			"name": "FIFTYONE_DO_EXPIRATION_MINUTES",
+			"value": ""
+		  },
           {
             "name": "FIFTYONE_ENV",
             "value": "production"
@@ -954,6 +994,14 @@ func (s *deploymentApiTemplateTest) TestContainerEnv() {
               }
             }
           },
+		  {
+			"name": "FIFTYONE_DO_EXPIRATION_DAYS",
+			"value": "1"
+		  },
+		  {
+			"name": "FIFTYONE_DO_EXPIRATION_MINUTES",
+			"value": ""
+		  },
           {
             "name": "FIFTYONE_ENV",
             "value": "production"

tests/unit/helm/delegated-operator-instance-deployment_test.go

@@ -4937,6 +4937,7 @@ func (s *deploymentDelegatedOperatorInstanceTemplateTest) TestContainerCmdArgs()
 						"teams-do",
 						"-d",
 						"Long running operations delegated to teams-do",
+						"-m",
 					}
 					s.Equal(expectedArgs, args, "Args should be equal")
 				},
@@ -4959,6 +4960,7 @@ func (s *deploymentDelegatedOperatorInstanceTemplateTest) TestContainerCmdArgs()
 						"teams-do",
 						"-d",
 						"Long running operations delegated to teams-do",
+						"-m",
 					}
 					s.Equal(expectedArgs, args, "Args should be equal")
 				},
@@ -4972,6 +4974,7 @@ func (s *deploymentDelegatedOperatorInstanceTemplateTest) TestContainerCmdArgs()
 						"teams-do-two",
 						"-d",
 						"Long running operations delegated to teams-do-two",
+						"-m",
 					}
 					s.Equal(expectedArgs, args, "Args should be equal")
 				},
@@ -4995,6 +4998,7 @@ func (s *deploymentDelegatedOperatorInstanceTemplateTest) TestContainerCmdArgs()
 						"teams-do",
 						"-d",
 						"Long running operations delegated to teams-do",
+						"-m",
 					}
 					s.Equal(expectedArgs, args, "Args should be equal")
 				},
@@ -5008,6 +5012,7 @@ func (s *deploymentDelegatedOperatorInstanceTemplateTest) TestContainerCmdArgs()
 						"teams-do-two",
 						"-d",
 						"Long running operations delegated to teams-do-two",
+						"-m",
 					}
 					s.Equal(expectedArgs, args, "Args should be equal")
 				},
@@ -5030,6 +5035,7 @@ func (s *deploymentDelegatedOperatorInstanceTemplateTest) TestContainerCmdArgs()
 						"teams-do",
 						"-d",
 						"Used for non-gpu workloads",
+						"-m",
 					}
 					s.Equal(expectedArgs, args, "Args should be equal")
 				},
@@ -5043,6 +5049,7 @@ func (s *deploymentDelegatedOperatorInstanceTemplateTest) TestContainerCmdArgs()
 						"teams-do-two",
 						"-d",
 						"Long running operations delegated to teams-do-two",
+						"-m",
 					}
 					s.Equal(expectedArgs, args, "Args should be equal")
 				},
@@ -5066,6 +5073,7 @@ func (s *deploymentDelegatedOperatorInstanceTemplateTest) TestContainerCmdArgs()
 						"teams-do",
 						"-d",
 						"Used for non-gpu workloads",
+						"-m",
 					}
 					s.Equal(expectedArgs, args, "Args should be equal")
 				},
@@ -5079,6 +5087,7 @@ func (s *deploymentDelegatedOperatorInstanceTemplateTest) TestContainerCmdArgs()
 						"teams-do-two",
 						"-d",
 						"Long running operations delegated to teams-do-two",
+						"-m",
 					}
 					s.Equal(expectedArgs, args, "Args should be equal")
 				},