Merge pull request #6801 from grondo/issue#6795

job-manager: add `stop-queues-on-restart` configuration parameter

Author: mergify[bot]

doc/man5/flux-config-job-manager.rst

@@ -27,6 +27,17 @@ inactive-age-limit
 inactive-num-limit
    (optional) Integer maximum number of inactive jobs retained in the KVS.
 
+stop-queues-on-restart
+   (optional) Boolean value indicating if the job manager should automatically
+   stop any started queues during a restart. Queues stopped in this manner will
+   have their stop reason set to ::
+
+      Automatically stopped due to restart
+
+   while queues stopped before a shutdown will remain stopped with their
+   original stop reason. The default value is ``false``, which means that
+   started queues will remain started upon restart.
+
 plugins
    (optional) An array of objects defining a list of jobtap plugin directives.
    Each directive follows the format defined in the :ref:`plugin_directive`

src/modules/job-manager/queue.c

@@ -82,6 +82,7 @@ struct queue_ctx {
         zhashx_t *named;
     };
     bool have_named_queues;
+    bool stop_on_restart;   // stop started queues on restart
 };
 
 static void dequeue_jobs (struct queue_ctx *qctx, const char *name);
@@ -403,7 +404,18 @@ static int restore_state_v1 (struct queue_ctx *qctx, json_t *entry)
                 return -1;
         }
         if (start) {
-            if (queue_start (q, false) < 0)
+            /* If job-manager.stop-queues-on-restart is enabled, then
+             * stop this started queue with an automated message, otherwise
+             * leave the queue started.
+             */
+            if (qctx->stop_on_restart) {
+                if (queue_stop_one (qctx,
+                                    q,
+                                    "Automatically stopped due to restart",
+                                    false) < 0)
+                    return -1;
+            }
+            else if (queue_start (q, false) < 0)
                 return -1;
         }
         else {
@@ -493,7 +505,12 @@ static int queue_configure (const flux_conf_t *conf,
     struct queue_ctx *qctx = arg;
     json_t *queues;
 
-    if (flux_conf_unpack (conf, NULL, "{s:o}", "queues", &queues) == 0
+    if (flux_conf_unpack (conf,
+                          NULL,
+                          "{s?{s?b} s:o}",
+                          "job-manager",
+                            "stop-queues-on-restart", &qctx->stop_on_restart,
+                          "queues", &queues) == 0
         && json_object_size (queues) > 0) {
         const char *name;
         json_t *value;
@@ -1025,6 +1042,8 @@ struct queue_ctx *queue_ctx_create (struct job_manager *ctx)
     if (!(qctx = calloc (1, sizeof (*qctx))))
         return NULL;
     qctx->ctx = ctx;
+    qctx->stop_on_restart = false;
+
     if (!(qctx->anon = queue_create (NULL, NULL)))
         goto error;
     if (flux_msg_handler_addvec (ctx->h,

t/t2219-job-manager-restart.t

@@ -159,7 +159,28 @@ test_expect_success 'verify that named queue start/stop persists across restart'
 	grep "^batch: Scheduling is stopped: xyzzy" dump_queue_start_3.out &&
 	grep "^SCHED" dump_queue_start_3.out
 '
-
+test_expect_success 'verify that stop-queues-on-restart config works' '
+	mkdir -p conf.d &&
+	cat >conf.d/queues.toml <<-EOT &&
+	[job-manager]
+	stop-queues-on-restart = true
+	[queues.debug]
+	[queues.batch]
+	EOT
+	flux start --config-path=$(pwd)/conf.d \
+	    -Scontent.dump=dump_queue_stop.tar \
+	    sh -c "flux queue start --all; flux queue stop -m xxyyzz batch"  &&
+	flux start --config-path=$(pwd)/conf.d \
+	    -Scontent.restore=dump_queue_stop.tar \
+	    flux queue status -v > dump_stop.out &&
+	test_debug "cat dump_stop.out" &&
+	grep \
+	  "batch: Scheduling is stopped: xxyyzz" \
+	  dump_stop.out &&
+	grep \
+	  "debug: Scheduling is stopped: Automatically stopped due to restart" \
+	  dump_stop.out
+'
 test_expect_success 'checkpointed queue no longer configured on restart is ignored' '
 	mkdir -p conf.d &&
 	cat >conf.d/queues.toml <<-EOT &&