Merge pull request #6791 from grondo/issue#6781

job-exec: only adjust timelimit for jobs when start delay exceeds a configurable percent of job duration

Author: mergify[bot]

doc/man5/flux-config-exec.rst

@@ -82,6 +82,17 @@ barrier-timeout
    nodes on which the barrier is waiting.  To disable the barrier timeout,
    set this value to ``"0"``. (Default: ``30m``).
 
+max-start-delay-percent
+   (optional) Specify the maximum allowed delay, as a percentage of a job's
+   duration, between when a job is allocated (i.e. the starttime recorded
+   in _R_) and when the execution system receives the start request from
+   the job manager. If the delay exceeds this percentage, then extend the
+   job's effective expiration by the delay. This prevents short duration
+   jobs from having their runtime significantly reduced, while avoiding a
+   differential between the actual resource set expiration and the time
+   at which a ``timeout`` exception is raised for longer running jobs,
+   where any runtime impact will be negligible. The default is 25 percent.
+
 testexec
    (options) A table of keys (see :ref:`testexec`) for configuring the
    **job-exec** test execution implementation (used in mainly for testing).

src/modules/job-exec/job-exec.c

@@ -107,6 +107,7 @@
 #include "checkpoint.h"
 #include "exec_config.h"
 
+static double max_start_delay_percent;
 static double kill_timeout;
 static int max_kill_count;
 static int term_signal;
@@ -568,6 +569,28 @@ static void timelimit_cb (flux_reactor_t *r,
     jobinfo_killtimer_start (job, job->kill_timeout);
 }
 
+static double jobinfo_adjust_expiration (struct jobinfo *job,
+                                         double starttime,
+                                         double expiration)
+{
+    double duration = expiration - starttime;
+    double delta = job->t0 - starttime;
+
+    /* If the difference between the job's official starttime and the
+     * time of the start request is a large percentage of the job's total
+     * duration, then adjust the job's actual expiration to account for this
+     * difference. This prevents a short job from having its runtime
+     * significantly reduced, while avoiding creating a differential between
+     * the actual resource set expiration for longer jobs (the common case)
+     * where it doesn't matter.
+     *
+     * See also: https://github.com/flux-framework/flux-core/issues/6781
+     */
+    if ((delta / duration) * 100 > max_start_delay_percent)
+        return expiration + delta;
+    return expiration;
+}
+
 static int jobinfo_set_expiration (struct jobinfo *job)
 {
     flux_watcher_t *w = NULL;
@@ -600,14 +623,8 @@ static int jobinfo_set_expiration (struct jobinfo *job)
     if (job->t0 == 0.)
        job->t0 = now;
 
-    /* Adjust expiration time based on delay between when scheduler
-     *  created R and when we received this job. O/w jobs may be
-     *  terminated due to timeouts prematurely when the system
-     *  is very busy, which can cause long delays between alloc and
-     *  start events.
-     */
     if (starttime > 0.)
-        expiration += job->t0 - starttime;
+        expiration = jobinfo_adjust_expiration (job, starttime, expiration);
 
     offset = expiration - now;
     if (offset < 0.) {
@@ -1483,15 +1500,17 @@ static int job_exec_set_config_globals (flux_t *h,
      *
      * So we must re-initialize globals everytime we reload the module.
      */
+    max_start_delay_percent = 25.0;
     kill_timeout = 5.0;
     max_kill_count = 8;
     term_signal = SIGTERM;
     kill_signal = SIGKILL;
 
     if (flux_conf_unpack (conf,
                           &error,
-                          "{s?{s?s s?s s?s s?i}}",
+                          "{s?{s?F s?s s?s s?s s?i}}",
                           "exec",
+                            "max-start-delay-percent", &max_start_delay_percent,
                             "kill-timeout", &kto,
                             "term-signal", &tsignal,
                             "kill-signal", &ksignal,
@@ -1621,12 +1640,18 @@ static json_t *running_job_stats (struct job_exec_ctx *ctx)
         json_t *entry;
         char *critical_ranks;
         json_t *impl_stats = NULL;
+        double expiration = 0.;
 
         if (!(critical_ranks = idset_encode (job->critical_ranks,
                                              IDSET_FLAG_RANGE)))
             goto error;
 
-        entry = json_pack ("{s:s s:s s:s s:i s:i s:i s:i s:i s:i s:f s:i s:i}",
+        if (job->expiration_timer)
+            expiration = flux_watcher_next_wakeup (job->expiration_timer);
+
+
+        entry = json_pack ("{s:s s:s s:s s:i s:i s:i s:i s:i s:i"
+                           " s:f s:f s:i s:i}",
                            "implementation",
                            job->impl ? job->impl->name : "none",
                            "ns", job->ns,
@@ -1637,6 +1662,7 @@ static json_t *running_job_stats (struct job_exec_ctx *ctx)
                            "started", job->started,
                            "running", job->running,
                            "finalizing", job->finalizing,
+                           "expiration", expiration,
                            "kill_timeout", job->kill_timeout,
                            "kill_count", job->kill_count,
                            "kill_shell_count", job->kill_shell_count);

t/t2900-job-timelimits.t

@@ -93,4 +93,22 @@ test_expect_success 'expired job can also be canceled' '
 	    expired_cancel_test $(perl -E "say $TIMEOUT*$scale") && break
 	done
 '
+test_expect_success 'expiration can be extended via max-start-delay-percent' '
+	flux config load <<-EOF &&
+	[exec]
+	max-start-delay-percent = 0
+	EOF
+	test_when_finished "echo | flux config load" &&
+	flux jobtap load \
+	    ${FLUX_BUILD_DIR}/t/job-manager/plugins/.libs/perilog-test.so &&
+	test_when_finished "flux jobtap remove perilog-test.so" &&
+	jobid=$(flux submit -t 1m sleep 120) &&
+	flux job wait-event -Hvt 30 $jobid start &&
+	exp=$(flux job info $jobid R | jq .execution.expiration) &&
+	exec_exp=$(flux module stats job-exec | jq .jobs.${jobid}.expiration) &&
+	test_debug "echo expiration=${exp} exec_expiration=${exec_exp}" &&
+	flux module stats job-exec | jq -e ".jobs.${jobid}.expiration > $exp" &&
+	flux cancel $jobid &&
+	flux job wait-event -t 30 $jobid clean
+'
 test_done