Merge pull request #156 from grisu48/requests

grisu48 · web-flow · commit 3468b6052f32 · 2024-07-22T08:46:25.000+02:00
Limit number of jobs per http request
diff --git a/_review/opensuse-containers.toml b/_review/opensuse-containers.toml
@@ -8,6 +8,7 @@ RefreshInterval = 60                                            # Refresh from A
 MaxJobs = 20                                                    # Max. job per group to display
 GroupBy = "groups"                                              # Group by defined groups ("none" or "groups")
 DefaultParams = { distri="opensuse", version = "Tumbleweed" }   # Set of default parameters
+RequestJobLimit = 100                                           # Query up to 100 jobs per http request
 
 ## Define job groups
 
diff --git a/_review/opensuse-microos.toml b/_review/opensuse-microos.toml
@@ -8,6 +8,7 @@ RefreshInterval = 60                                            # Refresh from A
 MaxJobs = 20                                                    # Max. job per group to display
 GroupBy = "groups"                                              # Group by defined groups ("none" or "groups")
 DefaultParams = { distri="microos", version = "Tumbleweed" }    # Set of default parameters
+RequestJobLimit = 100                                           # Query up to 100 jobs per http request
 
 ## Define job groups
 [[Groups]]
diff --git a/_review/opensuse-tumbleweed.toml b/_review/opensuse-tumbleweed.toml
@@ -8,6 +8,7 @@ RefreshInterval = 60                                            # Refresh from A
 MaxJobs = 20                                                    # Max. job per group to display
 GroupBy = "groups"                                              # Group by defined groups ("none" or "groups")
 DefaultParams = { distri="opensuse", version = "Tumbleweed" }   # Set of default parameters
+RequestJobLimit = 100                                           # Query up to 100 jobs per http request
 
 ## Define job groups
 
diff --git a/_review/qe-Maintenance-QEC.toml b/_review/qe-Maintenance-QEC.toml
@@ -7,6 +7,7 @@ HideStatus = ["scheduled", "passed", "softfailed", "cancelled", "skipped", "runn
 RefreshInterval = 60                             # Refresh from API once every minute
 MaxJobs = 20                                     # Max. job per group to display
 GroupBy = "groups"                               # Group by defined groups ("none" or "groups")
+RequestJobLimit = 100                                           # Query up to 100 jobs per http request
 
 ## BCI and Base images #########################################################
 
diff --git a/_review/qe-core-O3.toml b/_review/qe-core-O3.toml
@@ -8,6 +8,7 @@ HideStatus = ["scheduled", "passed", "softfailed", "cancelled", "skipped", "runn
 RefreshInterval = 60                                            # Refresh from API once every minute
 MaxJobs = 20                                                    # Max. job per group to display
 GroupBy = "groups"                                              # Group by defined groups ("none" or "groups")
+RequestJobLimit = 100                                           # Query up to 100 jobs per http request
 
 # Tumbleweed
 
diff --git a/_review/qe-core-OSD.toml b/_review/qe-core-OSD.toml
@@ -8,6 +8,7 @@ HideStatus = ["scheduled", "passed", "softfailed", "cancelled", "skipped", "runn
 RefreshInterval = 60                             # Refresh from API once every minute
 MaxJobs = 20                                     # Max. job per group to display
 GroupBy = "groups"                               # Group by defined groups ("none" or "groups")
+RequestJobLimit = 100                                           # Query up to 100 jobs per http request
 
 # SLE Maintenance: Test Repo / Core Maintenance Updates
 
diff --git a/_review/qec-bci.toml b/_review/qec-bci.toml
@@ -7,6 +7,7 @@ HideStatus = ["scheduled", "passed", "softfailed", "cancelled", "skipped", "runn
 RefreshInterval = 60                             # Refresh from API once every minute
 MaxJobs = 20                                     # Max. job per group to display
 GroupBy = "groups"                               # Group by defined groups ("none" or "groups")
+RequestJobLimit = 100                                           # Query up to 100 jobs per http request
 
 ## BCI and Base images
 [[Groups]]
diff --git a/_review/qec-containers.toml b/_review/qec-containers.toml
@@ -7,6 +7,7 @@ HideStatus = ["scheduled", "passed", "softfailed", "cancelled", "skipped", "runn
 RefreshInterval = 60                             # Refresh from API once every minute
 MaxJobs = 20                                     # Max. job per group to display
 GroupBy = "groups"                               # Group by defined groups ("none" or "groups")
+RequestJobLimit = 100                                           # Query up to 100 jobs per http request
 
 
 ## Maintenance updates
diff --git a/_review/qec-minimalvm.toml b/_review/qec-minimalvm.toml
@@ -7,6 +7,7 @@ HideStatus = ["scheduled", "passed", "softfailed", "cancelled", "skipped", "runn
 RefreshInterval = 60                             # Refresh from API once every minute
 MaxJobs = 20                                     # Max. job per group to display
 GroupBy = "groups"                               # Group by defined groups ("none" or "groups")
+RequestJobLimit = 100                                           # Query up to 100 jobs per http request
 
 ## Maintenance updates
 [[Groups]]
diff --git a/_review/qec-publiccloud.toml b/_review/qec-publiccloud.toml
@@ -9,6 +9,7 @@ RefreshInterval = 60                             # Refresh from API once every m
 MaxJobs = 20                                     # Max. job per group to display
 GroupBy = "groups"                               # Group by defined groups ("none" or "groups")
 DefaultParams = { distri = "sle" }               # Set of default parameters
+RequestJobLimit = 100                                           # Query up to 100 jobs per http request
 
 
 ## Maintenance updates
diff --git a/_review/qec-sle-micro.toml b/_review/qec-sle-micro.toml
@@ -7,6 +7,7 @@ HideStatus = [ "scheduled", "passed", "softfailed", "running", "reviewed", "user
 RefreshInterval = 60                             # Refresh from API once every minute
 MaxJobs = 20                                     # Max. job per group to display
 GroupBy = "groups"                               # Group by defined groups ("none" or "groups")
+RequestJobLimit = 100                                           # Query up to 100 jobs per http request
 
 
 ## SLE Micro image updates
diff --git a/_review/qec-wsl.toml b/_review/qec-wsl.toml
@@ -7,6 +7,7 @@ HideStatus = [ "scheduled", "passed", "softfailed", "running", "reviewed", "user
 RefreshInterval = 60                             # Refresh from API once every minute
 MaxJobs = 20                                     # Max. job per group to display
 GroupBy = "groups"                               # Group by defined groups ("none" or "groups")
+RequestJobLimit = 100                                           # Query up to 100 jobs per http request
 
 
 ## WSL builds
diff --git a/cmd/openqa-revtui/config.go b/cmd/openqa-revtui/config.go
@@ -26,6 +26,7 @@ type Config struct {
 	Groups          []Group           // Groups that will be monitord
 	MaxJobs         int               // Maximum number of jobs per group to consider
 	GroupBy         string            // Display group mode: "none", "groups"
+	RequestJobLimit int               // Maximum number of jobs in a single request
 }
 
 var cf Config
@@ -68,5 +69,6 @@ func CreateConfig() Config {
 	cf.DefaultParams = make(map[string]string, 0)
 	cf.Groups = make([]Group, 0)
 	cf.MaxJobs = 20
+	cf.RequestJobLimit = 100
 	return cf
 }
diff --git a/cmd/openqa-revtui/openqa-revtui.go b/cmd/openqa-revtui/openqa-revtui.go
@@ -231,7 +231,7 @@ func refreshJobs(tui *TUI, instance *gopenqa.Instance) error {
 	for _, job := range oldJobs {
 		ids = append(ids, job.ID)
 	}
-	jobs, err := instance.GetJobsFollow(ids)
+	jobs, err := fetchJobsFollow(ids, instance)
 	if err != nil {
 		return err
 	}
diff --git a/cmd/openqa-revtui/openqa.go b/cmd/openqa-revtui/openqa.go
@@ -107,21 +107,47 @@ func FetchJob(id int64, instance *gopenqa.Instance) (gopenqa.Job, error) {
 	return job, fmt.Errorf("max recursion depth reached")
 }
 
+/* Fetch the given jobs and follow their clones */
+func fetchJobsFollow(ids []int64, instance *gopenqa.Instance) ([]gopenqa.Job, error) {
+	// Obey the maximum number of job per requests.
+	// We split the job ids into multiple requests if necessary
+	jobs := make([]gopenqa.Job, 0)
+	for len(ids) > 0 {
+		n := min(cf.RequestJobLimit, len(ids))
+		chunk, err := instance.GetJobsFollow(ids[:n])
+		ids = ids[n:]
+		if err != nil {
+			return jobs, err
+		}
+		jobs = append(jobs, chunk...)
+	}
+
+	return jobs, nil
+}
+
 /* Fetch the given jobs from the instance at once */
 func fetchJobs(ids []int64, instance *gopenqa.Instance) ([]gopenqa.Job, error) {
-	jobs, err := instance.GetJobs(ids)
-	if err != nil {
-		return jobs, err
+	// Obey the maximum number of job per requests.
+	// We split the job ids into multiple requests if necessary
+	jobs := make([]gopenqa.Job, 0)
+	for len(ids) > 0 {
+		n := min(cf.RequestJobLimit, len(ids))
+		chunk, err := instance.GetJobs(ids[:n])
+		ids = ids[n:]
+		if err != nil {
+			return jobs, err
+		}
+		jobs = append(jobs, chunk...)
 	}
 
 	// Get cloned jobs, if present
 	for i, job := range jobs {
 		if job.IsCloned() {
-			job, err = FetchJob(job.ID, instance)
-			if err != nil {
+			if job, err := FetchJob(job.ID, instance); err != nil {
 				return jobs, err
+			} else {
+				jobs[i] = job
 			}
-			jobs[i] = job
 		}
 	}
 	return jobs, nil
