From 93a73bf9de9f7bb87fc96e489033bc4f434fc333 Mon Sep 17 00:00:00 2001 From: mamiller Date: Thu, 20 Jan 2022 20:45:30 -0500 Subject: [PATCH] add rate limit backoffs & retries for uptime.com --- api/v1alpha1/zz_generated.deepcopy.go | 1 - pkg/http/httpClient.go | 10 +++++- pkg/monitors/uptime/uptime-monitor.go | 46 +++++++++++++++++++++++++-- 3 files changed, 52 insertions(+), 5 deletions(-) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index cde0e3d0..b0a4d340 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1,4 +1,3 @@ -//go:build !ignore_autogenerated // +build !ignore_autogenerated /* diff --git a/pkg/http/httpClient.go b/pkg/http/httpClient.go index 465c8276..87bd387f 100644 --- a/pkg/http/httpClient.go +++ b/pkg/http/httpClient.go @@ -4,6 +4,7 @@ import ( "bytes" "io/ioutil" "net/http" + "strings" logf "sigs.k8s.io/controller-runtime/pkg/log" ) @@ -17,6 +18,7 @@ type HttpClient struct { type HttpResponse struct { StatusCode int Bytes []byte + Headers map[string]string } func CreateHttpClient(url string) *HttpClient { @@ -31,6 +33,8 @@ func (client *HttpClient) addHeaders(request *http.Request, headers map[string]s } func (client *HttpClient) RequestWithHeaders(requestType string, body []byte, headers map[string]string) HttpResponse { + respHeaders := make(map[string]string) + reader := bytes.NewReader(body) // log.Info("NewRequest: METHOD: " + requestType + " URL: " + client.url + " PAYLOAD: " + string(body)) @@ -47,11 +51,15 @@ func (client *HttpClient) RequestWithHeaders(requestType string, body []byte, he } response, err := http.DefaultClient.Do(request) + + for k, v := range response.Header { + respHeaders[strings.ToLower(k)] = string(v[0]) + } if err != nil { log.Error(err, "") } - httpResponse := HttpResponse{StatusCode: response.StatusCode} + httpResponse := HttpResponse{StatusCode: response.StatusCode, Headers: respHeaders} defer response.Body.Close() responseBytes, _ := ioutil.ReadAll(response.Body) diff --git a/pkg/monitors/uptime/uptime-monitor.go b/pkg/monitors/uptime/uptime-monitor.go index 270dedac..8fb66030 100644 --- a/pkg/monitors/uptime/uptime-monitor.go +++ b/pkg/monitors/uptime/uptime-monitor.go @@ -36,8 +36,10 @@ func (monitor *UpTimeMonitorService) Equal(oldMonitor models.Monitor, newMonitor // using processed config to avoid unnecessary update call because of default values // like contacts and sorted locations - if !(reflect.DeepEqual(processProviderConfig(oldMonitor), processProviderConfig(newMonitor))) { - log.Info(fmt.Sprintf("There are some new changes in %s monitor", newMonitor.Name)) + oldConfig := processProviderConfig(oldMonitor) + newConfig := processProviderConfig(newMonitor) + if !(reflect.DeepEqual(oldConfig, newConfig)) { + log.Info(fmt.Sprintf("There are some new changes in %s monitor: old: %s, new: %s", newMonitor.Name, oldConfig, newConfig)) return false } return true @@ -85,7 +87,11 @@ func (monitor *UpTimeMonitorService) GetAll() []models.Monitor { checksUrl := fmt.Sprintf("%schecks/?page=%d", monitor.url, pageNo) client := http.CreateHttpClient(checksUrl) response := client.GetUrl(headers, []byte("")) - if response.StatusCode != Http.StatusOK { + if response.StatusCode == Http.StatusTooManyRequests { + log.Info("failed getting monitors due to rate limit") + ObserveRateLimit(response) + return nil + } else if response.StatusCode != Http.StatusOK { log.Info("GetAllMonitors Request for Uptime failed. Status Code: " + strconv.Itoa(response.StatusCode)) return nil } @@ -135,6 +141,12 @@ func (monitor *UpTimeMonitorService) Add(m models.Monitor) { "Response: ") log.Info(string(response.Bytes)) } + } else if response.StatusCode == Http.StatusTooManyRequests { + log.Info("failed adding monitor due to rate limit") + err := ObserveRateLimit(response) + if err == nil { + monitor.Add(m) + } } else { log.Info("AddMonitor Request failed. Status Code: " + strconv.Itoa(response.StatusCode) + "\n" + string(response.Bytes)) } @@ -174,6 +186,12 @@ func (monitor *UpTimeMonitorService) Update(m models.Monitor) { } else { log.Info("Monitor couldn't be updated: " + m.Name) } + } else if response.StatusCode == Http.StatusTooManyRequests { + log.Info("failed updating monitor due to rate limit") + err := ObserveRateLimit(response) + if err == nil { + monitor.Update(m) + } } else { log.Info("UpdateMonitor Request failed. Status Code: " + strconv.Itoa(response.StatusCode)) } @@ -206,6 +224,12 @@ func (monitor *UpTimeMonitorService) Remove(m models.Monitor) { } else { log.Info("Monitor couldn't be removed: " + m.Name) } + } else if response.StatusCode == Http.StatusTooManyRequests { + log.Info("failed removing monitor due to rate limit") + err := ObserveRateLimit(response) + if err == nil { + monitor.Remove(m) + } } else { log.Info("RemoveMonitor Request failed. Status Code: " + strconv.Itoa(response.StatusCode)) } @@ -247,5 +271,21 @@ func processProviderConfig(m models.Monitor) map[string]interface{} { } return body +} + +func ObserveRateLimit(resp http.HttpResponse) error { + strDuration := resp.Headers["retry-after"] + if strDuration == "" { + log.Info("No retry-after header was present in the rate limited response") + return errors.New("No retry-after header was present in the rate limited response") + } + intDuration, err := strconv.Atoi(strDuration) + if err != nil { + log.Info(fmt.Sprintf("error parsing duration from value: %s", strDuration)) + return errors.New("failed parsing duration value from the retry-after response header") + } + log.Info(fmt.Sprintf("rate limit hit, backing off for %s seconds", strDuration)) + time.Sleep(time.Duration(intDuration) * time.Second) + return nil }