diff --git a/docs/_providers/hetzner.md b/docs/_providers/hetzner.md index 99f6228a2..d2db84927 100644 --- a/docs/_providers/hetzner.md +++ b/docs/_providers/hetzner.md @@ -54,19 +54,60 @@ At this time you cannot update SOA records via DNSControl. ### Rate Limiting -In case you are frequently seeing messages about being rate-limited: +Hetzner is rate limiting requests in multiple tiers: per Hour, per Minute and + per Second. -{% highlight txt %} -WARNING: request rate-limited, constant back-off is now at 1s. -{% endhighlight %} +Depending on how many requests you are planning to perform, you can adjust the + delay between requests in order to stay within your quota. -You may want to enable the `rate_limited` mode by default. +The setting `optimize_for_rate_limit_quota` controls this behavior and accepts + a case-insensitive value of +- `Hour` +- `Minute` +- `Second` + +The default for `optimize_for_rate_limit_quota` is `Second`. + +Example: Your per minute quota is 60 requests and in your settings you + specified `Minute`. DNSControl will perform at most one request per second. + DNSControl will emit a warning in case it breaches the next quota. In your `creds.json` for all `HETZNER` provider entries: {% highlight json %} { "hetzner": { - "rate_limited": "true", + "optimize_for_rate_limit_quota": "Minute", + "api_key": "your-api-key" + } +} +{% endhighlight %} + +Every response from the Hetzner DNS Console API includes your limits: + +{% highlight txt %} +$ curl --silent --include \ + --header 'Auth-API-Token: ...' \ + https://dns.hetzner.com/api/v1/zones \ + | grep x-ratelimit-limit +x-ratelimit-limit-second: 3 +x-ratelimit-limit-minute: 42 +x-ratelimit-limit-hour: 1337 +{% endhighlight %} + +Every DNSControl invocation starts from scratch in regard to rate-limiting. +In case you are frequently invoking DNSControl, you will likely hit a limit for + any first request. +You can either use an out-of-bound delay (e.g. `$ sleep 1`), or specify + `start_with_default_rate_limit` in the settings of the provider. +With `start_with_default_rate_limit` DNSControl uses a quota equivalent to + `x-ratelimit-limit-second: 1` until it could parse the actual quota from an + API response. + +In your `creds.json` for all `HETZNER` provider entries: +{% highlight json %} +{ + "hetzner": { + "start_with_default_rate_limit": "true", "api_key": "your-api-key" } } diff --git a/integrationTest/providers.json b/integrationTest/providers.json index 6bf602808..3b5dbc08d 100644 --- a/integrationTest/providers.json +++ b/integrationTest/providers.json @@ -72,7 +72,8 @@ "HETZNER": { "api_key": "$HETZNER_API_KEY", "domain": "$HETZNER_DOMAIN", - "rate_limited": "true" + "start_with_default_rate_limit": "true", + "optimize_for_rate_limit_quota": "Hour" }, "HEXONET": { "apientity": "$HEXONET_ENTITY", diff --git a/providers/hetzner/api.go b/providers/hetzner/api.go index 968606148..3ce7661f2 100644 --- a/providers/hetzner/api.go +++ b/providers/hetzner/api.go @@ -7,6 +7,8 @@ import ( "io" "io/ioutil" "net/http" + "strconv" + "strings" "time" ) @@ -29,6 +31,43 @@ func checkIsLockedSystemRecord(record record) error { return nil } +func getHomogenousDelay(headers http.Header, quotaName string) (time.Duration, error) { + quota, err := parseHeaderAsInt(headers, "X-Ratelimit-Limit-"+strings.Title(quotaName)) + if err != nil { + return 0, err + } + + var unit time.Duration + switch quotaName { + case "hour": + unit = time.Hour + case "minute": + unit = time.Minute + case "second": + unit = time.Second + } + + delay := time.Duration(int64(unit) / quota) + return delay, nil +} + +func getRetryAfterDelay(header http.Header) (time.Duration, error) { + retryAfter, err := parseHeaderAsInt(header, "Retry-After") + if err != nil { + return 0, err + } + delay := time.Duration(retryAfter * int64(time.Second)) + return delay, nil +} + +func parseHeaderAsInt(headers http.Header, headerName string) (int64, error) { + value, ok := headers[headerName] + if !ok { + return 0, fmt.Errorf("header %q is missing", headerName) + } + return strconv.ParseInt(value[0], 10, 0) +} + func (api *hetznerProvider) bulkCreateRecords(records []record) error { for _, record := range records { if err := checkIsLockedSystemRecord(record); err != nil { @@ -185,6 +224,8 @@ func (api *hetznerProvider) request(endpoint string, method string, request inte } } + api.requestRateLimiter.handleResponse(*resp) + // retry the request when rate-limited if resp.StatusCode == 429 { api.requestRateLimiter.handleRateLimitedRequest() cleanupResponseBody() @@ -206,9 +247,12 @@ func (api *hetznerProvider) request(endpoint string, method string, request inte } func (api *hetznerProvider) startRateLimited() { - // Simulate a request that is getting a 429 response. - api.requestRateLimiter.afterRequest() - api.requestRateLimiter.bumpDelay() + // _Now_ is the best reference we can get for the last request. + // Head-On-Head invocations of DNSControl benefit from fewer initial + // rate-limited requests. + api.requestRateLimiter.lastRequest = time.Now() + // use the default delay until we have had a chance to parse limits. + api.requestRateLimiter.setDefaultDelay() } func (api *hetznerProvider) updateRecord(record record) error { @@ -221,8 +265,9 @@ func (api *hetznerProvider) updateRecord(record record) error { } type requestRateLimiter struct { - delay time.Duration - lastRequest time.Time + delay time.Duration + lastRequest time.Time + optimizeForRateLimitQuota string } func (requestRateLimiter *requestRateLimiter) afterRequest() { @@ -236,23 +281,50 @@ func (requestRateLimiter *requestRateLimiter) beforeRequest() { time.Sleep(time.Until(requestRateLimiter.lastRequest.Add(requestRateLimiter.delay))) } -func (requestRateLimiter *requestRateLimiter) bumpDelay() string { - var backoffType string - if requestRateLimiter.delay == 0 { - // At the time this provider was implemented (2020-10-18), - // one request per second could go though when rate-limited. - requestRateLimiter.delay = time.Second - backoffType = "constant" - } else { - // The initial assumption of 1 req/s may no hold true forever. - // Future proof this provider, use exponential back-off. - requestRateLimiter.delay = requestRateLimiter.delay * 2 - backoffType = "exponential" +func (requestRateLimiter *requestRateLimiter) setDefaultDelay() { + // default to a rate-limit of 1 req/s -- the next response should update it. + requestRateLimiter.delay = time.Second +} + +func (requestRateLimiter *requestRateLimiter) setOptimizeForRateLimitQuota(quota string) error { + quotaNormalized := strings.ToLower(quota) + switch quotaNormalized { + case "hour", "minute", "second": + requestRateLimiter.optimizeForRateLimitQuota = quotaNormalized + case "": + requestRateLimiter.optimizeForRateLimitQuota = "second" + default: + return fmt.Errorf("%q is not a valid quota, expected 'Hour', 'Minute', 'Second' or unset", quota) } - return backoffType + return nil } func (requestRateLimiter *requestRateLimiter) handleRateLimitedRequest() { - backoffType := requestRateLimiter.bumpDelay() - fmt.Println(fmt.Sprintf("WARNING: request rate-limited, %s back-off is now at %s.", backoffType, requestRateLimiter.delay)) + message := "Rate-Limited, consider bumping the setting 'optimize_for_rate_limit_quota': %q -> %q" + switch requestRateLimiter.optimizeForRateLimitQuota { + case "hour": + message = "Rate-Limited, you are already using the slowest request rate. Consider contacting the Hetzner Support for raising your quota." + case "minute": + message = fmt.Sprintf(message, "Minute", "Hour") + case "second": + message = fmt.Sprintf(message, "Second", "Minute") + } + fmt.Println(message) +} + +func (requestRateLimiter *requestRateLimiter) handleResponse(resp http.Response) { + homogenousDelay, err := getHomogenousDelay(resp.Header, requestRateLimiter.optimizeForRateLimitQuota) + if err != nil { + requestRateLimiter.setDefaultDelay() + return + } + + delay := homogenousDelay + if resp.StatusCode == 429 { + retryAfterDelay, err := getRetryAfterDelay(resp.Header) + if err == nil { + delay = retryAfterDelay + } + } + requestRateLimiter.delay = delay } diff --git a/providers/hetzner/hetznerProvider.go b/providers/hetzner/hetznerProvider.go index 2fd12fad9..c4562da2a 100644 --- a/providers/hetzner/hetznerProvider.go +++ b/providers/hetzner/hetznerProvider.go @@ -40,9 +40,19 @@ func New(settings map[string]string, _ json.RawMessage) (providers.DNSServicePro api.apiKey = settings["api_key"] if settings["rate_limited"] == "true" { + // backwards compatibility + settings["start_with_default_rate_limit"] = "true" + } + if settings["start_with_default_rate_limit"] == "true" { api.startRateLimited() } + quota := settings["optimize_for_rate_limit_quota"] + err := api.requestRateLimiter.setOptimizeForRateLimitQuota(quota) + if err != nil { + return nil, fmt.Errorf("unexpected value for optimize_for_rate_limit_quota: %w", err) + } + return api, nil }