HETZNER: better rate limit handling (#936)

* HETZNER: better rate limit handling

- Hetzner is using a Proxy service 'kong' which broadcasts it limits
- honor 'Retry-After' of 429 responses
- delay requests per-se: see the amended docs for details

Signed-off-by: Jakob Ackermann <das7pad@outlook.com>

* HETZNER: apply review feedback: store quotaName as lower case

Co-authored-by: Tom Limoncelli <tlimoncelli@stackoverflow.com>
Signed-off-by: Jakob Ackermann <das7pad@outlook.com>

Co-authored-by: Tom Limoncelli <tlimoncelli@stackoverflow.com>

docs/_providers/hetzner.md

@@ -54,19 +54,60 @@ At this time you cannot update SOA records via DNSControl.
 
 ### Rate Limiting
 
-In case you are frequently seeing messages about being rate-limited:
+Hetzner is rate limiting requests in multiple tiers: per Hour, per Minute and
+ per Second.
 
-{% highlight txt %}
-WARNING: request rate-limited, constant back-off is now at 1s.
-{% endhighlight %}
+Depending on how many requests you are planning to perform, you can adjust the
+ delay between requests in order to stay within your quota.
 
-You may want to enable the `rate_limited` mode by default.
+The setting `optimize_for_rate_limit_quota` controls this behavior and accepts
+ a case-insensitive value of
+- `Hour`
+- `Minute`
+- `Second`
+
+The default for `optimize_for_rate_limit_quota` is `Second`.
+
+Example: Your per minute quota is 60 requests and in your settings you
+ specified `Minute`. DNSControl will perform at most one request per second.
+ DNSControl will emit a warning in case it breaches the next quota.
 
 In your `creds.json` for all `HETZNER` provider entries:
 {% highlight json %}
 {
   "hetzner": {
-    "rate_limited": "true",
+    "optimize_for_rate_limit_quota": "Minute",
+    "api_key": "your-api-key"
+  }
+}
+{% endhighlight %}
+
+Every response from the Hetzner DNS Console API includes your limits:
+
+{% highlight txt %}
+$ curl --silent --include \
+    --header 'Auth-API-Token: ...' \
+    https://dns.hetzner.com/api/v1/zones \
+  | grep x-ratelimit-limit
+x-ratelimit-limit-second: 3
+x-ratelimit-limit-minute: 42
+x-ratelimit-limit-hour: 1337
+{% endhighlight %}
+
+Every DNSControl invocation starts from scratch in regard to rate-limiting.
+In case you are frequently invoking DNSControl, you will likely hit a limit for
+ any first request.
+You can either use an out-of-bound delay (e.g. `$ sleep 1`), or specify
+ `start_with_default_rate_limit` in the settings of the provider.
+With `start_with_default_rate_limit` DNSControl uses a quota equivalent to
+ `x-ratelimit-limit-second: 1` until it could parse the actual quota from an
+ API response.
+
+In your `creds.json` for all `HETZNER` provider entries:
+{% highlight json %}
+{
+  "hetzner": {
+    "start_with_default_rate_limit": "true",
     "api_key": "your-api-key"
   }
 }

integrationTest/providers.json

@@ -72,7 +72,8 @@
   "HETZNER": {
     "api_key": "$HETZNER_API_KEY",
     "domain": "$HETZNER_DOMAIN",
-    "rate_limited": "true"
+    "start_with_default_rate_limit": "true",
+    "optimize_for_rate_limit_quota": "Hour"
   },
   "HEXONET": {
     "apientity": "$HEXONET_ENTITY",

providers/hetzner/api.go

@@ -7,6 +7,8 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
+	"strconv"
+	"strings"
 	"time"
 )
 
@@ -29,6 +31,43 @@ func checkIsLockedSystemRecord(record record) error {
 	return nil
 }
 
+func getHomogenousDelay(headers http.Header, quotaName string) (time.Duration, error) {
+	quota, err := parseHeaderAsInt(headers, "X-Ratelimit-Limit-"+strings.Title(quotaName))
+	if err != nil {
+		return 0, err
+	}
+
+	var unit time.Duration
+	switch quotaName {
+	case "hour":
+		unit = time.Hour
+	case "minute":
+		unit = time.Minute
+	case "second":
+		unit = time.Second
+	}
+
+	delay := time.Duration(int64(unit) / quota)
+	return delay, nil
+}
+
+func getRetryAfterDelay(header http.Header) (time.Duration, error) {
+	retryAfter, err := parseHeaderAsInt(header, "Retry-After")
+	if err != nil {
+		return 0, err
+	}
+	delay := time.Duration(retryAfter * int64(time.Second))
+	return delay, nil
+}
+
+func parseHeaderAsInt(headers http.Header, headerName string) (int64, error) {
+	value, ok := headers[headerName]
+	if !ok {
+		return 0, fmt.Errorf("header %q is missing", headerName)
+	}
+	return strconv.ParseInt(value[0], 10, 0)
+}
+
 func (api *hetznerProvider) bulkCreateRecords(records []record) error {
 	for _, record := range records {
 		if err := checkIsLockedSystemRecord(record); err != nil {
@@ -185,6 +224,8 @@ func (api *hetznerProvider) request(endpoint string, method string, request inte
 			}
 		}
 
+		api.requestRateLimiter.handleResponse(*resp)
+		// retry the request when rate-limited
 		if resp.StatusCode == 429 {
 			api.requestRateLimiter.handleRateLimitedRequest()
 			cleanupResponseBody()
@@ -206,9 +247,12 @@ func (api *hetznerProvider) request(endpoint string, method string, request inte
 }
 
 func (api *hetznerProvider) startRateLimited() {
-	// Simulate a request that is getting a 429 response.
-	api.requestRateLimiter.afterRequest()
-	api.requestRateLimiter.bumpDelay()
+	// _Now_ is the best reference we can get for the last request.
+	// Head-On-Head invocations of DNSControl benefit from fewer initial
+	//  rate-limited requests.
+	api.requestRateLimiter.lastRequest = time.Now()
+	// use the default delay until we have had a chance to parse limits.
+	api.requestRateLimiter.setDefaultDelay()
 }
 
 func (api *hetznerProvider) updateRecord(record record) error {
@@ -221,8 +265,9 @@ func (api *hetznerProvider) updateRecord(record record) error {
 }
 
 type requestRateLimiter struct {
-	delay       time.Duration
-	lastRequest time.Time
+	delay                     time.Duration
+	lastRequest               time.Time
+	optimizeForRateLimitQuota string
 }
 
 func (requestRateLimiter *requestRateLimiter) afterRequest() {
@@ -236,23 +281,50 @@ func (requestRateLimiter *requestRateLimiter) beforeRequest() {
 	time.Sleep(time.Until(requestRateLimiter.lastRequest.Add(requestRateLimiter.delay)))
 }
 
-func (requestRateLimiter *requestRateLimiter) bumpDelay() string {
-	var backoffType string
-	if requestRateLimiter.delay == 0 {
-		// At the time this provider was implemented (2020-10-18),
-		//  one request per second could go though when rate-limited.
-		requestRateLimiter.delay = time.Second
-		backoffType = "constant"
-	} else {
-		// The initial assumption of 1 req/s may no hold true forever.
-		// Future proof this provider, use exponential back-off.
-		requestRateLimiter.delay = requestRateLimiter.delay * 2
-		backoffType = "exponential"
+func (requestRateLimiter *requestRateLimiter) setDefaultDelay() {
+	// default to a rate-limit of 1 req/s -- the next response should update it.
+	requestRateLimiter.delay = time.Second
+}
+
+func (requestRateLimiter *requestRateLimiter) setOptimizeForRateLimitQuota(quota string) error {
+	quotaNormalized := strings.ToLower(quota)
+	switch quotaNormalized {
+	case "hour", "minute", "second":
+		requestRateLimiter.optimizeForRateLimitQuota = quotaNormalized
+	case "":
+		requestRateLimiter.optimizeForRateLimitQuota = "second"
+	default:
+		return fmt.Errorf("%q is not a valid quota, expected 'Hour', 'Minute', 'Second' or unset", quota)
 	}
-	return backoffType
+	return nil
 }
 
 func (requestRateLimiter *requestRateLimiter) handleRateLimitedRequest() {
-	backoffType := requestRateLimiter.bumpDelay()
-	fmt.Println(fmt.Sprintf("WARNING: request rate-limited, %s back-off is now at %s.", backoffType, requestRateLimiter.delay))
+	message := "Rate-Limited, consider bumping the setting 'optimize_for_rate_limit_quota': %q -> %q"
+	switch requestRateLimiter.optimizeForRateLimitQuota {
+	case "hour":
+		message = "Rate-Limited, you are already using the slowest request rate. Consider contacting the Hetzner Support for raising your quota."
+	case "minute":
+		message = fmt.Sprintf(message, "Minute", "Hour")
+	case "second":
+		message = fmt.Sprintf(message, "Second", "Minute")
+	}
+	fmt.Println(message)
+}
+
+func (requestRateLimiter *requestRateLimiter) handleResponse(resp http.Response) {
+	homogenousDelay, err := getHomogenousDelay(resp.Header, requestRateLimiter.optimizeForRateLimitQuota)
+	if err != nil {
+		requestRateLimiter.setDefaultDelay()
+		return
+	}
+
+	delay := homogenousDelay
+	if resp.StatusCode == 429 {
+		retryAfterDelay, err := getRetryAfterDelay(resp.Header)
+		if err == nil {
+			delay = retryAfterDelay
+		}
+	}
+	requestRateLimiter.delay = delay
 }

providers/hetzner/hetznerProvider.go

@@ -40,9 +40,19 @@ func New(settings map[string]string, _ json.RawMessage) (providers.DNSServicePro
 	api.apiKey = settings["api_key"]
 
 	if settings["rate_limited"] == "true" {
+		// backwards compatibility
+		settings["start_with_default_rate_limit"] = "true"
+	}
+	if settings["start_with_default_rate_limit"] == "true" {
 		api.startRateLimited()
 	}
 
+	quota := settings["optimize_for_rate_limit_quota"]
+	err := api.requestRateLimiter.setOptimizeForRateLimitQuota(quota)
+	if err != nil {
+		return nil, fmt.Errorf("unexpected value for optimize_for_rate_limit_quota: %w", err)
+	}
+
 	return api, nil
 }