mirror of
https://github.com/StackExchange/dnscontrol.git
synced 2024-09-20 14:56:20 +08:00
HETZNER: better rate limit handling (#936)
* HETZNER: better rate limit handling - Hetzner is using a Proxy service 'kong' which broadcasts it limits - honor 'Retry-After' of 429 responses - delay requests per-se: see the amended docs for details Signed-off-by: Jakob Ackermann <das7pad@outlook.com> * HETZNER: apply review feedback: store quotaName as lower case Co-authored-by: Tom Limoncelli <tlimoncelli@stackoverflow.com> Signed-off-by: Jakob Ackermann <das7pad@outlook.com> Co-authored-by: Tom Limoncelli <tlimoncelli@stackoverflow.com>
This commit is contained in:
parent
18c026d7d9
commit
550fa436ed
|
@ -54,19 +54,60 @@ At this time you cannot update SOA records via DNSControl.
|
|||
|
||||
### Rate Limiting
|
||||
|
||||
In case you are frequently seeing messages about being rate-limited:
|
||||
Hetzner is rate limiting requests in multiple tiers: per Hour, per Minute and
|
||||
per Second.
|
||||
|
||||
{% highlight txt %}
|
||||
WARNING: request rate-limited, constant back-off is now at 1s.
|
||||
{% endhighlight %}
|
||||
Depending on how many requests you are planning to perform, you can adjust the
|
||||
delay between requests in order to stay within your quota.
|
||||
|
||||
You may want to enable the `rate_limited` mode by default.
|
||||
The setting `optimize_for_rate_limit_quota` controls this behavior and accepts
|
||||
a case-insensitive value of
|
||||
- `Hour`
|
||||
- `Minute`
|
||||
- `Second`
|
||||
|
||||
The default for `optimize_for_rate_limit_quota` is `Second`.
|
||||
|
||||
Example: Your per minute quota is 60 requests and in your settings you
|
||||
specified `Minute`. DNSControl will perform at most one request per second.
|
||||
DNSControl will emit a warning in case it breaches the next quota.
|
||||
|
||||
In your `creds.json` for all `HETZNER` provider entries:
|
||||
{% highlight json %}
|
||||
{
|
||||
"hetzner": {
|
||||
"rate_limited": "true",
|
||||
"optimize_for_rate_limit_quota": "Minute",
|
||||
"api_key": "your-api-key"
|
||||
}
|
||||
}
|
||||
{% endhighlight %}
|
||||
|
||||
Every response from the Hetzner DNS Console API includes your limits:
|
||||
|
||||
{% highlight txt %}
|
||||
$ curl --silent --include \
|
||||
--header 'Auth-API-Token: ...' \
|
||||
https://dns.hetzner.com/api/v1/zones \
|
||||
| grep x-ratelimit-limit
|
||||
x-ratelimit-limit-second: 3
|
||||
x-ratelimit-limit-minute: 42
|
||||
x-ratelimit-limit-hour: 1337
|
||||
{% endhighlight %}
|
||||
|
||||
Every DNSControl invocation starts from scratch in regard to rate-limiting.
|
||||
In case you are frequently invoking DNSControl, you will likely hit a limit for
|
||||
any first request.
|
||||
You can either use an out-of-bound delay (e.g. `$ sleep 1`), or specify
|
||||
`start_with_default_rate_limit` in the settings of the provider.
|
||||
With `start_with_default_rate_limit` DNSControl uses a quota equivalent to
|
||||
`x-ratelimit-limit-second: 1` until it could parse the actual quota from an
|
||||
API response.
|
||||
|
||||
In your `creds.json` for all `HETZNER` provider entries:
|
||||
{% highlight json %}
|
||||
{
|
||||
"hetzner": {
|
||||
"start_with_default_rate_limit": "true",
|
||||
"api_key": "your-api-key"
|
||||
}
|
||||
}
|
||||
|
|
|
@ -72,7 +72,8 @@
|
|||
"HETZNER": {
|
||||
"api_key": "$HETZNER_API_KEY",
|
||||
"domain": "$HETZNER_DOMAIN",
|
||||
"rate_limited": "true"
|
||||
"start_with_default_rate_limit": "true",
|
||||
"optimize_for_rate_limit_quota": "Hour"
|
||||
},
|
||||
"HEXONET": {
|
||||
"apientity": "$HEXONET_ENTITY",
|
||||
|
|
|
@ -7,6 +7,8 @@ import (
|
|||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
|
@ -29,6 +31,43 @@ func checkIsLockedSystemRecord(record record) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func getHomogenousDelay(headers http.Header, quotaName string) (time.Duration, error) {
|
||||
quota, err := parseHeaderAsInt(headers, "X-Ratelimit-Limit-"+strings.Title(quotaName))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
var unit time.Duration
|
||||
switch quotaName {
|
||||
case "hour":
|
||||
unit = time.Hour
|
||||
case "minute":
|
||||
unit = time.Minute
|
||||
case "second":
|
||||
unit = time.Second
|
||||
}
|
||||
|
||||
delay := time.Duration(int64(unit) / quota)
|
||||
return delay, nil
|
||||
}
|
||||
|
||||
func getRetryAfterDelay(header http.Header) (time.Duration, error) {
|
||||
retryAfter, err := parseHeaderAsInt(header, "Retry-After")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
delay := time.Duration(retryAfter * int64(time.Second))
|
||||
return delay, nil
|
||||
}
|
||||
|
||||
func parseHeaderAsInt(headers http.Header, headerName string) (int64, error) {
|
||||
value, ok := headers[headerName]
|
||||
if !ok {
|
||||
return 0, fmt.Errorf("header %q is missing", headerName)
|
||||
}
|
||||
return strconv.ParseInt(value[0], 10, 0)
|
||||
}
|
||||
|
||||
func (api *hetznerProvider) bulkCreateRecords(records []record) error {
|
||||
for _, record := range records {
|
||||
if err := checkIsLockedSystemRecord(record); err != nil {
|
||||
|
@ -185,6 +224,8 @@ func (api *hetznerProvider) request(endpoint string, method string, request inte
|
|||
}
|
||||
}
|
||||
|
||||
api.requestRateLimiter.handleResponse(*resp)
|
||||
// retry the request when rate-limited
|
||||
if resp.StatusCode == 429 {
|
||||
api.requestRateLimiter.handleRateLimitedRequest()
|
||||
cleanupResponseBody()
|
||||
|
@ -206,9 +247,12 @@ func (api *hetznerProvider) request(endpoint string, method string, request inte
|
|||
}
|
||||
|
||||
func (api *hetznerProvider) startRateLimited() {
|
||||
// Simulate a request that is getting a 429 response.
|
||||
api.requestRateLimiter.afterRequest()
|
||||
api.requestRateLimiter.bumpDelay()
|
||||
// _Now_ is the best reference we can get for the last request.
|
||||
// Head-On-Head invocations of DNSControl benefit from fewer initial
|
||||
// rate-limited requests.
|
||||
api.requestRateLimiter.lastRequest = time.Now()
|
||||
// use the default delay until we have had a chance to parse limits.
|
||||
api.requestRateLimiter.setDefaultDelay()
|
||||
}
|
||||
|
||||
func (api *hetznerProvider) updateRecord(record record) error {
|
||||
|
@ -223,6 +267,7 @@ func (api *hetznerProvider) updateRecord(record record) error {
|
|||
type requestRateLimiter struct {
|
||||
delay time.Duration
|
||||
lastRequest time.Time
|
||||
optimizeForRateLimitQuota string
|
||||
}
|
||||
|
||||
func (requestRateLimiter *requestRateLimiter) afterRequest() {
|
||||
|
@ -236,23 +281,50 @@ func (requestRateLimiter *requestRateLimiter) beforeRequest() {
|
|||
time.Sleep(time.Until(requestRateLimiter.lastRequest.Add(requestRateLimiter.delay)))
|
||||
}
|
||||
|
||||
func (requestRateLimiter *requestRateLimiter) bumpDelay() string {
|
||||
var backoffType string
|
||||
if requestRateLimiter.delay == 0 {
|
||||
// At the time this provider was implemented (2020-10-18),
|
||||
// one request per second could go though when rate-limited.
|
||||
func (requestRateLimiter *requestRateLimiter) setDefaultDelay() {
|
||||
// default to a rate-limit of 1 req/s -- the next response should update it.
|
||||
requestRateLimiter.delay = time.Second
|
||||
backoffType = "constant"
|
||||
} else {
|
||||
// The initial assumption of 1 req/s may no hold true forever.
|
||||
// Future proof this provider, use exponential back-off.
|
||||
requestRateLimiter.delay = requestRateLimiter.delay * 2
|
||||
backoffType = "exponential"
|
||||
}
|
||||
|
||||
func (requestRateLimiter *requestRateLimiter) setOptimizeForRateLimitQuota(quota string) error {
|
||||
quotaNormalized := strings.ToLower(quota)
|
||||
switch quotaNormalized {
|
||||
case "hour", "minute", "second":
|
||||
requestRateLimiter.optimizeForRateLimitQuota = quotaNormalized
|
||||
case "":
|
||||
requestRateLimiter.optimizeForRateLimitQuota = "second"
|
||||
default:
|
||||
return fmt.Errorf("%q is not a valid quota, expected 'Hour', 'Minute', 'Second' or unset", quota)
|
||||
}
|
||||
return backoffType
|
||||
return nil
|
||||
}
|
||||
|
||||
func (requestRateLimiter *requestRateLimiter) handleRateLimitedRequest() {
|
||||
backoffType := requestRateLimiter.bumpDelay()
|
||||
fmt.Println(fmt.Sprintf("WARNING: request rate-limited, %s back-off is now at %s.", backoffType, requestRateLimiter.delay))
|
||||
message := "Rate-Limited, consider bumping the setting 'optimize_for_rate_limit_quota': %q -> %q"
|
||||
switch requestRateLimiter.optimizeForRateLimitQuota {
|
||||
case "hour":
|
||||
message = "Rate-Limited, you are already using the slowest request rate. Consider contacting the Hetzner Support for raising your quota."
|
||||
case "minute":
|
||||
message = fmt.Sprintf(message, "Minute", "Hour")
|
||||
case "second":
|
||||
message = fmt.Sprintf(message, "Second", "Minute")
|
||||
}
|
||||
fmt.Println(message)
|
||||
}
|
||||
|
||||
func (requestRateLimiter *requestRateLimiter) handleResponse(resp http.Response) {
|
||||
homogenousDelay, err := getHomogenousDelay(resp.Header, requestRateLimiter.optimizeForRateLimitQuota)
|
||||
if err != nil {
|
||||
requestRateLimiter.setDefaultDelay()
|
||||
return
|
||||
}
|
||||
|
||||
delay := homogenousDelay
|
||||
if resp.StatusCode == 429 {
|
||||
retryAfterDelay, err := getRetryAfterDelay(resp.Header)
|
||||
if err == nil {
|
||||
delay = retryAfterDelay
|
||||
}
|
||||
}
|
||||
requestRateLimiter.delay = delay
|
||||
}
|
||||
|
|
|
@ -40,9 +40,19 @@ func New(settings map[string]string, _ json.RawMessage) (providers.DNSServicePro
|
|||
api.apiKey = settings["api_key"]
|
||||
|
||||
if settings["rate_limited"] == "true" {
|
||||
// backwards compatibility
|
||||
settings["start_with_default_rate_limit"] = "true"
|
||||
}
|
||||
if settings["start_with_default_rate_limit"] == "true" {
|
||||
api.startRateLimited()
|
||||
}
|
||||
|
||||
quota := settings["optimize_for_rate_limit_quota"]
|
||||
err := api.requestRateLimiter.setOptimizeForRateLimitQuota(quota)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unexpected value for optimize_for_rate_limit_quota: %w", err)
|
||||
}
|
||||
|
||||
return api, nil
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in a new issue