mirror of
https://github.com/StackExchange/dnscontrol.git
synced 2024-09-20 14:56:20 +08:00
HETZNER: better rate limit handling (#936)
* HETZNER: better rate limit handling - Hetzner is using a Proxy service 'kong' which broadcasts it limits - honor 'Retry-After' of 429 responses - delay requests per-se: see the amended docs for details Signed-off-by: Jakob Ackermann <das7pad@outlook.com> * HETZNER: apply review feedback: store quotaName as lower case Co-authored-by: Tom Limoncelli <tlimoncelli@stackoverflow.com> Signed-off-by: Jakob Ackermann <das7pad@outlook.com> Co-authored-by: Tom Limoncelli <tlimoncelli@stackoverflow.com>
This commit is contained in:
parent
18c026d7d9
commit
550fa436ed
|
@ -54,19 +54,60 @@ At this time you cannot update SOA records via DNSControl.
|
||||||
|
|
||||||
### Rate Limiting
|
### Rate Limiting
|
||||||
|
|
||||||
In case you are frequently seeing messages about being rate-limited:
|
Hetzner is rate limiting requests in multiple tiers: per Hour, per Minute and
|
||||||
|
per Second.
|
||||||
|
|
||||||
{% highlight txt %}
|
Depending on how many requests you are planning to perform, you can adjust the
|
||||||
WARNING: request rate-limited, constant back-off is now at 1s.
|
delay between requests in order to stay within your quota.
|
||||||
{% endhighlight %}
|
|
||||||
|
|
||||||
You may want to enable the `rate_limited` mode by default.
|
The setting `optimize_for_rate_limit_quota` controls this behavior and accepts
|
||||||
|
a case-insensitive value of
|
||||||
|
- `Hour`
|
||||||
|
- `Minute`
|
||||||
|
- `Second`
|
||||||
|
|
||||||
|
The default for `optimize_for_rate_limit_quota` is `Second`.
|
||||||
|
|
||||||
|
Example: Your per minute quota is 60 requests and in your settings you
|
||||||
|
specified `Minute`. DNSControl will perform at most one request per second.
|
||||||
|
DNSControl will emit a warning in case it breaches the next quota.
|
||||||
|
|
||||||
In your `creds.json` for all `HETZNER` provider entries:
|
In your `creds.json` for all `HETZNER` provider entries:
|
||||||
{% highlight json %}
|
{% highlight json %}
|
||||||
{
|
{
|
||||||
"hetzner": {
|
"hetzner": {
|
||||||
"rate_limited": "true",
|
"optimize_for_rate_limit_quota": "Minute",
|
||||||
|
"api_key": "your-api-key"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
{% endhighlight %}
|
||||||
|
|
||||||
|
Every response from the Hetzner DNS Console API includes your limits:
|
||||||
|
|
||||||
|
{% highlight txt %}
|
||||||
|
$ curl --silent --include \
|
||||||
|
--header 'Auth-API-Token: ...' \
|
||||||
|
https://dns.hetzner.com/api/v1/zones \
|
||||||
|
| grep x-ratelimit-limit
|
||||||
|
x-ratelimit-limit-second: 3
|
||||||
|
x-ratelimit-limit-minute: 42
|
||||||
|
x-ratelimit-limit-hour: 1337
|
||||||
|
{% endhighlight %}
|
||||||
|
|
||||||
|
Every DNSControl invocation starts from scratch in regard to rate-limiting.
|
||||||
|
In case you are frequently invoking DNSControl, you will likely hit a limit for
|
||||||
|
any first request.
|
||||||
|
You can either use an out-of-bound delay (e.g. `$ sleep 1`), or specify
|
||||||
|
`start_with_default_rate_limit` in the settings of the provider.
|
||||||
|
With `start_with_default_rate_limit` DNSControl uses a quota equivalent to
|
||||||
|
`x-ratelimit-limit-second: 1` until it could parse the actual quota from an
|
||||||
|
API response.
|
||||||
|
|
||||||
|
In your `creds.json` for all `HETZNER` provider entries:
|
||||||
|
{% highlight json %}
|
||||||
|
{
|
||||||
|
"hetzner": {
|
||||||
|
"start_with_default_rate_limit": "true",
|
||||||
"api_key": "your-api-key"
|
"api_key": "your-api-key"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -72,7 +72,8 @@
|
||||||
"HETZNER": {
|
"HETZNER": {
|
||||||
"api_key": "$HETZNER_API_KEY",
|
"api_key": "$HETZNER_API_KEY",
|
||||||
"domain": "$HETZNER_DOMAIN",
|
"domain": "$HETZNER_DOMAIN",
|
||||||
"rate_limited": "true"
|
"start_with_default_rate_limit": "true",
|
||||||
|
"optimize_for_rate_limit_quota": "Hour"
|
||||||
},
|
},
|
||||||
"HEXONET": {
|
"HEXONET": {
|
||||||
"apientity": "$HEXONET_ENTITY",
|
"apientity": "$HEXONET_ENTITY",
|
||||||
|
|
|
@ -7,6 +7,8 @@ import (
|
||||||
"io"
|
"io"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -29,6 +31,43 @@ func checkIsLockedSystemRecord(record record) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func getHomogenousDelay(headers http.Header, quotaName string) (time.Duration, error) {
|
||||||
|
quota, err := parseHeaderAsInt(headers, "X-Ratelimit-Limit-"+strings.Title(quotaName))
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
var unit time.Duration
|
||||||
|
switch quotaName {
|
||||||
|
case "hour":
|
||||||
|
unit = time.Hour
|
||||||
|
case "minute":
|
||||||
|
unit = time.Minute
|
||||||
|
case "second":
|
||||||
|
unit = time.Second
|
||||||
|
}
|
||||||
|
|
||||||
|
delay := time.Duration(int64(unit) / quota)
|
||||||
|
return delay, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func getRetryAfterDelay(header http.Header) (time.Duration, error) {
|
||||||
|
retryAfter, err := parseHeaderAsInt(header, "Retry-After")
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
delay := time.Duration(retryAfter * int64(time.Second))
|
||||||
|
return delay, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseHeaderAsInt(headers http.Header, headerName string) (int64, error) {
|
||||||
|
value, ok := headers[headerName]
|
||||||
|
if !ok {
|
||||||
|
return 0, fmt.Errorf("header %q is missing", headerName)
|
||||||
|
}
|
||||||
|
return strconv.ParseInt(value[0], 10, 0)
|
||||||
|
}
|
||||||
|
|
||||||
func (api *hetznerProvider) bulkCreateRecords(records []record) error {
|
func (api *hetznerProvider) bulkCreateRecords(records []record) error {
|
||||||
for _, record := range records {
|
for _, record := range records {
|
||||||
if err := checkIsLockedSystemRecord(record); err != nil {
|
if err := checkIsLockedSystemRecord(record); err != nil {
|
||||||
|
@ -185,6 +224,8 @@ func (api *hetznerProvider) request(endpoint string, method string, request inte
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
api.requestRateLimiter.handleResponse(*resp)
|
||||||
|
// retry the request when rate-limited
|
||||||
if resp.StatusCode == 429 {
|
if resp.StatusCode == 429 {
|
||||||
api.requestRateLimiter.handleRateLimitedRequest()
|
api.requestRateLimiter.handleRateLimitedRequest()
|
||||||
cleanupResponseBody()
|
cleanupResponseBody()
|
||||||
|
@ -206,9 +247,12 @@ func (api *hetznerProvider) request(endpoint string, method string, request inte
|
||||||
}
|
}
|
||||||
|
|
||||||
func (api *hetznerProvider) startRateLimited() {
|
func (api *hetznerProvider) startRateLimited() {
|
||||||
// Simulate a request that is getting a 429 response.
|
// _Now_ is the best reference we can get for the last request.
|
||||||
api.requestRateLimiter.afterRequest()
|
// Head-On-Head invocations of DNSControl benefit from fewer initial
|
||||||
api.requestRateLimiter.bumpDelay()
|
// rate-limited requests.
|
||||||
|
api.requestRateLimiter.lastRequest = time.Now()
|
||||||
|
// use the default delay until we have had a chance to parse limits.
|
||||||
|
api.requestRateLimiter.setDefaultDelay()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (api *hetznerProvider) updateRecord(record record) error {
|
func (api *hetznerProvider) updateRecord(record record) error {
|
||||||
|
@ -221,8 +265,9 @@ func (api *hetznerProvider) updateRecord(record record) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
type requestRateLimiter struct {
|
type requestRateLimiter struct {
|
||||||
delay time.Duration
|
delay time.Duration
|
||||||
lastRequest time.Time
|
lastRequest time.Time
|
||||||
|
optimizeForRateLimitQuota string
|
||||||
}
|
}
|
||||||
|
|
||||||
func (requestRateLimiter *requestRateLimiter) afterRequest() {
|
func (requestRateLimiter *requestRateLimiter) afterRequest() {
|
||||||
|
@ -236,23 +281,50 @@ func (requestRateLimiter *requestRateLimiter) beforeRequest() {
|
||||||
time.Sleep(time.Until(requestRateLimiter.lastRequest.Add(requestRateLimiter.delay)))
|
time.Sleep(time.Until(requestRateLimiter.lastRequest.Add(requestRateLimiter.delay)))
|
||||||
}
|
}
|
||||||
|
|
||||||
func (requestRateLimiter *requestRateLimiter) bumpDelay() string {
|
func (requestRateLimiter *requestRateLimiter) setDefaultDelay() {
|
||||||
var backoffType string
|
// default to a rate-limit of 1 req/s -- the next response should update it.
|
||||||
if requestRateLimiter.delay == 0 {
|
requestRateLimiter.delay = time.Second
|
||||||
// At the time this provider was implemented (2020-10-18),
|
}
|
||||||
// one request per second could go though when rate-limited.
|
|
||||||
requestRateLimiter.delay = time.Second
|
func (requestRateLimiter *requestRateLimiter) setOptimizeForRateLimitQuota(quota string) error {
|
||||||
backoffType = "constant"
|
quotaNormalized := strings.ToLower(quota)
|
||||||
} else {
|
switch quotaNormalized {
|
||||||
// The initial assumption of 1 req/s may no hold true forever.
|
case "hour", "minute", "second":
|
||||||
// Future proof this provider, use exponential back-off.
|
requestRateLimiter.optimizeForRateLimitQuota = quotaNormalized
|
||||||
requestRateLimiter.delay = requestRateLimiter.delay * 2
|
case "":
|
||||||
backoffType = "exponential"
|
requestRateLimiter.optimizeForRateLimitQuota = "second"
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("%q is not a valid quota, expected 'Hour', 'Minute', 'Second' or unset", quota)
|
||||||
}
|
}
|
||||||
return backoffType
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (requestRateLimiter *requestRateLimiter) handleRateLimitedRequest() {
|
func (requestRateLimiter *requestRateLimiter) handleRateLimitedRequest() {
|
||||||
backoffType := requestRateLimiter.bumpDelay()
|
message := "Rate-Limited, consider bumping the setting 'optimize_for_rate_limit_quota': %q -> %q"
|
||||||
fmt.Println(fmt.Sprintf("WARNING: request rate-limited, %s back-off is now at %s.", backoffType, requestRateLimiter.delay))
|
switch requestRateLimiter.optimizeForRateLimitQuota {
|
||||||
|
case "hour":
|
||||||
|
message = "Rate-Limited, you are already using the slowest request rate. Consider contacting the Hetzner Support for raising your quota."
|
||||||
|
case "minute":
|
||||||
|
message = fmt.Sprintf(message, "Minute", "Hour")
|
||||||
|
case "second":
|
||||||
|
message = fmt.Sprintf(message, "Second", "Minute")
|
||||||
|
}
|
||||||
|
fmt.Println(message)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (requestRateLimiter *requestRateLimiter) handleResponse(resp http.Response) {
|
||||||
|
homogenousDelay, err := getHomogenousDelay(resp.Header, requestRateLimiter.optimizeForRateLimitQuota)
|
||||||
|
if err != nil {
|
||||||
|
requestRateLimiter.setDefaultDelay()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
delay := homogenousDelay
|
||||||
|
if resp.StatusCode == 429 {
|
||||||
|
retryAfterDelay, err := getRetryAfterDelay(resp.Header)
|
||||||
|
if err == nil {
|
||||||
|
delay = retryAfterDelay
|
||||||
|
}
|
||||||
|
}
|
||||||
|
requestRateLimiter.delay = delay
|
||||||
}
|
}
|
||||||
|
|
|
@ -40,9 +40,19 @@ func New(settings map[string]string, _ json.RawMessage) (providers.DNSServicePro
|
||||||
api.apiKey = settings["api_key"]
|
api.apiKey = settings["api_key"]
|
||||||
|
|
||||||
if settings["rate_limited"] == "true" {
|
if settings["rate_limited"] == "true" {
|
||||||
|
// backwards compatibility
|
||||||
|
settings["start_with_default_rate_limit"] = "true"
|
||||||
|
}
|
||||||
|
if settings["start_with_default_rate_limit"] == "true" {
|
||||||
api.startRateLimited()
|
api.startRateLimited()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
quota := settings["optimize_for_rate_limit_quota"]
|
||||||
|
err := api.requestRateLimiter.setOptimizeForRateLimitQuota(quota)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("unexpected value for optimize_for_rate_limit_quota: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
return api, nil
|
return api, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue