add debug option to save all map responses

Signed-off-by: Kristoffer Dalby <kristoffer@tailscale.com>
This commit is contained in:
Kristoffer Dalby 2023-07-17 11:13:48 +02:00 committed by Kristoffer Dalby
parent f73172fb21
commit 78268d78a0
3 changed files with 76 additions and 4 deletions

View file

@ -4,10 +4,14 @@ import (
"encoding/binary" "encoding/binary"
"encoding/json" "encoding/json"
"fmt" "fmt"
"io/fs"
"net/url" "net/url"
"os"
"path"
"sort" "sort"
"strings" "strings"
"sync" "sync"
"sync/atomic"
"time" "time"
mapset "github.com/deckarep/golang-set/v2" mapset "github.com/deckarep/golang-set/v2"
@ -18,6 +22,7 @@ import (
"github.com/klauspost/compress/zstd" "github.com/klauspost/compress/zstd"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
"github.com/samber/lo" "github.com/samber/lo"
"tailscale.com/envknob"
"tailscale.com/smallzstd" "tailscale.com/smallzstd"
"tailscale.com/tailcfg" "tailscale.com/tailcfg"
"tailscale.com/types/dnstype" "tailscale.com/types/dnstype"
@ -29,6 +34,8 @@ const (
reservedResponseHeaderSize = 4 reservedResponseHeaderSize = 4
) )
var debugDumpMapResponsePath = envknob.String("HEADSCALE_DEBUG_DUMP_MAPRESPONSE_PATH")
type Mapper struct { type Mapper struct {
db *db.HSDatabase db *db.HSDatabase
@ -413,6 +420,41 @@ func (m Mapper) marshalMapResponse(
Msg("Cannot marshal map response") Msg("Cannot marshal map response")
} }
if debugDumpMapResponsePath != "" {
data := map[string]interface{}{
"MapRequest": mapRequest,
"MapResponse": resp,
}
body, err := json.Marshal(data)
if err != nil {
log.Error().
Caller().
Err(err).
Msg("Cannot marshal map response")
}
perms := fs.FileMode(debugMapResponsePerm)
mPath := path.Join(debugDumpMapResponsePath, machine.Hostname)
err = os.MkdirAll(mPath, perms)
if err != nil {
panic(err)
}
now := time.Now().Unix()
mapResponsePath := path.Join(
mPath,
fmt.Sprintf("%d-%s-%d.json", now, m.uid, atomic.LoadUint64(&m.seq)),
)
log.Trace().Msgf("Writing MapResponse to %s", mapResponsePath)
err = os.WriteFile(mapResponsePath, body, perms)
if err != nil {
panic(err)
}
}
var respBody []byte var respBody []byte
if compression == util.ZstdCompression { if compression == util.ZstdCompression {
respBody = zstdEncode(jsonBody) respBody = zstdEncode(jsonBody)

View file

@ -225,8 +225,7 @@ func (h *Headscale) pollNetMapStream(
h.pollNetMapStreamWG.Add(1) h.pollNetMapStreamWG.Add(1)
defer h.pollNetMapStreamWG.Done() defer h.pollNetMapStreamWG.Done()
const chanSize = 8 updateChan := make(chan types.StateUpdate)
updateChan := make(chan types.StateUpdate, chanSize)
defer closeChanWithLog(updateChan, machine.Hostname, "updateChan") defer closeChanWithLog(updateChan, machine.Hostname, "updateChan")
// Register the node's update channel // Register the node's update channel
@ -271,14 +270,18 @@ func (h *Headscale) pollNetMapStream(
var err error var err error
switch update.Type { switch update.Type {
case types.StateFullUpdate:
data, err = mapp.FullMapResponse(mapRequest, machine, h.ACLPolicy)
case types.StatePeerChanged: case types.StatePeerChanged:
logInfo("Sending PeerChanged MapResponse")
data, err = mapp.PeerChangedResponse(mapRequest, machine, update.Changed, h.ACLPolicy) data, err = mapp.PeerChangedResponse(mapRequest, machine, update.Changed, h.ACLPolicy)
case types.StatePeerRemoved: case types.StatePeerRemoved:
logInfo("Sending PeerRemoved MapResponse")
data, err = mapp.PeerRemovedResponse(mapRequest, machine, update.Removed) data, err = mapp.PeerRemovedResponse(mapRequest, machine, update.Removed)
case types.StateDERPUpdated: case types.StateDERPUpdated:
logInfo("Sending DERPUpdate MapResponse")
data, err = mapp.DERPMapResponse(mapRequest, machine, update.DERPMap) data, err = mapp.DERPMapResponse(mapRequest, machine, update.DERPMap)
case types.StateFullUpdate:
logInfo("Sending Full MapResponse")
data, err = mapp.FullMapResponse(mapRequest, machine, h.ACLPolicy)
} }
if err != nil { if err != nil {

View file

@ -212,6 +212,7 @@ func New(
env := []string{ env := []string{
"HEADSCALE_PROFILING_ENABLED=1", "HEADSCALE_PROFILING_ENABLED=1",
"HEADSCALE_PROFILING_PATH=/tmp/profile", "HEADSCALE_PROFILING_PATH=/tmp/profile",
"HEADSCALE_DEBUG_DUMP_MAPRESPONSE_PATH=/tmp/mapresponses",
} }
for key, value := range hsic.env { for key, value := range hsic.env {
env = append(env, fmt.Sprintf("%s=%s", key, value)) env = append(env, fmt.Sprintf("%s=%s", key, value))
@ -339,6 +340,14 @@ func (t *HeadscaleInContainer) Shutdown() error {
) )
} }
err = t.SaveMapResponses("/tmp/control")
if err != nil {
log.Printf(
"Failed to save mapresponses from control: %s",
fmt.Errorf("failed to save mapresponses from control: %w", err),
)
}
return t.pool.Purge(t.container) return t.pool.Purge(t.container)
} }
@ -354,6 +363,24 @@ func (t *HeadscaleInContainer) SaveProfile(savePath string) error {
return err return err
} }
err = os.WriteFile(
path.Join(savePath, t.hostname+"maps.tar"),
tarFile,
os.ModePerm,
)
if err != nil {
return err
}
return nil
}
func (t *HeadscaleInContainer) SaveMapResponses(savePath string) error {
tarFile, err := t.FetchPath("/tmp/mapresponses")
if err != nil {
return err
}
err = os.WriteFile( err = os.WriteFile(
path.Join(savePath, t.hostname+".pprof.tar"), path.Join(savePath, t.hostname+".pprof.tar"),
tarFile, tarFile,