NET-2014: add audit log retention period, add timestamp for events (#3486)

* revert inet gws from acl policies

* add egress range with metric for inet gw

* link pro inet funcs

* add timestamp params to activity apis

migrate/migrate.go

@@ -578,4 +578,9 @@ func settings() {
 	if database.IsEmptyRecord(err) {
 		logic.UpsertServerSettings(logic.GetServerSettingsFromEnv())
 	}
+	settings := logic.GetServerSettings()
+	if settings.AuditLogsRetentionPeriodInDays == 0 {
+		settings.AuditLogsRetentionPeriodInDays = 30
+	}
+	logic.UpsertServerSettings(settings)
 }

models/settings.go

@@ -9,38 +9,39 @@ const (
 )
 
 type ServerSettings struct {
-	NetclientAutoUpdate        bool   `json:"netclientautoupdate"`
+	NetclientAutoUpdate            bool     `json:"netclientautoupdate"`
-	Verbosity                  int32  `json:"verbosity"`
+	Verbosity                      int32    `json:"verbosity"`
-	AuthProvider               string `json:"authprovider"`
+	AuthProvider                   string   `json:"authprovider"`
-	OIDCIssuer                 string `json:"oidcissuer"`
+	OIDCIssuer                     string   `json:"oidcissuer"`
-	ClientID                   string `json:"client_id"`
+	ClientID                       string   `json:"client_id"`
-	ClientSecret               string `json:"client_secret"`
+	ClientSecret                   string   `json:"client_secret"`
-	SyncEnabled                bool     `json:"sync_enabled"`
+	SyncEnabled                    bool     `json:"sync_enabled"`
-	GoogleAdminEmail           string   `json:"google_admin_email"`
+	GoogleAdminEmail               string   `json:"google_admin_email"`
-	GoogleSACredsJson          string   `json:"google_sa_creds_json"`
+	GoogleSACredsJson              string   `json:"google_sa_creds_json"`
-	AzureTenant                string `json:"azure_tenant"`
+	AzureTenant                    string   `json:"azure_tenant"`
-	UserFilters                []string `json:"user_filters"`
+	UserFilters                    []string `json:"user_filters"`
-	GroupFilters               []string `json:"group_filters"`
+	GroupFilters                   []string `json:"group_filters"`
-	IDPSyncInterval            string   `json:"idp_sync_interval"`
+	IDPSyncInterval                string   `json:"idp_sync_interval"`
-	Telemetry                  string `json:"telemetry"`
+	Telemetry                      string   `json:"telemetry"`
-	BasicAuth                  bool   `json:"basic_auth"`
+	BasicAuth                      bool     `json:"basic_auth"`
-	JwtValidityDuration        int    `json:"jwt_validity_duration"`
+	JwtValidityDuration            int      `json:"jwt_validity_duration"`
-	RacAutoDisable             bool   `json:"rac_auto_disable"`
+	RacAutoDisable                 bool     `json:"rac_auto_disable"`
-	RacRestrictToSingleNetwork bool   `json:"rac_restrict_to_single_network"`
+	RacRestrictToSingleNetwork     bool     `json:"rac_restrict_to_single_network"`
-	EndpointDetection          bool   `json:"endpoint_detection"`
+	EndpointDetection              bool     `json:"endpoint_detection"`
-	AllowedEmailDomains        string `json:"allowed_email_domains"`
+	AllowedEmailDomains            string   `json:"allowed_email_domains"`
-	EmailSenderAddr            string `json:"email_sender_addr"`
+	EmailSenderAddr                string   `json:"email_sender_addr"`
-	EmailSenderUser            string `json:"email_sender_user"`
+	EmailSenderUser                string   `json:"email_sender_user"`
-	EmailSenderPassword        string `json:"email_sender_password"`
+	EmailSenderPassword            string   `json:"email_sender_password"`
-	SmtpHost                   string `json:"smtp_host"`
+	SmtpHost                       string   `json:"smtp_host"`
-	SmtpPort                   int    `json:"smtp_port"`
+	SmtpPort                       int      `json:"smtp_port"`
-	MetricInterval             string `json:"metric_interval"`
+	MetricInterval                 string   `json:"metric_interval"`
-	MetricsPort                int    `json:"metrics_port"`
+	MetricsPort                    int      `json:"metrics_port"`
-	ManageDNS                  bool   `json:"manage_dns"`
+	ManageDNS                      bool     `json:"manage_dns"`
-	DefaultDomain              string `json:"default_domain"`
+	DefaultDomain                  string   `json:"default_domain"`
-	Stun                       bool   `json:"stun"`
+	Stun                           bool     `json:"stun"`
-	StunServers                string `json:"stun_servers"`
+	StunServers                    string   `json:"stun_servers"`
-	Theme                      Theme  `json:"theme"`
+	Theme                          Theme    `json:"theme"`
-	TextSize                   string `json:"text_size"`
+	TextSize                       string   `json:"text_size"`
-	ReducedMotion              bool   `json:"reduced_motion"`
+	ReducedMotion                  bool     `json:"reduced_motion"`
+	AuditLogsRetentionPeriodInDays int      `json:"audit_logs_retention_period"`
 }

pro/controllers/events.go

@@ -3,6 +3,7 @@ package controllers
 import (
 	"net/http"
 	"strconv"
+	"time"
 
 	"github.com/gorilla/mux"
 	"github.com/gravitl/netmaker/db"
@@ -33,10 +34,33 @@ func listNetworkActivity(w http.ResponseWriter, r *http.Request) {
 		})
 		return
 	}
+	fromDateStr := r.URL.Query().Get("from_date")
+	toDateStr := r.URL.Query().Get("to_date")
+	var err error
+	var fromDate, toDate time.Time
+	if fromDateStr != "" && toDateStr != "" {
+		fromDate, err = time.Parse(time.RFC3339, fromDateStr)
+		if err != nil {
+			logic.ReturnErrorResponse(w, r, models.ErrorResponse{
+				Code:    http.StatusBadRequest,
+				Message: err.Error(),
+			})
+			return
+		}
+		toDate, err = time.Parse(time.RFC3339, toDateStr)
+		if err != nil {
+			logic.ReturnErrorResponse(w, r, models.ErrorResponse{
+				Code:    http.StatusBadRequest,
+				Message: err.Error(),
+			})
+			return
+		}
+	}
+
 	page, _ := strconv.Atoi(r.URL.Query().Get("page"))
 	pageSize, _ := strconv.Atoi(r.URL.Query().Get("per_page"))
 	ctx := db.WithContext(r.Context())
-	netActivity, err := (&schema.Event{NetworkID: models.NetworkID(netID)}).ListByNetwork(db.SetPagination(ctx, page, pageSize))
+	netActivity, err := (&schema.Event{NetworkID: models.NetworkID(netID)}).ListByNetwork(db.SetPagination(ctx, page, pageSize), fromDate, toDate)
 	if err != nil {
 		logic.ReturnErrorResponse(w, r, models.ErrorResponse{
 			Code:    http.StatusInternalServerError,
@@ -64,10 +88,32 @@ func listUserActivity(w http.ResponseWriter, r *http.Request) {
 		})
 		return
 	}
+	fromDateStr := r.URL.Query().Get("from_date")
+	toDateStr := r.URL.Query().Get("to_date")
+	var err error
+	var fromDate, toDate time.Time
+	if fromDateStr != "" && toDateStr != "" {
+		fromDate, err = time.Parse(time.RFC3339, fromDateStr)
+		if err != nil {
+			logic.ReturnErrorResponse(w, r, models.ErrorResponse{
+				Code:    http.StatusBadRequest,
+				Message: err.Error(),
+			})
+			return
+		}
+		toDate, err = time.Parse(time.RFC3339, toDateStr)
+		if err != nil {
+			logic.ReturnErrorResponse(w, r, models.ErrorResponse{
+				Code:    http.StatusBadRequest,
+				Message: err.Error(),
+			})
+			return
+		}
+	}
 	page, _ := strconv.Atoi(r.URL.Query().Get("page"))
 	pageSize, _ := strconv.Atoi(r.URL.Query().Get("per_page"))
 	ctx := db.WithContext(r.Context())
-	userActivity, err := (&schema.Event{TriggeredBy: username}).ListByUser(db.SetPagination(ctx, page, pageSize))
+	userActivity, err := (&schema.Event{TriggeredBy: username}).ListByUser(db.SetPagination(ctx, page, pageSize), fromDate, toDate)
 	if err != nil {
 		logic.ReturnErrorResponse(w, r, models.ErrorResponse{
 			Code:    http.StatusInternalServerError,
@@ -91,16 +137,37 @@ func listActivity(w http.ResponseWriter, r *http.Request) {
 	pageSize, _ := strconv.Atoi(r.URL.Query().Get("per_page"))
 	ctx := db.WithContext(r.Context())
 	var err error
+	fromDateStr := r.URL.Query().Get("from_date")
+	toDateStr := r.URL.Query().Get("to_date")
+	var fromDate, toDate time.Time
+	if fromDateStr != "" && toDateStr != "" {
+		fromDate, err = time.Parse(time.RFC3339, fromDateStr)
+		if err != nil {
+			logic.ReturnErrorResponse(w, r, models.ErrorResponse{
+				Code:    http.StatusBadRequest,
+				Message: err.Error(),
+			})
+			return
+		}
+		toDate, err = time.Parse(time.RFC3339, toDateStr)
+		if err != nil {
+			logic.ReturnErrorResponse(w, r, models.ErrorResponse{
+				Code:    http.StatusBadRequest,
+				Message: err.Error(),
+			})
+			return
+		}
+	}
 	var events []schema.Event
 	e := &schema.Event{TriggeredBy: username, NetworkID: models.NetworkID(network)}
 	if username != "" && network != "" {
-		events, err = e.ListByUserAndNetwork(db.SetPagination(ctx, page, pageSize))
+		events, err = e.ListByUserAndNetwork(db.SetPagination(ctx, page, pageSize), fromDate, toDate)
 	} else if username != "" && network == "" {
-		events, err = e.ListByUser(db.SetPagination(ctx, page, pageSize))
+		events, err = e.ListByUser(db.SetPagination(ctx, page, pageSize), fromDate, toDate)
 	} else if username == "" && network != "" {
-		events, err = e.ListByNetwork(db.SetPagination(ctx, page, pageSize))
+		events, err = e.ListByNetwork(db.SetPagination(ctx, page, pageSize), fromDate, toDate)
 	} else {
-		events, err = e.List(db.SetPagination(ctx, page, pageSize))
+		events, err = e.List(db.SetPagination(ctx, page, pageSize), fromDate, toDate)
 	}
 	if err != nil {
 		logic.ReturnErrorResponse(w, r, models.ErrorResponse{

pro/logic/events.go

@@ -3,11 +3,13 @@ package logic
 import (
 	"context"
 	"encoding/json"
+	"log/slog"
 	"time"
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/uuid"
 	"github.com/gravitl/netmaker/db"
+	"github.com/gravitl/netmaker/logic"
 	"github.com/gravitl/netmaker/models"
 	"github.com/gravitl/netmaker/schema"
 )
@@ -18,8 +20,25 @@ func LogEvent(a *models.Event) {
 	EventActivityCh <- *a
 }
 
-func EventWatcher() {
+func EventRententionHook() error {
+	settings := logic.GetServerSettings()
+	retentionPeriod := settings.AuditLogsRetentionPeriodInDays
+	if retentionPeriod <= 0 {
+		retentionPeriod = 30
+	}
+	err := (&schema.Event{}).DeleteOldEvents(db.WithContext(context.TODO()), retentionPeriod)
+	if err != nil {
+		slog.Warn("failed to delete old events pas retention period", "error", err)
+	}
+	return nil
 
+}
+
+func EventWatcher() {
+	logic.HookManagerCh <- models.HookDetails{
+		Hook:     EventRententionHook,
+		Interval: time.Hour * 24,
+	}
 	for e := range EventActivityCh {
 		if e.Action == models.Update {
 			// check if diff

schema/activity.go

@@ -1,4 +0,0 @@
-package schema
-
-type Activity struct {
-}

schema/event.go

@@ -33,23 +33,49 @@ func (a *Event) Create(ctx context.Context) error {
 	return db.FromContext(ctx).Model(&Event{}).Create(&a).Error
 }
 
-func (a *Event) ListByNetwork(ctx context.Context) (ats []Event, err error) {
+func (a *Event) ListByNetwork(ctx context.Context, from, to time.Time) (ats []Event, err error) {
+	if !from.IsZero() && !to.IsZero() {
+		// "created_at BETWEEN ? AND ?
+		err = db.FromContext(ctx).Model(&Event{}).Where("network_id = ? AND time_stamp BETWEEN ? AND ?",
+			a.NetworkID, from, to).Order("time_stamp DESC").Find(&ats).Error
+		return
+	}
 	err = db.FromContext(ctx).Model(&Event{}).Where("network_id = ?", a.NetworkID).Order("time_stamp DESC").Find(&ats).Error
+
 	return
 }
 
-func (a *Event) ListByUser(ctx context.Context) (ats []Event, err error) {
+func (a *Event) ListByUser(ctx context.Context, from, to time.Time) (ats []Event, err error) {
+	if !from.IsZero() && !to.IsZero() {
+		err = db.FromContext(ctx).Model(&Event{}).Where("triggered_by = ? AND time_stamp BETWEEN ? AND ?",
+			a.TriggeredBy, from, to).Order("time_stamp DESC").Find(&ats).Error
+		return
+	}
 	err = db.FromContext(ctx).Model(&Event{}).Where("triggered_by = ?", a.TriggeredBy).Order("time_stamp DESC").Find(&ats).Error
 	return
 }
 
-func (a *Event) ListByUserAndNetwork(ctx context.Context) (ats []Event, err error) {
+func (a *Event) ListByUserAndNetwork(ctx context.Context, from, to time.Time) (ats []Event, err error) {
+	if !from.IsZero() && !to.IsZero() {
+		err = db.FromContext(ctx).Model(&Event{}).Where("network_id = ? AND triggered_by = ? AND time_stamp BETWEEN ? AND ?",
+			a.NetworkID, a.TriggeredBy, from, to).Order("time_stamp DESC").Find(&ats).Error
+		return
+	}
 	err = db.FromContext(ctx).Model(&Event{}).Where("network_id = ? AND triggered_by = ?",
 		a.NetworkID, a.TriggeredBy).Order("time_stamp DESC").Find(&ats).Error
 	return
 }
 
-func (a *Event) List(ctx context.Context) (ats []Event, err error) {
+func (a *Event) List(ctx context.Context, from, to time.Time) (ats []Event, err error) {
+	if !from.IsZero() && !to.IsZero() {
+		err = db.FromContext(ctx).Model(&Event{}).Where("time_stamp BETWEEN ? AND ?", from, to).Order("time_stamp DESC").Find(&ats).Error
+		return
+	}
 	err = db.FromContext(ctx).Model(&Event{}).Order("time_stamp DESC").Find(&ats).Error
 	return
 }
+
+func (a *Event) DeleteOldEvents(ctx context.Context, retentionDays int) error {
+	cutoff := time.Now().AddDate(0, 0, -retentionDays)
+	return db.FromContext(ctx).Model(&Event{}).Where("created_at < ?", cutoff).Delete(&Event{}).Error
+}

scripts/netmaker.default.env

@@ -105,3 +105,4 @@ PUBLISH_METRIC_INTERVAL=15
 # auto delete offline nodes
 AUTO_DELETE_OFFLINE_NODES=false
 
+