From ded8f1d4cbeccc6064996d32370551cf0f9a5038 Mon Sep 17 00:00:00 2001 From: Vishal Dalwadi Date: Tue, 26 Aug 2025 17:40:53 +0530 Subject: [PATCH 1/5] feat(go): filter out users and groups using membership; --- pro/auth/sync.go | 80 +++++++++++++++++++-- pro/idp/azure/azure.go | 145 +++++++++++++++++++++++---------------- pro/idp/google/google.go | 5 +- pro/idp/idp.go | 4 +- pro/idp/okta/okta.go | 5 +- 5 files changed, 170 insertions(+), 69 deletions(-) diff --git a/pro/auth/sync.go b/pro/auth/sync.go index e98921ae..bb5a5d7f 100644 --- a/pro/auth/sync.go +++ b/pro/auth/sync.go @@ -3,6 +3,10 @@ package auth import ( "context" "fmt" + "strings" + "sync" + "time" + "github.com/gravitl/netmaker/database" "github.com/gravitl/netmaker/logger" "github.com/gravitl/netmaker/logic" @@ -12,9 +16,6 @@ import ( "github.com/gravitl/netmaker/pro/idp/google" "github.com/gravitl/netmaker/pro/idp/okta" proLogic "github.com/gravitl/netmaker/pro/logic" - "strings" - "sync" - "time" ) var ( @@ -85,15 +86,23 @@ func SyncFromIDP() error { } if settings.AuthProvider != "" && idpClient != nil { - idpUsers, err = idpClient.GetUsers() + idpUsers, err = idpClient.GetUsers(settings.UserFilters) if err != nil { return err } - idpGroups, err = idpClient.GetGroups() + idpGroups, err = idpClient.GetGroups(settings.GroupFilters) if err != nil { return err } + + if len(settings.GroupFilters) > 0 { + idpUsers = filterUsersByGroupMembership(idpUsers, idpGroups) + } + + if len(settings.UserFilters) > 0 { + idpGroups = filterGroupsByMembers(idpGroups, idpUsers) + } } err = syncUsers(idpUsers) @@ -316,3 +325,64 @@ func syncGroups(idpGroups []idp.Group) error { return nil } + +func filterUsersByGroupMembership(idpUsers []idp.User, idpGroups []idp.Group) []idp.User { + usersMap := make(map[string]int) + for i, user := range idpUsers { + usersMap[user.Username] = i + } + + filteredUsersMap := make(map[string]int) + for _, group := range idpGroups { + for _, member := range group.Members { + if userIdx, ok := usersMap[member]; ok { + // user at index `userIdx` is a member of at least one of the + // groups in the `idpGroups` list, so we keep it. + filteredUsersMap[member] = userIdx + } + } + } + + i := 0 + filteredUsers := make([]idp.User, len(filteredUsersMap)) + for _, userIdx := range filteredUsersMap { + filteredUsers[i] = idpUsers[userIdx] + i++ + } + + return filteredUsers +} + +func filterGroupsByMembers(idpGroups []idp.Group, idpUsers []idp.User) []idp.Group { + usersMap := make(map[string]int) + for i, user := range idpUsers { + usersMap[user.Username] = i + } + + filteredGroupsMap := make(map[int]bool) + for i, group := range idpGroups { + var members []string + for _, member := range group.Members { + if _, ok := usersMap[member]; ok { + members = append(members, member) + } + + if len(members) > 0 { + // the group at index `i` has members from the `idpUsers` list, + // so we keep it. + filteredGroupsMap[i] = true + // filter out members that were not provided in the `idpUsers` list. + idpGroups[i].Members = members + } + } + } + + i := 0 + filteredGroups := make([]idp.Group, len(filteredGroupsMap)) + for groupIdx := range filteredGroupsMap { + filteredGroups[i] = idpGroups[groupIdx] + i++ + } + + return filteredGroups +} diff --git a/pro/idp/azure/azure.go b/pro/idp/azure/azure.go index 57fc736d..03e8c737 100644 --- a/pro/idp/azure/azure.go +++ b/pro/idp/azure/azure.go @@ -4,10 +4,11 @@ import ( "encoding/json" "errors" "fmt" - "github.com/gravitl/netmaker/logic" - "github.com/gravitl/netmaker/pro/idp" "net/http" "net/url" + + "github.com/gravitl/netmaker/logic" + "github.com/gravitl/netmaker/pro/idp" ) type Client struct { @@ -26,89 +27,103 @@ func NewAzureEntraIDClient() *Client { } } -func (a *Client) GetUsers() ([]idp.User, error) { +func (a *Client) GetUsers(filters []string) ([]idp.User, error) { accessToken, err := a.getAccessToken() if err != nil { return nil, err } client := &http.Client{} - req, err := http.NewRequest("GET", "https://graph.microsoft.com/v1.0/users?$select=id,userPrincipalName,displayName,accountEnabled", nil) - if err != nil { - return nil, err + getUsersURL := "https://graph.microsoft.com/v1.0/users?$select=id,userPrincipalName,displayName,accountEnabled" + if len(filters) > 0 { + getUsersURL += "&" + buildPrefixFilter("userPrincipalName", filters) } - req.Header.Add("Authorization", "Bearer "+accessToken) - req.Header.Add("Accept", "application/json") - - resp, err := client.Do(req) - if err != nil { - return nil, err - } - defer func() { - _ = resp.Body.Close() - }() - - var users getUsersResponse - err = json.NewDecoder(resp.Body).Decode(&users) - if err != nil { - return nil, err - } - - retval := make([]idp.User, len(users.Value)) - for i, user := range users.Value { - retval[i] = idp.User{ - ID: user.Id, - Username: user.UserPrincipalName, - DisplayName: user.DisplayName, - AccountDisabled: !user.AccountEnabled, + var retval []idp.User + for getUsersURL != "" { + req, err := http.NewRequest("GET", getUsersURL, nil) + if err != nil { + return nil, err } + + req.Header.Add("Authorization", "Bearer "+accessToken) + req.Header.Add("Accept", "application/json") + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + + var users getUsersResponse + err = json.NewDecoder(resp.Body).Decode(&users) + _ = resp.Body.Close() + if err != nil { + return nil, err + } + + for _, user := range users.Value { + retval = append(retval, idp.User{ + ID: user.Id, + Username: user.UserPrincipalName, + DisplayName: user.DisplayName, + AccountDisabled: !user.AccountEnabled, + }) + } + + getUsersURL = users.NextLink } return retval, nil } -func (a *Client) GetGroups() ([]idp.Group, error) { +func (a *Client) GetGroups(filters []string) ([]idp.Group, error) { accessToken, err := a.getAccessToken() if err != nil { return nil, err } client := &http.Client{} - req, err := http.NewRequest("GET", "https://graph.microsoft.com/v1.0/groups?$select=id,displayName&$expand=members($select=id)", nil) - if err != nil { - return nil, err + getGroupsURL := "https://graph.microsoft.com/v1.0/groups?$select=id,displayName&$expand=members($select=id)" + if len(filters) > 0 { + getGroupsURL += "&" + buildPrefixFilter("displayName", filters) } - req.Header.Add("Authorization", "Bearer "+accessToken) - req.Header.Add("Accept", "application/json") + var retval []idp.Group + for getGroupsURL != "" { + req, err := http.NewRequest("GET", getGroupsURL, nil) + if err != nil { + return nil, err + } - resp, err := client.Do(req) - if err != nil { - return nil, err - } - defer func() { + req.Header.Add("Authorization", "Bearer "+accessToken) + req.Header.Add("Accept", "application/json") + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + + var groups getGroupsResponse + err = json.NewDecoder(resp.Body).Decode(&groups) _ = resp.Body.Close() - }() - - var groups getGroupsResponse - err = json.NewDecoder(resp.Body).Decode(&groups) - if err != nil { - return nil, err - } - - retval := make([]idp.Group, len(groups.Value)) - for i, group := range groups.Value { - retvalMembers := make([]string, len(group.Members)) - for j, member := range group.Members { - retvalMembers[j] = member.Id + if err != nil { + return nil, err } - retval[i] = idp.Group{ - ID: group.Id, - Name: group.DisplayName, - Members: retvalMembers, + for _, group := range groups.Value { + retvalMembers := make([]string, len(group.Members)) + for j, member := range group.Members { + retvalMembers[j] = member.Id + } + + retval = append(retval, idp.Group{ + ID: group.Id, + Name: group.DisplayName, + Members: retvalMembers, + }) } + + getGroupsURL = groups.NextLink } return retval, nil @@ -144,6 +159,18 @@ func (a *Client) getAccessToken() (string, error) { return "", errors.New("failed to get access token") } +func buildPrefixFilter(field string, prefixes []string) string { + if len(prefixes) == 0 { + return "" + } + + if len(prefixes) == 1 { + return fmt.Sprintf("$filter=startswith(%s,'%s')", field, prefixes[0]) + } + + return buildPrefixFilter(field, prefixes[1:]) + fmt.Sprintf("%%20or%%20startswith(%s,'%s')", field, prefixes[0]) +} + type getUsersResponse struct { OdataContext string `json:"@odata.context"` Value []struct { @@ -152,6 +179,7 @@ type getUsersResponse struct { DisplayName string `json:"displayName"` AccountEnabled bool `json:"accountEnabled"` } `json:"value"` + NextLink string `json:"@odata.nextLink"` } type getGroupsResponse struct { @@ -164,4 +192,5 @@ type getGroupsResponse struct { Id string `json:"id"` } `json:"members"` } `json:"value"` + NextLink string `json:"@odata.nextLink"` } diff --git a/pro/idp/google/google.go b/pro/idp/google/google.go index f1039ef2..96d2ae63 100644 --- a/pro/idp/google/google.go +++ b/pro/idp/google/google.go @@ -4,6 +4,7 @@ import ( "context" "encoding/base64" "encoding/json" + "github.com/gravitl/netmaker/logic" "github.com/gravitl/netmaker/pro/idp" admindir "google.golang.org/api/admin/directory/v1" @@ -59,7 +60,7 @@ func NewGoogleWorkspaceClient() (*Client, error) { }, nil } -func (g *Client) GetUsers() ([]idp.User, error) { +func (g *Client) GetUsers(filters []string) ([]idp.User, error) { var retval []idp.User err := g.service.Users.List(). Customer("my_customer"). @@ -81,7 +82,7 @@ func (g *Client) GetUsers() ([]idp.User, error) { return retval, err } -func (g *Client) GetGroups() ([]idp.Group, error) { +func (g *Client) GetGroups(filters []string) ([]idp.Group, error) { var retval []idp.Group err := g.service.Groups.List(). Customer("my_customer"). diff --git a/pro/idp/idp.go b/pro/idp/idp.go index 7e190c73..dece85ce 100644 --- a/pro/idp/idp.go +++ b/pro/idp/idp.go @@ -1,8 +1,8 @@ package idp type Client interface { - GetUsers() ([]User, error) - GetGroups() ([]Group, error) + GetUsers(filters []string) ([]User, error) + GetGroups(filters []string) ([]Group, error) } type User struct { diff --git a/pro/idp/okta/okta.go b/pro/idp/okta/okta.go index 497a6e03..449844e2 100644 --- a/pro/idp/okta/okta.go +++ b/pro/idp/okta/okta.go @@ -3,6 +3,7 @@ package okta import ( "context" "fmt" + "github.com/gravitl/netmaker/logic" "github.com/gravitl/netmaker/pro/idp" "github.com/okta/okta-sdk-golang/v5/okta" @@ -42,7 +43,7 @@ func (o *Client) Verify() error { return err } -func (o *Client) GetUsers() ([]idp.User, error) { +func (o *Client) GetUsers(filters []string) ([]idp.User, error) { var retval []idp.User var allUsersFetched bool @@ -81,7 +82,7 @@ func (o *Client) GetUsers() ([]idp.User, error) { return retval, nil } -func (o *Client) GetGroups() ([]idp.Group, error) { +func (o *Client) GetGroups(filters []string) ([]idp.Group, error) { var retval []idp.Group var allGroupsFetched bool From 7f57339c5a6a078b729b0bbfc4824769c1c2584a Mon Sep 17 00:00:00 2001 From: Vishal Dalwadi Date: Tue, 26 Aug 2025 17:53:39 +0530 Subject: [PATCH 2/5] feat(go): remove empty string filters from user and group filters; --- logic/settings.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/logic/settings.go b/logic/settings.go index 5217c2e9..c9fa0b28 100644 --- a/logic/settings.go +++ b/logic/settings.go @@ -38,6 +38,24 @@ func UpsertServerSettings(s models.ServerSettings) error { s.BasicAuth = true } + var userFilters []string + for _, userFilter := range s.UserFilters { + userFilter = strings.TrimSpace(userFilter) + if userFilter != "" { + userFilters = append(userFilters, userFilter) + } + } + s.UserFilters = userFilters + + var groupFilters []string + for _, groupFilter := range s.GroupFilters { + groupFilter = strings.TrimSpace(groupFilter) + if groupFilter != "" { + groupFilters = append(groupFilters, groupFilter) + } + } + s.GroupFilters = groupFilters + data, err := json.Marshal(s) if err != nil { return err From 56c4e4b15577f01cee8b7785c91d87df2d67a280 Mon Sep 17 00:00:00 2001 From: Vishal Dalwadi Date: Tue, 26 Aug 2025 22:04:53 +0530 Subject: [PATCH 3/5] fix(go): use user.id while matching against member id; --- pro/auth/sync.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pro/auth/sync.go b/pro/auth/sync.go index bb5a5d7f..41078dbf 100644 --- a/pro/auth/sync.go +++ b/pro/auth/sync.go @@ -329,7 +329,7 @@ func syncGroups(idpGroups []idp.Group) error { func filterUsersByGroupMembership(idpUsers []idp.User, idpGroups []idp.Group) []idp.User { usersMap := make(map[string]int) for i, user := range idpUsers { - usersMap[user.Username] = i + usersMap[user.ID] = i } filteredUsersMap := make(map[string]int) @@ -356,7 +356,7 @@ func filterUsersByGroupMembership(idpUsers []idp.User, idpGroups []idp.Group) [] func filterGroupsByMembers(idpGroups []idp.Group, idpUsers []idp.User) []idp.Group { usersMap := make(map[string]int) for i, user := range idpUsers { - usersMap[user.Username] = i + usersMap[user.ID] = i } filteredGroupsMap := make(map[int]bool) From 146d907f5d39cac49b77d287a914cc02d7861c95 Mon Sep 17 00:00:00 2001 From: Vishal Dalwadi Date: Wed, 27 Aug 2025 23:17:36 +0530 Subject: [PATCH 4/5] feat(go): add filters support for google and okta; --- pro/idp/azure/azure.go | 2 +- pro/idp/google/google.go | 23 +++++++++++++++++++++++ pro/idp/okta/okta.go | 20 ++++++++++++++++++-- 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/pro/idp/azure/azure.go b/pro/idp/azure/azure.go index 03e8c737..5a01f36d 100644 --- a/pro/idp/azure/azure.go +++ b/pro/idp/azure/azure.go @@ -168,7 +168,7 @@ func buildPrefixFilter(field string, prefixes []string) string { return fmt.Sprintf("$filter=startswith(%s,'%s')", field, prefixes[0]) } - return buildPrefixFilter(field, prefixes[1:]) + fmt.Sprintf("%%20or%%20startswith(%s,'%s')", field, prefixes[0]) + return buildPrefixFilter(field, prefixes[:1]) + "%20or%20" + buildPrefixFilter(field, prefixes[1:]) } type getUsersResponse struct { diff --git a/pro/idp/google/google.go b/pro/idp/google/google.go index 96d2ae63..84ff5f7f 100644 --- a/pro/idp/google/google.go +++ b/pro/idp/google/google.go @@ -4,6 +4,7 @@ import ( "context" "encoding/base64" "encoding/json" + "strings" "github.com/gravitl/netmaker/logic" "github.com/gravitl/netmaker/pro/idp" @@ -67,6 +68,17 @@ func (g *Client) GetUsers(filters []string) ([]idp.User, error) { Fields("users(id,primaryEmail,name,suspended,archived)", "nextPageToken"). Pages(context.TODO(), func(users *admindir.Users) error { for _, user := range users.Users { + var found bool + for _, filter := range filters { + if strings.HasPrefix(user.PrimaryEmail, filter) { + found = true + } + } + + if !found { + continue + } + retval = append(retval, idp.User{ ID: user.Id, Username: user.PrimaryEmail, @@ -89,6 +101,17 @@ func (g *Client) GetGroups(filters []string) ([]idp.Group, error) { Fields("groups(id,name)", "nextPageToken"). Pages(context.TODO(), func(groups *admindir.Groups) error { for _, group := range groups.Groups { + var found bool + for _, filter := range filters { + if strings.HasPrefix(group.Name, filter) { + found = true + } + } + + if !found { + continue + } + var retvalMembers []string err := g.service.Members.List(group.Id). Fields("members(id)", "nextPageToken"). diff --git a/pro/idp/okta/okta.go b/pro/idp/okta/okta.go index 449844e2..f76f2cdd 100644 --- a/pro/idp/okta/okta.go +++ b/pro/idp/okta/okta.go @@ -48,7 +48,9 @@ func (o *Client) GetUsers(filters []string) ([]idp.User, error) { var allUsersFetched bool for !allUsersFetched { - users, resp, err := o.client.UserAPI.ListUsers(context.TODO()).Execute() + users, resp, err := o.client.UserAPI.ListUsers(context.TODO()). + Search(buildPrefixFilter("profile.login", filters)). + Execute() if err != nil { return nil, err } @@ -87,7 +89,9 @@ func (o *Client) GetGroups(filters []string) ([]idp.Group, error) { var allGroupsFetched bool for !allGroupsFetched { - groups, resp, err := o.client.GroupAPI.ListGroups(context.TODO()).Execute() + groups, resp, err := o.client.GroupAPI.ListGroups(context.TODO()). + Search(buildPrefixFilter("profile.name", filters)). + Execute() if err != nil { return nil, err } @@ -123,3 +127,15 @@ func (o *Client) GetGroups(filters []string) ([]idp.Group, error) { return retval, nil } + +func buildPrefixFilter(field string, prefixes []string) string { + if len(prefixes) == 0 { + return "" + } + + if len(prefixes) == 1 { + return fmt.Sprintf("%s sw \"%s\"", field, prefixes[0]) + } + + return buildPrefixFilter(field, prefixes[:1]) + " or " + buildPrefixFilter(field, prefixes[1:]) +} From 5eea06f8878a8f225b99f260c720850804d1795f Mon Sep 17 00:00:00 2001 From: Vishal Dalwadi Date: Thu, 28 Aug 2025 13:58:21 +0530 Subject: [PATCH 5/5] feat(go): apply filters only if they exist; --- pro/idp/google/google.go | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/pro/idp/google/google.go b/pro/idp/google/google.go index 84ff5f7f..c117cc63 100644 --- a/pro/idp/google/google.go +++ b/pro/idp/google/google.go @@ -68,14 +68,18 @@ func (g *Client) GetUsers(filters []string) ([]idp.User, error) { Fields("users(id,primaryEmail,name,suspended,archived)", "nextPageToken"). Pages(context.TODO(), func(users *admindir.Users) error { for _, user := range users.Users { - var found bool - for _, filter := range filters { - if strings.HasPrefix(user.PrimaryEmail, filter) { - found = true + var keep bool + if len(filters) > 0 { + for _, filter := range filters { + if strings.HasPrefix(user.PrimaryEmail, filter) { + keep = true + } } + } else { + keep = true } - if !found { + if !keep { continue } @@ -101,14 +105,18 @@ func (g *Client) GetGroups(filters []string) ([]idp.Group, error) { Fields("groups(id,name)", "nextPageToken"). Pages(context.TODO(), func(groups *admindir.Groups) error { for _, group := range groups.Groups { - var found bool - for _, filter := range filters { - if strings.HasPrefix(group.Name, filter) { - found = true + var keep bool + if len(filters) > 0 { + for _, filter := range filters { + if strings.HasPrefix(group.Name, filter) { + keep = true + } } + } else { + keep = true } - if !found { + if !keep { continue }