IDP Group Filtering Update (#3656)

* fix(go): prevent creating network with fully-masked cidr;

* fix(go): filter out static non-user nodes;

* fix(go): prevent creation of networks with only broadcast and network ip;

* fix(go): cleanup user and groups resources on idp sync;

* fix(go): add "$filter" only once;

* fix(go): add "$filter" only once;

* fix(go): add "$filter" only once;

* fix(go): escape "$filter";

* feat(go): assign synced group global user role id;

---------

Co-authored-by: VishalDalwadi <dalwadivishal26@gmail.com>
Co-authored-by: Vishal Dalwadi <51291657+VishalDalwadi@users.noreply.github.com>

pro/auth/sync.go

@@ -11,11 +11,13 @@ import (
 	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/logic"
 	"github.com/gravitl/netmaker/models"
+	"github.com/gravitl/netmaker/mq"
 	"github.com/gravitl/netmaker/pro/idp"
 	"github.com/gravitl/netmaker/pro/idp/azure"
 	"github.com/gravitl/netmaker/pro/idp/google"
 	"github.com/gravitl/netmaker/pro/idp/okta"
 	proLogic "github.com/gravitl/netmaker/pro/logic"
+	"github.com/gravitl/netmaker/servercfg"
 )
 
 var (
@@ -149,7 +151,8 @@ func syncUsers(idpUsers []idp.User) error {
 	for _, user := range idpUsers {
 		if user.AccountArchived {
 			// delete the user if it has been archived.
-			_ = logic.DeleteUser(user.Username)
+			user := dbUsersMap[user.Username]
+			_ = deleteAndCleanUpUser(&user)
 			continue
 		}
 
@@ -209,14 +212,14 @@ func syncUsers(idpUsers []idp.User) error {
 	}
 
 	for _, user := range dbUsersMap {
-		if user.ExternalIdentityProviderID == "" {
+		if user.ExternalIdentityProviderID != "" {
-			continue
+			if _, ok := idpUsersMap[user.UserName]; !ok {
-		}
+				// delete the user if it has been deleted on idp
-		if _, ok := idpUsersMap[user.UserName]; !ok {
+				// or is filtered out.
-			// delete the user if it has been deleted on idp.
+				err = deleteAndCleanUpUser(&user)
-			err = logic.DeleteUser(user.UserName)
+				if err != nil {
-			if err != nil {
+					return err
-				return err
+				}
 			}
 		}
 	}
@@ -277,7 +280,11 @@ func syncGroups(idpGroups []idp.Group) error {
 			dbGroup.ExternalIdentityProviderID = group.ID
 			dbGroup.Name = group.Name
 			dbGroup.Default = false
-			dbGroup.NetworkRoles = make(map[models.NetworkID]map[models.UserRoleID]struct{})
+			dbGroup.NetworkRoles = map[models.NetworkID]map[models.UserRoleID]struct{}{
+				models.AllNetworks: {
+					proLogic.GetDefaultGlobalUserRoleID(): {},
+				},
+			}
 			err := proLogic.CreateUserGroup(&dbGroup)
 			if err != nil {
 				return err
@@ -324,8 +331,9 @@ func syncGroups(idpGroups []idp.Group) error {
 	for _, group := range dbGroups {
 		if group.ExternalIdentityProviderID != "" {
 			if _, ok := idpGroupsMap[group.ExternalIdentityProviderID]; !ok {
-				// delete the group if it has been deleted on idp.
+				// delete the group if it has been deleted on idp
-				err = proLogic.DeleteUserGroup(group.ID)
+				// or is filtered out.
+				err = proLogic.DeleteAndCleanUpGroup(&group)
 				if err != nil {
 					return err
 				}
@@ -355,6 +363,7 @@ func GetIDPSyncStatus() models.IDPSyncStatus {
 		}
 	}
 }
+
 func filterUsersByGroupMembership(idpUsers []idp.User, idpGroups []idp.Group) []idp.User {
 	usersMap := make(map[string]int)
 	for i, user := range idpUsers {
@@ -395,14 +404,14 @@ func filterGroupsByMembers(idpGroups []idp.Group, idpUsers []idp.User) []idp.Gro
 			if _, ok := usersMap[member]; ok {
 				members = append(members, member)
 			}
+		}
 
-			if len(members) > 0 {
+		if len(members) > 0 {
-				// the group at index `i` has members from the `idpUsers` list,
+			// the group at index `i` has members from the `idpUsers` list,
-				// so we keep it.
+			// so we keep it.
-				filteredGroupsMap[i] = true
+			filteredGroupsMap[i] = true
-				// filter out members that were not provided in the `idpUsers` list.
+			// filter out members that were not provided in the `idpUsers` list.
-				idpGroups[i].Members = members
+			idpGroups[i].Members = members
-			}
 		}
 	}
 
@@ -415,3 +424,37 @@ func filterGroupsByMembers(idpGroups []idp.Group, idpUsers []idp.User) []idp.Gro
 
 	return filteredGroups
 }
+
+// TODO: deduplicate
+// The cyclic import between the package logic and mq requires this
+// function to be duplicated in multiple places.
+func deleteAndCleanUpUser(user *models.User) error {
+	err := logic.DeleteUser(user.UserName)
+	if err != nil {
+		return err
+	}
+
+	// check and delete extclient with this ownerID
+	go func() {
+		extclients, err := logic.GetAllExtClients()
+		if err != nil {
+			return
+		}
+		for _, extclient := range extclients {
+			if extclient.OwnerID == user.UserName {
+				err = logic.DeleteExtClientAndCleanup(extclient)
+				if err == nil {
+					_ = mq.PublishDeletedClientPeerUpdate(&extclient)
+				}
+			}
+		}
+
+		go logic.DeleteUserInvite(user.UserName)
+		go mq.PublishPeerUpdate(false)
+		if servercfg.IsDNSMode() {
+			go logic.SetDNS()
+		}
+	}()
+
+	return nil
+}

pro/controllers/users.go

@@ -808,11 +808,13 @@ func deleteUserGroup(w http.ResponseWriter, r *http.Request) {
 		logic.ReturnErrorResponse(w, r, logic.FormatError(errors.New("cannot delete default user group"), "badrequest"))
 		return
 	}
-	err = proLogic.DeleteUserGroup(models.UserGroupID(gid))
+	err = proLogic.DeleteAndCleanUpGroup(&userG)
 	if err != nil {
 		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
 		return
 	}
+
+	// TODO: log event in proLogic.DeleteAndCleanUpGroup so that all deletions are logged.
 	logic.LogEvent(&models.Event{
 		Action: models.Delete,
 		Source: models.Subject{
@@ -828,42 +830,7 @@ func deleteUserGroup(w http.ResponseWriter, r *http.Request) {
 		},
 		Origin: models.Dashboard,
 	})
-	replacePeers := false
-	go func() {
-		for networkID := range userG.NetworkRoles {
-			acls, err := logic.ListAclsByNetwork(networkID)
-			if err != nil {
-				continue
-			}
 
-			for _, acl := range acls {
-				var hasGroupSrc bool
-				newAclSrc := make([]models.AclPolicyTag, 0)
-				for _, src := range acl.Src {
-					if src.ID == models.UserGroupAclID && src.Value == userG.ID.String() {
-						hasGroupSrc = true
-					} else {
-						newAclSrc = append(newAclSrc, src)
-					}
-				}
-
-				if hasGroupSrc {
-					if len(newAclSrc) == 0 {
-						// no other src exists, delete acl.
-						_ = logic.DeleteAcl(acl)
-					} else {
-						// other sources exist, update acl.
-						acl.Src = newAclSrc
-						_ = logic.UpsertAcl(acl)
-					}
-					replacePeers = true
-				}
-			}
-		}
-	}()
-
-	go proLogic.UpdatesUserGwAccessOnGrpUpdates(userG.ID, userG.NetworkRoles, make(map[models.NetworkID]map[models.UserRoleID]struct{}))
-	go mq.PublishPeerUpdate(replacePeers)
 	logic.ReturnSuccessResponseWithJson(w, r, nil, "deleted user group")
 }
 

pro/idp/azure/azure.go

@@ -226,15 +226,19 @@ func (a *Client) getAccessToken() (string, error) {
 }
 
 func buildPrefixFilter(field string, prefixes []string) string {
+	return url.PathEscape("$filter=" + buildCondition(field, prefixes))
+}
+
+func buildCondition(field string, prefixes []string) string {
 	if len(prefixes) == 0 {
 		return ""
 	}
 
 	if len(prefixes) == 1 {
-		return fmt.Sprintf("$filter=startswith(%s,'%s')", field, prefixes[0])
+		return fmt.Sprintf("startswith(%s,'%s')", field, prefixes[0])
 	}
 
-	return buildPrefixFilter(field, prefixes[:1]) + "%20or%20" + buildPrefixFilter(field, prefixes[1:])
+	return buildCondition(field, prefixes[:1]) + " or " + buildCondition(field, prefixes[1:])
 }
 
 type getUsersResponse struct {

pro/logic/user_mgmt.go

@@ -620,6 +620,22 @@ func GetUserGroup(gid models.UserGroupID) (models.UserGroup, error) {
 	return ug, nil
 }
 
+func GetDefaultGlobalAdminGroupID() models.UserGroupID {
+	return globalNetworksAdminGroupID
+}
+
+func GetDefaultGlobalUserGroupID() models.UserGroupID {
+	return globalNetworksUserGroupID
+}
+
+func GetDefaultGlobalAdminRoleID() models.UserRoleID {
+	return globalNetworksAdminRoleID
+}
+
+func GetDefaultGlobalUserRoleID() models.UserRoleID {
+	return globalNetworksUserRoleID
+}
+
 func GetDefaultNetworkAdminGroupID(networkID models.NetworkID) models.UserGroupID {
 	return models.UserGroupID(fmt.Sprintf("%s-%s-grp", networkID, models.NetworkAdmin))
 }
@@ -672,6 +688,52 @@ func UpdateUserGroup(g models.UserGroup) error {
 	return database.Insert(g.ID.String(), string(d), database.USER_GROUPS_TABLE_NAME)
 }
 
+func DeleteAndCleanUpGroup(group *models.UserGroup) error {
+	err := DeleteUserGroup(group.ID)
+	if err != nil {
+		return err
+	}
+
+	go func() {
+		var replacePeers bool
+		for networkID := range group.NetworkRoles {
+			acls, err := logic.ListAclsByNetwork(networkID)
+			if err != nil {
+				continue
+			}
+
+			for _, acl := range acls {
+				var hasGroupSrc bool
+				newAclSrc := make([]models.AclPolicyTag, 0)
+				for _, src := range acl.Src {
+					if src.ID == models.UserGroupAclID && src.Value == group.ID.String() {
+						hasGroupSrc = true
+					} else {
+						newAclSrc = append(newAclSrc, src)
+					}
+				}
+
+				if hasGroupSrc {
+					if len(newAclSrc) == 0 {
+						// no other src exists, delete acl.
+						_ = logic.DeleteAcl(acl)
+					} else {
+						// other sources exist, update acl.
+						acl.Src = newAclSrc
+						_ = logic.UpsertAcl(acl)
+					}
+					replacePeers = true
+				}
+			}
+		}
+
+		go UpdatesUserGwAccessOnGrpUpdates(group.ID, group.NetworkRoles, make(map[models.NetworkID]map[models.UserRoleID]struct{}))
+		go mq.PublishPeerUpdate(replacePeers)
+	}()
+
+	return nil
+}
+
 // DeleteUserGroup - deletes user group
 func DeleteUserGroup(gid models.UserGroupID) error {
 	g, err := GetUserGroup(gid)