From 165467f2fdd910d9bdfd59877eefa2d602041f7a Mon Sep 17 00:00:00 2001 From: 88lex Date: Tue, 2 May 2023 13:08:07 +0800 Subject: [PATCH] Add:Create group;Auto add SA to group;make-tds --- LICENSE | 0 README.md | 32 +++++++- install_fclone | 19 +++++ install_gcloud | 7 ++ make-tds | 28 +++++++ sa-gen_old => old/sa-gen_2020 | 0 old/sa-gen_2022 | 107 ++++++++++++++++++++++++++ sa-gen | 137 ++++++++++------------------------ sa-gen-default.conf | 11 +++ 9 files changed, 242 insertions(+), 99 deletions(-) mode change 100644 => 100755 LICENSE mode change 100644 => 100755 README.md create mode 100755 install_fclone create mode 100755 install_gcloud create mode 100755 make-tds rename sa-gen_old => old/sa-gen_2020 (100%) mode change 100644 => 100755 create mode 100755 old/sa-gen_2022 create mode 100644 sa-gen-default.conf diff --git a/LICENSE b/LICENSE old mode 100644 new mode 100755 diff --git a/README.md b/README.md old mode 100644 new mode 100755 index 41bbff5..3d11c84 --- a/README.md +++ b/README.md @@ -1,8 +1,34 @@ ### **sagen** -NOTE: `sagen` now has a sister application called [safire]( https://github.com/88lex/safire ), a python app with a number of -additional features and better speed. `safire` can be installed via git clone, pip install safire or from Colab. -`sagen` still runs perfectly well for small/medium service account needs, and is based on gcloud sdk rather than python. +May 2023 NOTE: `sa-gen` has been updated to + - Create a Google Group if it does not already exist (set in sa-gen.conf) + - Automatically add service account emails to the specified Google Group + - Install gcloud sdk (if you need) with the `install_gcloud` script + - Use sa-gen.conf to specify variables (You must rename or copy sa-gen-default.conf to sa-gen.conf, then edit it) + - Generate a random alphanumeric SUFFIX for your projects (Google needs a unique project name) + - SUFFIX may also be set manually in `sa-gen.conf` if you like + - Prior versions of sa-gen still run, and have been put in the /old directory + + - A separate, simple script called `make-tds` (from sasync/utils) is added here as well + - `make-tds` will create as many new Shared Drives/Team Drives as you like + - It uses Mawaya's fclone which you can install with `install_fclone` + - If you specify one existing Shared Drive on the same account then all new Shared Drives/TDs will be created with the same Members and permissions as the existing Shared Drive + +THINGS TO CONSIDER: + - Google accounts generally allow 12, 25 or 50 projects depending on the type of account + - You may request additional projects. But Google has been a little more selective about granting additional projects recently + - Each project may have up to 100 service accounts. If you try to create more than 100 you will see an error + - For each service account you can create and download up to 10 keys. If you try to create more than 10 keys then you will see an error + - Projects can be deleted. But they stay in a kind of deleted-project-bin for ~30 days. After the 30 day period your unused project quota will increase again + - Generally, project names are irrelevant. They can be used by most applications/ecosystems regardless of name + - Service accounts and/or their keys can be deleted. Unlike with projects, deleted service accounts and/or keys can be recreated immediately + - The naming of service accounts and their respective email addresses are, as with projects, generally irrelevant. As long as they are added to a Google Group then you can use them with Shared Drives/Team Drives. + +======================= +The documentation below is mostly unchanged, but there may be small differences from the prior version (e.g. slight variable names changes). When I have spare time I will update the docs. If anything glaring, please ping me. + + +NOTE: `sagen` has a sister application called [safire]( https://github.com/88lex/safire ), a python app with a number of additional features and better speed. `safire` can be installed via git clone, pip install safire or from Colab. [[ It has been a while since `safire` was updated, whereas sa-gen still works nicely, but more slowly. ]] Please see: https://github.com/88lex/sa-guide For more info on sagen, safire and setting up service accounts. diff --git a/install_fclone b/install_fclone new file mode 100755 index 0000000..22a434e --- /dev/null +++ b/install_fclone @@ -0,0 +1,19 @@ +#!/bin/bash + +# Download the fclone zip file +wget https://github.com/mawaya/rclone/releases/download/fclone-v0.4.1/fclone-v0.4.1-linux-amd64.zip + +# Unzip the fclone zip file +unzip fclone-v0.4.1-linux-amd64.zip + +# Move the fclone binary to /usr/local/bin +sudo mv fclone-v0.4.1-linux-amd64/fclone /usr/local/bin + +# Make the fclone binary executable +sudo chmod +x /usr/local/bin/fclone + +# Add the fclone binary to your PATH +export PATH=$PATH:/usr/local/bin + +# Test that fclone is installed correctly +fclone -V diff --git a/install_gcloud b/install_gcloud new file mode 100755 index 0000000..7f377cc --- /dev/null +++ b/install_gcloud @@ -0,0 +1,7 @@ +#!/bin/bash +echo "Installing Google Cloud SDK..." +export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" +echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list +curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - +sudo apt-get update && sudo apt-get install google-cloud-sdk +echo "Google Cloud SDK installation complete." diff --git a/make-tds b/make-tds new file mode 100755 index 0000000..63eb991 --- /dev/null +++ b/make-tds @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# This script is a simple wrapper for mawaya's fclone backed command `add-drive` +# Grab the executable from https://github.com/mawaya/rclone/releases and put in /usr/bin/ +# For a couple TDs you can simply use the fclone command manually +# +# `gdrive` is a mydrive or td that has full administrative permissions - used for authentication +# `members` is an existing TD whose members/groups you want to mirror in the new TDs +# `declare -a prefix`` is a common prefix or prefixes for the TDs you want to create [optional] +# `declare -a td` are the non common suffices for the new drives. Keep parentheses and use double quotes +# +# Note: gdrive and members include colon: while prefix and td list do not +# The default entries below would create 10 TDs with the same members/perms as my-td. +# That is, TDs named `goober-001` .. `goober-005` and `gawber-001` .. `gawber-005` + +gdrive="mydrive:" +members="my-td:" +declare -a prefix=("goober" "gawber") +declare -a td=("-001" "-002" "-003" "-004" "-005") + +for j in "${prefix[@]}"; do + for i in "${td[@]}"; do + echo "Creating new TD named $j$i" + fclone backend add-drive $gdrive $j$i --tpslimit=1 -o copy-members=$members + sleep 10s + done +done + +fclone backend lsdrives $gdrive | grep "$prefix" diff --git a/sa-gen_old b/old/sa-gen_2020 old mode 100644 new mode 100755 similarity index 100% rename from sa-gen_old rename to old/sa-gen_2020 diff --git a/old/sa-gen_2022 b/old/sa-gen_2022 new file mode 100755 index 0000000..e9a4988 --- /dev/null +++ b/old/sa-gen_2022 @@ -0,0 +1,107 @@ +#!/bin/bash +# Running this script requires gcloud command line tools. To install go to https://cloud.google.com/sdk/docs/quickstarts +# See readme.md to understand the variables used in this script + +KEYS_DIR=/opt/sa/all # path where you want to store sa JSON files [you will need to add the /all here, most likely] +ORGANIZATION_ID="insertyourorganizationID" # organization ID from gcloud SDK step +GROUP_NAME="mygroup@mydomain.com" # the group [full address as shown] you created previously +PROJECT_BASE_NAME="myprojectbasename" # any unique prefix or the prefix you generated for saltbox +FIRST_PROJECT_NUM=1 +LAST_PROJECT_NUM=3 +SA_EMAIL_BASE_NAME="insertuniquename" # any unique prefix or the prefix you generated for saltbox +FIRST_SA_NUM=1 +NUM_SAS_PER_PROJECT=100 +CYCLE_DELAY=0.1s # If issues with Google back end not recognizing SAs increase this number. Set = 0 for no delay +SECTION_DELAY=5s # If issues with Google back end not recognizing SAs increase this number. Set = 0 for no delay + + +create_projects() { + PROJECT="$PROJECT_BASE_NAME$project_num" + echo -e "Creating project = $PROJECT" + set -x + gcloud projects create $PROJECT --organization=$ORGANIZATION_ID + set +x + sleep $CYCLE_DELAY +} + +enable_apis() { + PROJECT="$PROJECT_BASE_NAME$project_num" + echo -e "Enabling apis for project = $PROJECT" + set -x + gcloud config set project $PROJECT + gcloud services enable drive.googleapis.com sheets.googleapis.com \ + admin.googleapis.com cloudresourcemanager.googleapis.com servicemanagement.googleapis.com + set +x + sleep $CYCLE_DELAY +} + +create_sas() { + PROJECT="$PROJECT_BASE_NAME$project_num" + set -x + gcloud config set project $PROJECT + set +x + echo -e "Create service accounts for project = $PROJECT" + let LAST_SA_NUM=$COUNT+$NUM_SAS_PER_PROJECT-1 + for name in $(seq $COUNT $LAST_SA_NUM); do + saname="$SA_EMAIL_BASE_NAME""$name" + echo -e "Creating service account number $name in project = $PROJECT ==> $saname@$PROJECT" + set -x + gcloud iam service-accounts create $saname --display-name=$saname + set +x + sleep $CYCLE_DELAY + done + sleep $SECTION_DELAY + SA_COUNT=`gcloud iam service-accounts list | grep gservice | wc -l` + echo -e "Total number of service accounts (SAs) in project $PROJECT = $SA_COUNT" + let COUNT=$COUNT+$NUM_SAS_PER_PROJECT +} + +create_keys() { + PROJECT="$PROJECT_BASE_NAME$project_num" + set -x + gcloud config set project $PROJECT + set +x + echo -e "create json keys for $PROJECT" + TOTAL_JSONS_BEF=`ls $KEYS_DIR | grep ".json" | wc -l` + let LAST_SA_NUM=$COUNT+$NUM_SAS_PER_PROJECT-1 + for name in $(seq $COUNT $LAST_SA_NUM); do + saname="$SA_EMAIL_BASE_NAME""$name" + echo -e "Creating json key $name.json in project = $PROJECT for service account = $saname@$PROJECT" + set -x + gcloud iam service-accounts keys create $KEYS_DIR/$name.json --iam-account=$saname@$PROJECT.iam.gserviceaccount.com + set +x + # NEED to fix syntax for below command to add SA email to group + #gcloud iam service-accounts add-iam-policy-binding "$saname@$PROJECT.iam.gserviceaccount.com" --member="group:$GROUP_NAME" --role="roles/editor" + echo "$GROUP_NAME,$saname@$PROJECT.iam.gserviceaccount.com,USER,MEMBER" | tee -a $KEYS_DIR/members.csv $KEYS_DIR/allmembers.csv + sleep $CYCLE_DELAY + done + MEMBER_COUNT=`cat $KEYS_DIR/members.csv | grep "gservice" | wc -l` + echo -e "\nNumber of service accounts in members.csv = $MEMBER_COUNT" + TOTAL_JSONS_NOW=`ls $KEYS_DIR | grep ".json" | wc -l` + let TOTAL_JSONS_MADE=$TOTAL_JSONS_NOW-$TOTAL_JSONS_BEF + echo -e "Total SA json keys created for project $PROJECT = $TOTAL_JSONS_MADE" + let COUNT=$COUNT+$NUM_SAS_PER_PROJECT +} + +main() { + mkdir -p $KEYS_DIR + [ -f $KEYS_DIR/members.csv ] && cat $KEYS_DIR/members.csv >> $KEYS_DIR/allmembers.csv && \ + sort -uo $KEYS_DIR/allmembers.csv $KEYS_DIR/allmembers.csv + echo "Group Email [Required],Member Email,Member Type,Member Role" >$KEYS_DIR/members.csv + TOTAL_JSONS_START=`ls $KEYS_DIR | grep ".json" | wc -l` + echo -e "\nTotal SA json keys before running sa-gen = $TOTAL_JSONS_START" + for function in create_projects enable_apis create_sas create_keys ; do + COUNT=$FIRST_SA_NUM + for project_num in $(seq $FIRST_PROJECT_NUM $LAST_PROJECT_NUM); do + eval $function + sleep $SECTION_DELAY + done + done + TOTAL_JSONS_END=`ls $KEYS_DIR | grep ".json" | wc -l` + echo -e "\n\nTotal SA json keys BEFORE running sa-gen = $TOTAL_JSONS_START" + echo -e "Total SA json keys AFTER running sa-gen = $TOTAL_JSONS_END" + let TOTAL_JSONS_MADE=$TOTAL_JSONS_END-$TOTAL_JSONS_START + echo -e "Total SA jsons CREATED = $TOTAL_JSONS_MADE" +} + +main \ No newline at end of file diff --git a/sa-gen b/sa-gen index d4764ae..092dc61 100755 --- a/sa-gen +++ b/sa-gen @@ -1,107 +1,52 @@ -#!/bin/bash -# Running this script requires gcloud command line tools. To install go to https://cloud.google.com/sdk/docs/quickstarts -# See readme.md to understand the variables used in this script +#!/bin/bash -v -KEYS_DIR=/opt/sa/all # path where you want to store sa JSON files [you will need to add the /all here, most likely] -ORGANIZATION_ID="insertyourorganizationID" # organization ID from gcloud SDK step -GROUP_NAME="mygroup@mydomain.com" # the group [full address as shown] you created previously -PROJECT_BASE_NAME="myprojectbasename" # any unique prefix or the prefix you generated for saltbox -FIRST_PROJECT_NUM=1 -LAST_PROJECT_NUM=3 -SA_EMAIL_BASE_NAME="insertuniquename" # any unique prefix or the prefix you generated for saltbox -FIRST_SA_NUM=1 -NUM_SAS_PER_PROJECT=100 -CYCLE_DELAY=0.1s # If issues with Google back end not recognizing SAs increase this number. Set = 0 for no delay -SECTION_DELAY=5s # If issues with Google back end not recognizing SAs increase this number. Set = 0 for no delay - - -create_projects() { - PROJECT="$PROJECT_BASE_NAME$project_num" - echo -e "Creating project = $PROJECT" - set -x - gcloud projects create $PROJECT --organization=$ORGANIZATION_ID - set +x - sleep $CYCLE_DELAY +# Display gcloud sdk organization and active admin project and enable admin APIs +function check_prereqs() { + gcloud organizations list + ADMIN_PROJECT=`gcloud config get-value project` + gcloud services enable --project "$ADMIN_PROJECT" drive.googleapis.com sheets.googleapis.com \ + cloudidentity.googleapis.com admin.googleapis.com cloudresourcemanager.googleapis.com servicemanagement.googleapis.com + read -p "Check the org and project. If wrong or blank then rerun gcloud init. Continue? y/n " yn + if [[ $yn =~ ^[Nn]$ ]];then exit;fi } -enable_apis() { - PROJECT="$PROJECT_BASE_NAME$project_num" - echo -e "Enabling apis for project = $PROJECT" - set -x - gcloud config set project $PROJECT - gcloud services enable drive.googleapis.com sheets.googleapis.com \ - admin.googleapis.com cloudresourcemanager.googleapis.com servicemanagement.googleapis.com - set +x - sleep $CYCLE_DELAY +function create_projects() { + for i in $(seq -f "%03g" "$FIRST_PROJECT_NUM" "$LAST_PROJECT_NUM"); do + PROJECT_NAME="$PREFIX-$i" + PROJECT_ID="$PROJECT_NAME-$SUFFIX" + gcloud projects create "$PROJECT_ID" --name="$PROJECT_NAME" + gcloud services enable --project "$PROJECT_ID" drive.googleapis.com sheets.googleapis.com + done } -create_sas() { - PROJECT="$PROJECT_BASE_NAME$project_num" - set -x - gcloud config set project $PROJECT - set +x - echo -e "Create service accounts for project = $PROJECT" - let LAST_SA_NUM=$COUNT+$NUM_SAS_PER_PROJECT-1 - for name in $(seq $COUNT $LAST_SA_NUM); do - saname="$SA_EMAIL_BASE_NAME""$name" - echo -e "Creating service account number $name in project = $PROJECT ==> $saname@$PROJECT" - set -x - gcloud iam service-accounts create $saname --display-name=$saname - set +x - sleep $CYCLE_DELAY +function create_group() { + ORG_ID=$(gcloud organizations list --format="value(ID)") + gcloud identity groups create "$GROUP_EMAIL" --description="$GROUP_NAME" --organization="$ORG_ID" +} + +function create_service_accounts() { + JSON_COUNT=$((FJSON-1)) + mkdir -p "$KEYS_DIR/all/" + for i in $(seq -f "%03g" "$FIRST_PROJECT_NUM" "$LAST_PROJECT_NUM"); do + PROJECT_NAME="$PREFIX-$i" + PROJECT_ID="$PROJECT_NAME-$SUFFIX" + for j in $(seq -f "%03g" 1 "$SAS_PER_PROJECT"); do + SA_NAME="$PREFIX-$i-$j" + EMAIL="$SA_NAME@$PROJECT_ID.iam.gserviceaccount.com" + gcloud iam service-accounts create "$SA_NAME" --project="$PROJECT_ID" --display-name="$SA_NAME" + gcloud identity groups memberships add --group-email="$GROUP_EMAIL" --member-email="$EMAIL" + gcloud iam service-accounts keys create "$KEYS_DIR/$PROJECT_NAME/$((++JSON_COUNT)).json" --iam-account="$SA_NAME@$PROJECT_ID.iam.gserviceaccount.com" --project="$PROJECT_ID" done - sleep $SECTION_DELAY - SA_COUNT=`gcloud iam service-accounts list | grep gservice | wc -l` - echo -e "Total number of service accounts (SAs) in project $PROJECT = $SA_COUNT" - let COUNT=$COUNT+$NUM_SAS_PER_PROJECT + cp "$KEYS_DIR/$PROJECT_NAME/"*.json "$KEYS_DIR/all/" + done } -create_keys() { - PROJECT="$PROJECT_BASE_NAME$project_num" - set -x - gcloud config set project $PROJECT - set +x - echo -e "create json keys for $PROJECT" - TOTAL_JSONS_BEF=`ls $KEYS_DIR | grep ".json" | wc -l` - let LAST_SA_NUM=$COUNT+$NUM_SAS_PER_PROJECT-1 - for name in $(seq $COUNT $LAST_SA_NUM); do - saname="$SA_EMAIL_BASE_NAME""$name" - echo -e "Creating json key $name.json in project = $PROJECT for service account = $saname@$PROJECT" - set -x - gcloud iam service-accounts keys create $KEYS_DIR/$name.json --iam-account=$saname@$PROJECT.iam.gserviceaccount.com - set +x - # NEED to fix syntax for below command to add SA email to group - #gcloud iam service-accounts add-iam-policy-binding "$saname@$PROJECT.iam.gserviceaccount.com" --member="group:$GROUP_NAME" --role="roles/editor" - echo "$GROUP_NAME,$saname@$PROJECT.iam.gserviceaccount.com,USER,MEMBER" | tee -a $KEYS_DIR/members.csv $KEYS_DIR/allmembers.csv - sleep $CYCLE_DELAY - done - MEMBER_COUNT=`cat $KEYS_DIR/members.csv | grep "gservice" | wc -l` - echo -e "\nNumber of service accounts in members.csv = $MEMBER_COUNT" - TOTAL_JSONS_NOW=`ls $KEYS_DIR | grep ".json" | wc -l` - let TOTAL_JSONS_MADE=$TOTAL_JSONS_NOW-$TOTAL_JSONS_BEF - echo -e "Total SA json keys created for project $PROJECT = $TOTAL_JSONS_MADE" - let COUNT=$COUNT+$NUM_SAS_PER_PROJECT -} - -main() { - mkdir -p $KEYS_DIR - [ -f $KEYS_DIR/members.csv ] && cat $KEYS_DIR/members.csv >> $KEYS_DIR/allmembers.csv && \ - sort -uo $KEYS_DIR/allmembers.csv $KEYS_DIR/allmembers.csv - echo "Group Email [Required],Member Email,Member Type,Member Role" >$KEYS_DIR/members.csv - TOTAL_JSONS_START=`ls $KEYS_DIR | grep ".json" | wc -l` - echo -e "\nTotal SA json keys before running sa-gen = $TOTAL_JSONS_START" - for function in create_projects enable_apis create_sas create_keys ; do - COUNT=$FIRST_SA_NUM - for project_num in $(seq $FIRST_PROJECT_NUM $LAST_PROJECT_NUM); do - eval $function - sleep $SECTION_DELAY - done - done - TOTAL_JSONS_END=`ls $KEYS_DIR | grep ".json" | wc -l` - echo -e "\n\nTotal SA json keys BEFORE running sa-gen = $TOTAL_JSONS_START" - echo -e "Total SA json keys AFTER running sa-gen = $TOTAL_JSONS_END" - let TOTAL_JSONS_MADE=$TOTAL_JSONS_END-$TOTAL_JSONS_START - echo -e "Total SA jsons CREATED = $TOTAL_JSONS_MADE" +function main() { + check_prereqs + source sa-gen.conf + create_projects + create_group + create_service_accounts } main diff --git a/sa-gen-default.conf b/sa-gen-default.conf new file mode 100644 index 0000000..4471b69 --- /dev/null +++ b/sa-gen-default.conf @@ -0,0 +1,11 @@ +# Copy or rename this file to sa-gen.conf and edit values as needed +KEYS_DIR="/opt/sa" # path where you want to store sa JSON files +FIRST_PROJECT_NUM=1 # First project number +LAST_PROJECT_NUM=10 # Last project number +SAS_PER_PROJECT=100 # Number of service accounts per project (max 100) +FJSON=1 # First json file number when downloading +PREFIX="sa" # Project prefix +SUFFIX=`echo $RANDOM | md5sum | head -c 4` # Project suffix. Random 4 char alpha numeric +# SUFFIX="" # Set this manually to use your own unique suffix +GROUP_NAME="sa-group" # Display name for the group that contains all SA emails +GROUP_EMAIL="sa-group@mydomain.com" # Full email name for group that contains SAs