88lex/safire
1
1
Fork 0
mirror of https://github.com/88lex/safire.git synced 2024-09-20 14:56:06 +08:00
Code Issues Packages Projects Releases Wiki Activity

Created using Colaboratory

Browse source
This commit is contained in:
88lex 2020-08-12 22:20:32 +08:00
parent 28b4cf9c01
commit b18b5e2213
1 changed files with 48 additions and 63 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

111
safire.ipynb
View file

@ -7,12 +7,7 @@
"provenance": [],
"private_outputs": true,
"collapsed_sections": [
"1FV-VHZsLkhG",
"tjDqfHn89Ilg",
"ouhOikY1qEZG",
"2NcemUMdp92X",
"0882m2pipfVU",
"FEwTp1m6pA_-"
"Ov-m1nFU9jry"
],
"toc_visible": true,
"include_colab_link": true
@ -52,7 +47,7 @@
"colab": {}
},
"source": [
"command1 = \"\" #@param {type:\"string\"}\n",
"command1 = \"list projects\" #@param {type:\"string\"}\n",
"command2 = \"\" #@param {type:\"string\"}\n",
"command3 = \"\" #@param {type:\"string\"}\n",
"command4 = \"\" #@param {type:\"string\"}\n",
@ -81,7 +76,7 @@
"colab_type": "text"
},
"source": [
"## Safire"
"### Safire"
]
},
{
@ -153,13 +148,12 @@
"metadata": {
"id": "e2xK3xi2MlXs",
"colab_type": "code",
"cellView": "form",
"cellView": "both",
"colab": {}
},
"source": [
"import os, importlib, sys, uuid, pickle\n",
"import pandas as pd\n",
"\n",
"from base64 import b64decode\n",
"from glob import glob\n",
"from json import loads\n",
@ -170,7 +164,6 @@
"from googleapiclient.discovery import build\n",
"\n",
"if importlib.util.find_spec('fire') is None: os.system('pip install fire')\n",
" # %pip install fire\n",
"import fire\n",
"\n",
"#@markdown Root dir when using Colab GDrive mount(recommended to leave default):\n",
@ -184,9 +177,10 @@
" #@markdown Root directory when safire running locally (dropdown choices):\n",
" local_safire_dir = '/opt/safire/safire' #@param ['/opt/safire/safire', '~/safire', '~/../opt/safire/safire', \"safire\"] {allow-input: true}\n",
" safire_dir = local_safire_dir\n",
"\n",
"# print(f\"safire_dir = {os.path.abspath(safire_dir)} \\nDir content:\" )\n",
"# !ls \"$safire_dir\""
"#@markdown External config file located in safire_dir. Use any name. Def: config.py:\n",
"use_external_config = True #@param {type:\"boolean\"}\n",
"ext_conf_file = \"/config.py\" #@param [\"/config.py\"] {allow-input: true}\n",
"ext_conf_file = safire_dir + ext_conf_file"
],
"execution_count": null,
"outputs": []
@ -202,24 +196,6 @@
"- Tells safire where to read and write files, how many 'things' to create and what to call them"
]
},
{
"cell_type": "code",
"metadata": {
"id": "RXrq4xh89OUY",
"colab_type": "code",
"cellView": "form",
"colab": {}
},
"source": [
"#@markdown External config file located in safire dir. Use any name. Def: config.py:\n",
"# print(safire_dir)\n",
"use_external_config = False #@param {type:\"boolean\"}\n",
"ext_conf_file = \"/config.py\" #@param [\"/config.py\"] {allow-input: true}\n",
"ext_conf_file = safire_dir + ext_conf_file\n"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
@ -292,8 +268,8 @@
"spad = 6 #@param {type:\"integer\"}\n",
"jpad = 6 #@param {type:\"integer\"}\n",
"#@markdown retry and sleep_time help safire deal with slow google apis. Adjust as needed\n",
"retry = 20 #@param {type:\"integer\"}\n",
"sleep_time = 6 #@param {type:\"integer\"}\n",
"retry = 5#@param {type:\"integer\"}\n",
"sleep_time = 5#@param {type:\"integer\"}\n",
"\n",
"## #@markdown [API Service Names and Versions]\n",
"DRIVE = [\"drive\", \"v3\"]\n",
@ -870,13 +846,13 @@
" file_tag=\"\",\n",
" prt=False,\n",
" ):\n",
" \"\"\"Create N service accounts/SAs in projects which match 'filter'. Usage: 'safire add sas 5 xyz'\n",
" \"\"\"Create N service accounts/SAs in projects which match 'filter'. Usage: 'safire add sas xyz 5'\n",
" will add 5 SAs to all projects containing 'xys' if fewer than 100 exist. Will not overwrite SAs.\"\"\"\n",
" iam = self._svc(*IAM, token)\n",
" projId_list = self._list.projects(filter, file_tag, token, prt)\n",
" all_sas = []\n",
" for project in projId_list:\n",
" batch = BatchJob(iam)\n",
" # batch = BatchJob(iam)\n",
" sa_emails, _ = self._list.sas(project, True, file_tag, token, False)\n",
" start_sa_emails = sa_emails\n",
" start_sa_count = len(sa_emails)\n",
@ -888,6 +864,7 @@
" new_sas = []\n",
" retries = retry\n",
" while count and retries:\n",
" batch = BatchJob(iam)\n",
" for _ in range(count):\n",
" while [s for s in all_sas if str(next_sa_num) in s.split(\"@\")[0]]:\n",
" next_sa_num += 1\n",
@ -907,7 +884,7 @@
" sa_emails, _ = self._list.sas(project, True, file_tag, token, False)\n",
" curr_sa_count = len(sa_emails)\n",
" count = count - curr_sa_count + start_sa_count\n",
" sleep(sleep_time)\n",
" sleep(sleep_time/5)\n",
" new_sa_emails = [i for i in sa_emails if i not in start_sa_emails]\n",
" num_sas_created = len(new_sa_emails)\n",
" print(\n",
@ -1041,17 +1018,18 @@
" f\"{len(sa_emails)-len(add_sas)} SA emails from {project} are in {group}. {len(add_sas)} failed.\"\n",
" )\n",
"\n",
" def user(self, td_id, user, role=\"organizer\"):\n",
" \"\"\"Add user (typically group name) to a shared/team drive. Usage: 'safire add someTDid mygroup@domain.com'\"\"\"\n",
" drive = self._svc(*DRIVE, token)\n",
" body = {\"type\": \"user\", \"role\": role, \"emailAddress\": user}\n",
" return (\n",
" drive.permissions()\n",
" .create(body=body, fileId=td_id, supportsAllDrives=True, fields=\"id\")\n",
" .execute()\n",
" .get(\"id\")\n",
" )\n",
"\n"
" def user(self, user, *td_id, role=\"organizer\"):\n",
" \"\"\"Add user (typically group name) to shared/team drive(s). Usage: 'safire add mygroup@domain.com td_filter'\"\"\"\n",
" for td_filt in td_id:\n",
" drives = self._list.drives(td_filt, \"\", token, False)\n",
" for td in drives:\n",
" if td_filt in td['name']:\n",
" print(f\"Adding {user} to {td['name']} {td['id']}\")\n",
" drive = self._svc(*DRIVE, token)\n",
" body = {\"type\": \"user\", \"role\": role, \"emailAddress\": user}\n",
" drive.permissions().create(body=body, fileId=td['id'], supportsAllDrives=True, fields=\"id\").execute()\n",
" else:\n",
" pass\n"
],
"execution_count": null,
"outputs": []
@ -1109,6 +1087,7 @@
" name = f\"projects/{project}/serviceAccounts/{i}\"\n",
" batch.add(iam.projects().serviceAccounts().delete(name=name))\n",
" batch.execute()\n",
" sleep(sleep_time)\n",
" sas, _ = self._list.sas(project, True, file_tag, token, False)\n",
"\n",
" def jsons(self, filter=\"\", sa_path=sa_path):\n",
@ -1178,18 +1157,22 @@
" print(f\"Deleting {teamDriveId}\")\n",
" self.drive(teamDriveId.rstrip())\n",
"\n",
" def user(self, td_id, user, role=\"organizer\", token=token):\n",
" \"\"\"Remove user (typically group name) from a shared/team drive. Usage: 'safire remove someTDid mygroup@domain.com'\"\"\"\n",
" drvsvc = self._svc(*DRIVE, token)\n",
" return (\n",
" drvsvc.permissions()\n",
" .delete(\n",
" permissionId=user, fileId=td_id, supportsAllDrives=True, fields=\"id\"\n",
" )\n",
" .execute()\n",
" .get(\"id\")\n",
" )\n",
"\n"
" def user(self, user, *td_id, role=\"organizer\", token=token):\n",
" \"\"\"Remove user (typically group name) from shared/team drive(s). Usage: 'safire remove mygroup@domain.com td_filter'\"\"\"\n",
" drives = self._list.drives(\"\", \"\", token, False)\n",
" for td_filt in td_id:\n",
" for td in drives:\n",
" if td_filt in td['name']:\n",
" drvsvc = self._svc(*DRIVE, token)\n",
" user_info = []\n",
" resp = {'nextPageToken':None}\n",
" while 'nextPageToken' in resp:\n",
" resp = drvsvc.permissions().list(fileId=td['id'],pageSize=100,fields='nextPageToken,permissions(id,emailAddress,role)',supportsAllDrives=True,pageToken=resp['nextPageToken']).execute()\n",
" user_info += resp['permissions']\n",
" for i in user_info:\n",
" if user in i['emailAddress']:\n",
" print(f\"Removing {user} with id {i['id']} from {td['name']} {td['id']}\")\n",
" drvsvc.permissions().delete(permissionId=i['id'], fileId=td['id'], supportsAllDrives=True, fields=\"id\").execute()\n"
],
"execution_count": null,
"outputs": []
@ -1217,11 +1200,9 @@
" \"\"\"Rename json/key files to their email prefix, email numeric (omit prefix), uniqId or in a sequence.\n",
" Usage: 'safire rename jsons email' [choice email, email_seq, uniq, seq]\n",
" Renaming is repeatable. Can always delete and redownload keys if needed.\"\"\"\n",
"\n",
" def jsons(self, rename_type, dir=f\"{sa_path}/\"):\n",
" \"\"\"Usage: 'safire rename jsons email' [choice email, email_seq, uniq, seq]\"\"\"\n",
" import json, os\n",
"\n",
" filelist = os.listdir(dir)\n",
" print(\"\\nOriginal filenames:\", *sorted(filelist), sep=\"\\n\")\n",
" new_name = []\n",
@ -1286,6 +1267,9 @@
"colab": {}
},
"source": [
"%%time\n",
"# %%timeit\n",
"\n",
"#@markdown\n",
"if \"safire.py\" == sys.argv[0] and __name__ == \"__main__\":\n",
" fire.Fire(Commands)\n",
@ -1338,10 +1322,11 @@
"metadata": {
"id": "F2vRbufNwJg6",
"colab_type": "code",
"cellView": "both",
"cellView": "form",
"colab": {}
},
"source": [
"#@markdown\n",
"%whos"
],
"execution_count": null,