mirror of
https://github.com/88lex/safire.git
synced 2024-09-20 23:06:08 +08:00
Created using Colaboratory
This commit is contained in:
parent
28b4cf9c01
commit
b18b5e2213
111
safire.ipynb
111
safire.ipynb
|
@ -7,12 +7,7 @@
|
||||||
"provenance": [],
|
"provenance": [],
|
||||||
"private_outputs": true,
|
"private_outputs": true,
|
||||||
"collapsed_sections": [
|
"collapsed_sections": [
|
||||||
"1FV-VHZsLkhG",
|
"Ov-m1nFU9jry"
|
||||||
"tjDqfHn89Ilg",
|
|
||||||
"ouhOikY1qEZG",
|
|
||||||
"2NcemUMdp92X",
|
|
||||||
"0882m2pipfVU",
|
|
||||||
"FEwTp1m6pA_-"
|
|
||||||
],
|
],
|
||||||
"toc_visible": true,
|
"toc_visible": true,
|
||||||
"include_colab_link": true
|
"include_colab_link": true
|
||||||
|
@ -52,7 +47,7 @@
|
||||||
"colab": {}
|
"colab": {}
|
||||||
},
|
},
|
||||||
"source": [
|
"source": [
|
||||||
"command1 = \"\" #@param {type:\"string\"}\n",
|
"command1 = \"list projects\" #@param {type:\"string\"}\n",
|
||||||
"command2 = \"\" #@param {type:\"string\"}\n",
|
"command2 = \"\" #@param {type:\"string\"}\n",
|
||||||
"command3 = \"\" #@param {type:\"string\"}\n",
|
"command3 = \"\" #@param {type:\"string\"}\n",
|
||||||
"command4 = \"\" #@param {type:\"string\"}\n",
|
"command4 = \"\" #@param {type:\"string\"}\n",
|
||||||
|
@ -81,7 +76,7 @@
|
||||||
"colab_type": "text"
|
"colab_type": "text"
|
||||||
},
|
},
|
||||||
"source": [
|
"source": [
|
||||||
"## Safire"
|
"### Safire"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
@ -153,13 +148,12 @@
|
||||||
"metadata": {
|
"metadata": {
|
||||||
"id": "e2xK3xi2MlXs",
|
"id": "e2xK3xi2MlXs",
|
||||||
"colab_type": "code",
|
"colab_type": "code",
|
||||||
"cellView": "form",
|
"cellView": "both",
|
||||||
"colab": {}
|
"colab": {}
|
||||||
},
|
},
|
||||||
"source": [
|
"source": [
|
||||||
"import os, importlib, sys, uuid, pickle\n",
|
"import os, importlib, sys, uuid, pickle\n",
|
||||||
"import pandas as pd\n",
|
"import pandas as pd\n",
|
||||||
"\n",
|
|
||||||
"from base64 import b64decode\n",
|
"from base64 import b64decode\n",
|
||||||
"from glob import glob\n",
|
"from glob import glob\n",
|
||||||
"from json import loads\n",
|
"from json import loads\n",
|
||||||
|
@ -170,7 +164,6 @@
|
||||||
"from googleapiclient.discovery import build\n",
|
"from googleapiclient.discovery import build\n",
|
||||||
"\n",
|
"\n",
|
||||||
"if importlib.util.find_spec('fire') is None: os.system('pip install fire')\n",
|
"if importlib.util.find_spec('fire') is None: os.system('pip install fire')\n",
|
||||||
" # %pip install fire\n",
|
|
||||||
"import fire\n",
|
"import fire\n",
|
||||||
"\n",
|
"\n",
|
||||||
"#@markdown Root dir when using Colab GDrive mount(recommended to leave default):\n",
|
"#@markdown Root dir when using Colab GDrive mount(recommended to leave default):\n",
|
||||||
|
@ -184,9 +177,10 @@
|
||||||
" #@markdown Root directory when safire running locally (dropdown choices):\n",
|
" #@markdown Root directory when safire running locally (dropdown choices):\n",
|
||||||
" local_safire_dir = '/opt/safire/safire' #@param ['/opt/safire/safire', '~/safire', '~/../opt/safire/safire', \"safire\"] {allow-input: true}\n",
|
" local_safire_dir = '/opt/safire/safire' #@param ['/opt/safire/safire', '~/safire', '~/../opt/safire/safire', \"safire\"] {allow-input: true}\n",
|
||||||
" safire_dir = local_safire_dir\n",
|
" safire_dir = local_safire_dir\n",
|
||||||
"\n",
|
"#@markdown External config file located in safire_dir. Use any name. Def: config.py:\n",
|
||||||
"# print(f\"safire_dir = {os.path.abspath(safire_dir)} \\nDir content:\" )\n",
|
"use_external_config = True #@param {type:\"boolean\"}\n",
|
||||||
"# !ls \"$safire_dir\""
|
"ext_conf_file = \"/config.py\" #@param [\"/config.py\"] {allow-input: true}\n",
|
||||||
|
"ext_conf_file = safire_dir + ext_conf_file"
|
||||||
],
|
],
|
||||||
"execution_count": null,
|
"execution_count": null,
|
||||||
"outputs": []
|
"outputs": []
|
||||||
|
@ -202,24 +196,6 @@
|
||||||
"- Tells safire where to read and write files, how many 'things' to create and what to call them"
|
"- Tells safire where to read and write files, how many 'things' to create and what to call them"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"metadata": {
|
|
||||||
"id": "RXrq4xh89OUY",
|
|
||||||
"colab_type": "code",
|
|
||||||
"cellView": "form",
|
|
||||||
"colab": {}
|
|
||||||
},
|
|
||||||
"source": [
|
|
||||||
"#@markdown External config file located in safire dir. Use any name. Def: config.py:\n",
|
|
||||||
"# print(safire_dir)\n",
|
|
||||||
"use_external_config = False #@param {type:\"boolean\"}\n",
|
|
||||||
"ext_conf_file = \"/config.py\" #@param [\"/config.py\"] {allow-input: true}\n",
|
|
||||||
"ext_conf_file = safire_dir + ext_conf_file\n"
|
|
||||||
],
|
|
||||||
"execution_count": null,
|
|
||||||
"outputs": []
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"cell_type": "code",
|
"cell_type": "code",
|
||||||
"metadata": {
|
"metadata": {
|
||||||
|
@ -292,8 +268,8 @@
|
||||||
"spad = 6 #@param {type:\"integer\"}\n",
|
"spad = 6 #@param {type:\"integer\"}\n",
|
||||||
"jpad = 6 #@param {type:\"integer\"}\n",
|
"jpad = 6 #@param {type:\"integer\"}\n",
|
||||||
"#@markdown retry and sleep_time help safire deal with slow google apis. Adjust as needed\n",
|
"#@markdown retry and sleep_time help safire deal with slow google apis. Adjust as needed\n",
|
||||||
"retry = 20 #@param {type:\"integer\"}\n",
|
"retry = 5#@param {type:\"integer\"}\n",
|
||||||
"sleep_time = 6 #@param {type:\"integer\"}\n",
|
"sleep_time = 5#@param {type:\"integer\"}\n",
|
||||||
"\n",
|
"\n",
|
||||||
"## #@markdown [API Service Names and Versions]\n",
|
"## #@markdown [API Service Names and Versions]\n",
|
||||||
"DRIVE = [\"drive\", \"v3\"]\n",
|
"DRIVE = [\"drive\", \"v3\"]\n",
|
||||||
|
@ -870,13 +846,13 @@
|
||||||
" file_tag=\"\",\n",
|
" file_tag=\"\",\n",
|
||||||
" prt=False,\n",
|
" prt=False,\n",
|
||||||
" ):\n",
|
" ):\n",
|
||||||
" \"\"\"Create N service accounts/SAs in projects which match 'filter'. Usage: 'safire add sas 5 xyz'\n",
|
" \"\"\"Create N service accounts/SAs in projects which match 'filter'. Usage: 'safire add sas xyz 5'\n",
|
||||||
" will add 5 SAs to all projects containing 'xys' if fewer than 100 exist. Will not overwrite SAs.\"\"\"\n",
|
" will add 5 SAs to all projects containing 'xys' if fewer than 100 exist. Will not overwrite SAs.\"\"\"\n",
|
||||||
" iam = self._svc(*IAM, token)\n",
|
" iam = self._svc(*IAM, token)\n",
|
||||||
" projId_list = self._list.projects(filter, file_tag, token, prt)\n",
|
" projId_list = self._list.projects(filter, file_tag, token, prt)\n",
|
||||||
" all_sas = []\n",
|
" all_sas = []\n",
|
||||||
" for project in projId_list:\n",
|
" for project in projId_list:\n",
|
||||||
" batch = BatchJob(iam)\n",
|
" # batch = BatchJob(iam)\n",
|
||||||
" sa_emails, _ = self._list.sas(project, True, file_tag, token, False)\n",
|
" sa_emails, _ = self._list.sas(project, True, file_tag, token, False)\n",
|
||||||
" start_sa_emails = sa_emails\n",
|
" start_sa_emails = sa_emails\n",
|
||||||
" start_sa_count = len(sa_emails)\n",
|
" start_sa_count = len(sa_emails)\n",
|
||||||
|
@ -888,6 +864,7 @@
|
||||||
" new_sas = []\n",
|
" new_sas = []\n",
|
||||||
" retries = retry\n",
|
" retries = retry\n",
|
||||||
" while count and retries:\n",
|
" while count and retries:\n",
|
||||||
|
" batch = BatchJob(iam)\n",
|
||||||
" for _ in range(count):\n",
|
" for _ in range(count):\n",
|
||||||
" while [s for s in all_sas if str(next_sa_num) in s.split(\"@\")[0]]:\n",
|
" while [s for s in all_sas if str(next_sa_num) in s.split(\"@\")[0]]:\n",
|
||||||
" next_sa_num += 1\n",
|
" next_sa_num += 1\n",
|
||||||
|
@ -907,7 +884,7 @@
|
||||||
" sa_emails, _ = self._list.sas(project, True, file_tag, token, False)\n",
|
" sa_emails, _ = self._list.sas(project, True, file_tag, token, False)\n",
|
||||||
" curr_sa_count = len(sa_emails)\n",
|
" curr_sa_count = len(sa_emails)\n",
|
||||||
" count = count - curr_sa_count + start_sa_count\n",
|
" count = count - curr_sa_count + start_sa_count\n",
|
||||||
" sleep(sleep_time)\n",
|
" sleep(sleep_time/5)\n",
|
||||||
" new_sa_emails = [i for i in sa_emails if i not in start_sa_emails]\n",
|
" new_sa_emails = [i for i in sa_emails if i not in start_sa_emails]\n",
|
||||||
" num_sas_created = len(new_sa_emails)\n",
|
" num_sas_created = len(new_sa_emails)\n",
|
||||||
" print(\n",
|
" print(\n",
|
||||||
|
@ -1041,17 +1018,18 @@
|
||||||
" f\"{len(sa_emails)-len(add_sas)} SA emails from {project} are in {group}. {len(add_sas)} failed.\"\n",
|
" f\"{len(sa_emails)-len(add_sas)} SA emails from {project} are in {group}. {len(add_sas)} failed.\"\n",
|
||||||
" )\n",
|
" )\n",
|
||||||
"\n",
|
"\n",
|
||||||
" def user(self, td_id, user, role=\"organizer\"):\n",
|
" def user(self, user, *td_id, role=\"organizer\"):\n",
|
||||||
" \"\"\"Add user (typically group name) to a shared/team drive. Usage: 'safire add someTDid mygroup@domain.com'\"\"\"\n",
|
" \"\"\"Add user (typically group name) to shared/team drive(s). Usage: 'safire add mygroup@domain.com td_filter'\"\"\"\n",
|
||||||
" drive = self._svc(*DRIVE, token)\n",
|
" for td_filt in td_id:\n",
|
||||||
" body = {\"type\": \"user\", \"role\": role, \"emailAddress\": user}\n",
|
" drives = self._list.drives(td_filt, \"\", token, False)\n",
|
||||||
" return (\n",
|
" for td in drives:\n",
|
||||||
" drive.permissions()\n",
|
" if td_filt in td['name']:\n",
|
||||||
" .create(body=body, fileId=td_id, supportsAllDrives=True, fields=\"id\")\n",
|
" print(f\"Adding {user} to {td['name']} {td['id']}\")\n",
|
||||||
" .execute()\n",
|
" drive = self._svc(*DRIVE, token)\n",
|
||||||
" .get(\"id\")\n",
|
" body = {\"type\": \"user\", \"role\": role, \"emailAddress\": user}\n",
|
||||||
" )\n",
|
" drive.permissions().create(body=body, fileId=td['id'], supportsAllDrives=True, fields=\"id\").execute()\n",
|
||||||
"\n"
|
" else:\n",
|
||||||
|
" pass\n"
|
||||||
],
|
],
|
||||||
"execution_count": null,
|
"execution_count": null,
|
||||||
"outputs": []
|
"outputs": []
|
||||||
|
@ -1109,6 +1087,7 @@
|
||||||
" name = f\"projects/{project}/serviceAccounts/{i}\"\n",
|
" name = f\"projects/{project}/serviceAccounts/{i}\"\n",
|
||||||
" batch.add(iam.projects().serviceAccounts().delete(name=name))\n",
|
" batch.add(iam.projects().serviceAccounts().delete(name=name))\n",
|
||||||
" batch.execute()\n",
|
" batch.execute()\n",
|
||||||
|
" sleep(sleep_time)\n",
|
||||||
" sas, _ = self._list.sas(project, True, file_tag, token, False)\n",
|
" sas, _ = self._list.sas(project, True, file_tag, token, False)\n",
|
||||||
"\n",
|
"\n",
|
||||||
" def jsons(self, filter=\"\", sa_path=sa_path):\n",
|
" def jsons(self, filter=\"\", sa_path=sa_path):\n",
|
||||||
|
@ -1178,18 +1157,22 @@
|
||||||
" print(f\"Deleting {teamDriveId}\")\n",
|
" print(f\"Deleting {teamDriveId}\")\n",
|
||||||
" self.drive(teamDriveId.rstrip())\n",
|
" self.drive(teamDriveId.rstrip())\n",
|
||||||
"\n",
|
"\n",
|
||||||
" def user(self, td_id, user, role=\"organizer\", token=token):\n",
|
" def user(self, user, *td_id, role=\"organizer\", token=token):\n",
|
||||||
" \"\"\"Remove user (typically group name) from a shared/team drive. Usage: 'safire remove someTDid mygroup@domain.com'\"\"\"\n",
|
" \"\"\"Remove user (typically group name) from shared/team drive(s). Usage: 'safire remove mygroup@domain.com td_filter'\"\"\"\n",
|
||||||
" drvsvc = self._svc(*DRIVE, token)\n",
|
" drives = self._list.drives(\"\", \"\", token, False)\n",
|
||||||
" return (\n",
|
" for td_filt in td_id:\n",
|
||||||
" drvsvc.permissions()\n",
|
" for td in drives:\n",
|
||||||
" .delete(\n",
|
" if td_filt in td['name']:\n",
|
||||||
" permissionId=user, fileId=td_id, supportsAllDrives=True, fields=\"id\"\n",
|
" drvsvc = self._svc(*DRIVE, token)\n",
|
||||||
" )\n",
|
" user_info = []\n",
|
||||||
" .execute()\n",
|
" resp = {'nextPageToken':None}\n",
|
||||||
" .get(\"id\")\n",
|
" while 'nextPageToken' in resp:\n",
|
||||||
" )\n",
|
" resp = drvsvc.permissions().list(fileId=td['id'],pageSize=100,fields='nextPageToken,permissions(id,emailAddress,role)',supportsAllDrives=True,pageToken=resp['nextPageToken']).execute()\n",
|
||||||
"\n"
|
" user_info += resp['permissions']\n",
|
||||||
|
" for i in user_info:\n",
|
||||||
|
" if user in i['emailAddress']:\n",
|
||||||
|
" print(f\"Removing {user} with id {i['id']} from {td['name']} {td['id']}\")\n",
|
||||||
|
" drvsvc.permissions().delete(permissionId=i['id'], fileId=td['id'], supportsAllDrives=True, fields=\"id\").execute()\n"
|
||||||
],
|
],
|
||||||
"execution_count": null,
|
"execution_count": null,
|
||||||
"outputs": []
|
"outputs": []
|
||||||
|
@ -1217,11 +1200,9 @@
|
||||||
" \"\"\"Rename json/key files to their email prefix, email numeric (omit prefix), uniqId or in a sequence.\n",
|
" \"\"\"Rename json/key files to their email prefix, email numeric (omit prefix), uniqId or in a sequence.\n",
|
||||||
" Usage: 'safire rename jsons email' [choice email, email_seq, uniq, seq]\n",
|
" Usage: 'safire rename jsons email' [choice email, email_seq, uniq, seq]\n",
|
||||||
" Renaming is repeatable. Can always delete and redownload keys if needed.\"\"\"\n",
|
" Renaming is repeatable. Can always delete and redownload keys if needed.\"\"\"\n",
|
||||||
"\n",
|
|
||||||
" def jsons(self, rename_type, dir=f\"{sa_path}/\"):\n",
|
" def jsons(self, rename_type, dir=f\"{sa_path}/\"):\n",
|
||||||
" \"\"\"Usage: 'safire rename jsons email' [choice email, email_seq, uniq, seq]\"\"\"\n",
|
" \"\"\"Usage: 'safire rename jsons email' [choice email, email_seq, uniq, seq]\"\"\"\n",
|
||||||
" import json, os\n",
|
" import json, os\n",
|
||||||
"\n",
|
|
||||||
" filelist = os.listdir(dir)\n",
|
" filelist = os.listdir(dir)\n",
|
||||||
" print(\"\\nOriginal filenames:\", *sorted(filelist), sep=\"\\n\")\n",
|
" print(\"\\nOriginal filenames:\", *sorted(filelist), sep=\"\\n\")\n",
|
||||||
" new_name = []\n",
|
" new_name = []\n",
|
||||||
|
@ -1286,6 +1267,9 @@
|
||||||
"colab": {}
|
"colab": {}
|
||||||
},
|
},
|
||||||
"source": [
|
"source": [
|
||||||
|
"%%time\n",
|
||||||
|
"# %%timeit\n",
|
||||||
|
"\n",
|
||||||
"#@markdown\n",
|
"#@markdown\n",
|
||||||
"if \"safire.py\" == sys.argv[0] and __name__ == \"__main__\":\n",
|
"if \"safire.py\" == sys.argv[0] and __name__ == \"__main__\":\n",
|
||||||
" fire.Fire(Commands)\n",
|
" fire.Fire(Commands)\n",
|
||||||
|
@ -1338,10 +1322,11 @@
|
||||||
"metadata": {
|
"metadata": {
|
||||||
"id": "F2vRbufNwJg6",
|
"id": "F2vRbufNwJg6",
|
||||||
"colab_type": "code",
|
"colab_type": "code",
|
||||||
"cellView": "both",
|
"cellView": "form",
|
||||||
"colab": {}
|
"colab": {}
|
||||||
},
|
},
|
||||||
"source": [
|
"source": [
|
||||||
|
"#@markdown\n",
|
||||||
"%whos"
|
"%whos"
|
||||||
],
|
],
|
||||||
"execution_count": null,
|
"execution_count": null,
|
||||||
|
|
Loading…
Reference in a new issue