88lex/safire
1
1
Fork 0
mirror of https://github.com/88lex/safire.git synced 2024-09-20 23:06:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

Created using Colaboratory

Browse source
This commit is contained in:
88lex 2020-08-12 22:20:32 +08:00
parent 28b4cf9c01
commit b18b5e2213
1 changed files with 48 additions and 63 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

105
safire.ipynb
View file

@ -7,12 +7,7 @@
"provenance": [], "provenance": [],
"private_outputs": true, "private_outputs": true,
"collapsed_sections": [ "collapsed_sections": [
"1FV-VHZsLkhG", "Ov-m1nFU9jry"
"tjDqfHn89Ilg",
"ouhOikY1qEZG",
"2NcemUMdp92X",
"0882m2pipfVU",
"FEwTp1m6pA_-"
], ],
"toc_visible": true, "toc_visible": true,
"include_colab_link": true "include_colab_link": true
@ -52,7 +47,7 @@
"colab": {} "colab": {}
}, },
"source": [ "source": [
"command1 = \"\" #@param {type:\"string\"}\n", "command1 = \"list projects\" #@param {type:\"string\"}\n",
"command2 = \"\" #@param {type:\"string\"}\n", "command2 = \"\" #@param {type:\"string\"}\n",
"command3 = \"\" #@param {type:\"string\"}\n", "command3 = \"\" #@param {type:\"string\"}\n",
"command4 = \"\" #@param {type:\"string\"}\n", "command4 = \"\" #@param {type:\"string\"}\n",
@ -81,7 +76,7 @@
"colab_type": "text" "colab_type": "text"
}, },
"source": [ "source": [
"## Safire" "### Safire"
] ]
}, },
{ {
@ -153,13 +148,12 @@
"metadata": { "metadata": {
"id": "e2xK3xi2MlXs", "id": "e2xK3xi2MlXs",
"colab_type": "code", "colab_type": "code",
"cellView": "form", "cellView": "both",
"colab": {} "colab": {}
}, },
"source": [ "source": [
"import os, importlib, sys, uuid, pickle\n", "import os, importlib, sys, uuid, pickle\n",
"import pandas as pd\n", "import pandas as pd\n",
"\n",
"from base64 import b64decode\n", "from base64 import b64decode\n",
"from glob import glob\n", "from glob import glob\n",
"from json import loads\n", "from json import loads\n",
@ -170,7 +164,6 @@
"from googleapiclient.discovery import build\n", "from googleapiclient.discovery import build\n",
"\n", "\n",
"if importlib.util.find_spec('fire') is None: os.system('pip install fire')\n", "if importlib.util.find_spec('fire') is None: os.system('pip install fire')\n",
" # %pip install fire\n",
"import fire\n", "import fire\n",
"\n", "\n",
"#@markdown Root dir when using Colab GDrive mount(recommended to leave default):\n", "#@markdown Root dir when using Colab GDrive mount(recommended to leave default):\n",
@ -184,9 +177,10 @@
" #@markdown Root directory when safire running locally (dropdown choices):\n", " #@markdown Root directory when safire running locally (dropdown choices):\n",
" local_safire_dir = '/opt/safire/safire' #@param ['/opt/safire/safire', '~/safire', '~/../opt/safire/safire', \"safire\"] {allow-input: true}\n", " local_safire_dir = '/opt/safire/safire' #@param ['/opt/safire/safire', '~/safire', '~/../opt/safire/safire', \"safire\"] {allow-input: true}\n",
" safire_dir = local_safire_dir\n", " safire_dir = local_safire_dir\n",
"\n", "#@markdown External config file located in safire_dir. Use any name. Def: config.py:\n",
"# print(f\"safire_dir = {os.path.abspath(safire_dir)} \\nDir content:\" )\n", "use_external_config = True #@param {type:\"boolean\"}\n",
"# !ls \"$safire_dir\"" "ext_conf_file = \"/config.py\" #@param [\"/config.py\"] {allow-input: true}\n",
"ext_conf_file = safire_dir + ext_conf_file"
], ],
"execution_count": null, "execution_count": null,
"outputs": [] "outputs": []
@ -202,24 +196,6 @@
"- Tells safire where to read and write files, how many 'things' to create and what to call them" "- Tells safire where to read and write files, how many 'things' to create and what to call them"
] ]
}, },
{
"cell_type": "code",
"metadata": {
"id": "RXrq4xh89OUY",
"colab_type": "code",
"cellView": "form",
"colab": {}
},
"source": [
"#@markdown External config file located in safire dir. Use any name. Def: config.py:\n",
"# print(safire_dir)\n",
"use_external_config = False #@param {type:\"boolean\"}\n",
"ext_conf_file = \"/config.py\" #@param [\"/config.py\"] {allow-input: true}\n",
"ext_conf_file = safire_dir + ext_conf_file\n"
],
"execution_count": null,
"outputs": []
},
{ {
"cell_type": "code", "cell_type": "code",
"metadata": { "metadata": {
@ -292,8 +268,8 @@
"spad = 6 #@param {type:\"integer\"}\n", "spad = 6 #@param {type:\"integer\"}\n",
"jpad = 6 #@param {type:\"integer\"}\n", "jpad = 6 #@param {type:\"integer\"}\n",
"#@markdown retry and sleep_time help safire deal with slow google apis. Adjust as needed\n", "#@markdown retry and sleep_time help safire deal with slow google apis. Adjust as needed\n",
"retry = 20 #@param {type:\"integer\"}\n", "retry = 5#@param {type:\"integer\"}\n",
"sleep_time = 6 #@param {type:\"integer\"}\n", "sleep_time = 5#@param {type:\"integer\"}\n",
"\n", "\n",
"## #@markdown [API Service Names and Versions]\n", "## #@markdown [API Service Names and Versions]\n",
"DRIVE = [\"drive\", \"v3\"]\n", "DRIVE = [\"drive\", \"v3\"]\n",
@ -870,13 +846,13 @@
" file_tag=\"\",\n", " file_tag=\"\",\n",
" prt=False,\n", " prt=False,\n",
" ):\n", " ):\n",
" \"\"\"Create N service accounts/SAs in projects which match 'filter'. Usage: 'safire add sas 5 xyz'\n", " \"\"\"Create N service accounts/SAs in projects which match 'filter'. Usage: 'safire add sas xyz 5'\n",
" will add 5 SAs to all projects containing 'xys' if fewer than 100 exist. Will not overwrite SAs.\"\"\"\n", " will add 5 SAs to all projects containing 'xys' if fewer than 100 exist. Will not overwrite SAs.\"\"\"\n",
" iam = self._svc(*IAM, token)\n", " iam = self._svc(*IAM, token)\n",
" projId_list = self._list.projects(filter, file_tag, token, prt)\n", " projId_list = self._list.projects(filter, file_tag, token, prt)\n",
" all_sas = []\n", " all_sas = []\n",
" for project in projId_list:\n", " for project in projId_list:\n",
" batch = BatchJob(iam)\n", " # batch = BatchJob(iam)\n",
" sa_emails, _ = self._list.sas(project, True, file_tag, token, False)\n", " sa_emails, _ = self._list.sas(project, True, file_tag, token, False)\n",
" start_sa_emails = sa_emails\n", " start_sa_emails = sa_emails\n",
" start_sa_count = len(sa_emails)\n", " start_sa_count = len(sa_emails)\n",
@ -888,6 +864,7 @@
" new_sas = []\n", " new_sas = []\n",
" retries = retry\n", " retries = retry\n",
" while count and retries:\n", " while count and retries:\n",
" batch = BatchJob(iam)\n",
" for _ in range(count):\n", " for _ in range(count):\n",
" while [s for s in all_sas if str(next_sa_num) in s.split(\"@\")[0]]:\n", " while [s for s in all_sas if str(next_sa_num) in s.split(\"@\")[0]]:\n",
" next_sa_num += 1\n", " next_sa_num += 1\n",
@ -907,7 +884,7 @@
" sa_emails, _ = self._list.sas(project, True, file_tag, token, False)\n", " sa_emails, _ = self._list.sas(project, True, file_tag, token, False)\n",
" curr_sa_count = len(sa_emails)\n", " curr_sa_count = len(sa_emails)\n",
" count = count - curr_sa_count + start_sa_count\n", " count = count - curr_sa_count + start_sa_count\n",
" sleep(sleep_time)\n", " sleep(sleep_time/5)\n",
" new_sa_emails = [i for i in sa_emails if i not in start_sa_emails]\n", " new_sa_emails = [i for i in sa_emails if i not in start_sa_emails]\n",
" num_sas_created = len(new_sa_emails)\n", " num_sas_created = len(new_sa_emails)\n",
" print(\n", " print(\n",
@ -1041,17 +1018,18 @@
" f\"{len(sa_emails)-len(add_sas)} SA emails from {project} are in {group}. {len(add_sas)} failed.\"\n", " f\"{len(sa_emails)-len(add_sas)} SA emails from {project} are in {group}. {len(add_sas)} failed.\"\n",
" )\n", " )\n",
"\n", "\n",
" def user(self, td_id, user, role=\"organizer\"):\n", " def user(self, user, *td_id, role=\"organizer\"):\n",
" \"\"\"Add user (typically group name) to a shared/team drive. Usage: 'safire add someTDid mygroup@domain.com'\"\"\"\n", " \"\"\"Add user (typically group name) to shared/team drive(s). Usage: 'safire add mygroup@domain.com td_filter'\"\"\"\n",
" for td_filt in td_id:\n",
" drives = self._list.drives(td_filt, \"\", token, False)\n",
" for td in drives:\n",
" if td_filt in td['name']:\n",
" print(f\"Adding {user} to {td['name']} {td['id']}\")\n",
" drive = self._svc(*DRIVE, token)\n", " drive = self._svc(*DRIVE, token)\n",
" body = {\"type\": \"user\", \"role\": role, \"emailAddress\": user}\n", " body = {\"type\": \"user\", \"role\": role, \"emailAddress\": user}\n",
" return (\n", " drive.permissions().create(body=body, fileId=td['id'], supportsAllDrives=True, fields=\"id\").execute()\n",
" drive.permissions()\n", " else:\n",
" .create(body=body, fileId=td_id, supportsAllDrives=True, fields=\"id\")\n", " pass\n"
" .execute()\n",
" .get(\"id\")\n",
" )\n",
"\n"
], ],
"execution_count": null, "execution_count": null,
"outputs": [] "outputs": []
@ -1109,6 +1087,7 @@
" name = f\"projects/{project}/serviceAccounts/{i}\"\n", " name = f\"projects/{project}/serviceAccounts/{i}\"\n",
" batch.add(iam.projects().serviceAccounts().delete(name=name))\n", " batch.add(iam.projects().serviceAccounts().delete(name=name))\n",
" batch.execute()\n", " batch.execute()\n",
" sleep(sleep_time)\n",
" sas, _ = self._list.sas(project, True, file_tag, token, False)\n", " sas, _ = self._list.sas(project, True, file_tag, token, False)\n",
"\n", "\n",
" def jsons(self, filter=\"\", sa_path=sa_path):\n", " def jsons(self, filter=\"\", sa_path=sa_path):\n",
@ -1178,18 +1157,22 @@
" print(f\"Deleting {teamDriveId}\")\n", " print(f\"Deleting {teamDriveId}\")\n",
" self.drive(teamDriveId.rstrip())\n", " self.drive(teamDriveId.rstrip())\n",
"\n", "\n",
" def user(self, td_id, user, role=\"organizer\", token=token):\n", " def user(self, user, *td_id, role=\"organizer\", token=token):\n",
" \"\"\"Remove user (typically group name) from a shared/team drive. Usage: 'safire remove someTDid mygroup@domain.com'\"\"\"\n", " \"\"\"Remove user (typically group name) from shared/team drive(s). Usage: 'safire remove mygroup@domain.com td_filter'\"\"\"\n",
" drives = self._list.drives(\"\", \"\", token, False)\n",
" for td_filt in td_id:\n",
" for td in drives:\n",
" if td_filt in td['name']:\n",
" drvsvc = self._svc(*DRIVE, token)\n", " drvsvc = self._svc(*DRIVE, token)\n",
" return (\n", " user_info = []\n",
" drvsvc.permissions()\n", " resp = {'nextPageToken':None}\n",
" .delete(\n", " while 'nextPageToken' in resp:\n",
" permissionId=user, fileId=td_id, supportsAllDrives=True, fields=\"id\"\n", " resp = drvsvc.permissions().list(fileId=td['id'],pageSize=100,fields='nextPageToken,permissions(id,emailAddress,role)',supportsAllDrives=True,pageToken=resp['nextPageToken']).execute()\n",
" )\n", " user_info += resp['permissions']\n",
" .execute()\n", " for i in user_info:\n",
" .get(\"id\")\n", " if user in i['emailAddress']:\n",
" )\n", " print(f\"Removing {user} with id {i['id']} from {td['name']} {td['id']}\")\n",
"\n" " drvsvc.permissions().delete(permissionId=i['id'], fileId=td['id'], supportsAllDrives=True, fields=\"id\").execute()\n"
], ],
"execution_count": null, "execution_count": null,
"outputs": [] "outputs": []
@ -1217,11 +1200,9 @@
" \"\"\"Rename json/key files to their email prefix, email numeric (omit prefix), uniqId or in a sequence.\n", " \"\"\"Rename json/key files to their email prefix, email numeric (omit prefix), uniqId or in a sequence.\n",
" Usage: 'safire rename jsons email' [choice email, email_seq, uniq, seq]\n", " Usage: 'safire rename jsons email' [choice email, email_seq, uniq, seq]\n",
" Renaming is repeatable. Can always delete and redownload keys if needed.\"\"\"\n", " Renaming is repeatable. Can always delete and redownload keys if needed.\"\"\"\n",
"\n",
" def jsons(self, rename_type, dir=f\"{sa_path}/\"):\n", " def jsons(self, rename_type, dir=f\"{sa_path}/\"):\n",
" \"\"\"Usage: 'safire rename jsons email' [choice email, email_seq, uniq, seq]\"\"\"\n", " \"\"\"Usage: 'safire rename jsons email' [choice email, email_seq, uniq, seq]\"\"\"\n",
" import json, os\n", " import json, os\n",
"\n",
" filelist = os.listdir(dir)\n", " filelist = os.listdir(dir)\n",
" print(\"\\nOriginal filenames:\", *sorted(filelist), sep=\"\\n\")\n", " print(\"\\nOriginal filenames:\", *sorted(filelist), sep=\"\\n\")\n",
" new_name = []\n", " new_name = []\n",
@ -1286,6 +1267,9 @@
"colab": {} "colab": {}
}, },
"source": [ "source": [
"%%time\n",
"# %%timeit\n",
"\n",
"#@markdown\n", "#@markdown\n",
"if \"safire.py\" == sys.argv[0] and __name__ == \"__main__\":\n", "if \"safire.py\" == sys.argv[0] and __name__ == \"__main__\":\n",
" fire.Fire(Commands)\n", " fire.Fire(Commands)\n",
@ -1338,10 +1322,11 @@
"metadata": { "metadata": {
"id": "F2vRbufNwJg6", "id": "F2vRbufNwJg6",
"colab_type": "code", "colab_type": "code",
"cellView": "both", "cellView": "form",
"colab": {} "colab": {}
}, },
"source": [ "source": [
"#@markdown\n",
"%whos" "%whos"
], ],
"execution_count": null, "execution_count": null,