From dcf2293ec0de046fd20e3545c1c65747986578ce Mon Sep 17 00:00:00 2001 From: Orsiris de Jong Date: Wed, 1 Feb 2023 01:28:42 +0100 Subject: [PATCH] Add inline copy of ofunctions.requestor --- npbackup/upgrade_client/requestor.py | 434 +++++++++++++++++++++++++++ 1 file changed, 434 insertions(+) create mode 100644 npbackup/upgrade_client/requestor.py diff --git a/npbackup/upgrade_client/requestor.py b/npbackup/upgrade_client/requestor.py new file mode 100644 index 0000000..f6b4f4d --- /dev/null +++ b/npbackup/upgrade_client/requestor.py @@ -0,0 +1,434 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# +# This file is part of npbackup + +__intname__ = "ofunctions.requestor" +__author__ = "Orsiris de Jong" +__copyright__ = "Copyright (C) 2014-2023 NetInvent" +__license__ = "BSD-3-Clause" +__build__ = "2022072201" + + +from typing import List, Optional, Any, Union +from logging import getLogger +import json +import requests + + +logger = getLogger(__intname__) + + +class Requestor: + """ + A class that handles JSON APIs elegantly, with server fallback, ACLS and error control + + data_model: high level json api function + requestor: standard requests + get_raw: low level binary download + """ + + def __init__( + self, + servers: List[str], + username: str = None, + password: str = None, + cert_verify: bool = True, + ): + self.api_session = None + self.username = username + self.password = password + self.cert_verify = cert_verify + self._endpoint = None + + self._app_name = "ofunctions-requestor-app" + self._user_agent = "ofunctions-requestor-ua" + + self._action_list = ["create", "read", "update", "delete", "exists"] + self._allowed_models = [] + self._acls = {} + """ + ACLS would look like a dict giving each allowed action for each model, eg: + { + 'users': 'create', 'read', 'update', 'delete', 'exists', + 'items': 'read', 'exists' + } + """ + self._action_requests_equiv = { + "create": "post", + "read": "get", + "update": "put", + "delete": "delete", + "exists": "get", + } + + # Headers need to be set to Accept: application/json or else Laravel will send HTML as return + # Do not set Content-Type or else uploads will fail + self._headers = { + "Accept": "application/json", + "Accept-Encoding": "deflate, gzip", + "User-Agent": self._user_agent, + "Referer": self._app_name, + } + + self._proxy_dict = {} + + self.servers = [] + # servers can be multiple servers for failover + if isinstance(servers, list): + self.servers = servers + elif servers is not None: + for server in servers.split(","): + # Remove trailing & ending spaces + # Make sure server ends with '/' + self.servers.append(server.strip().rstrip("/") + "/") + self.connected_server = None + + @property + def app_name(self): + return self._app_name + + @app_name.setter + def app_name(self, value: str): + if isinstance(value, str): + self._app_name = value.strip() + else: + raise ValueError("Bogus app name") + + @property + def user_agent(self): + return self._user_agent + + @user_agent.setter + def user_agent(self, value: str): + if isinstance(value, str): + self._user_agent = value.strip() + else: + raise ValueError("Bogus user agent") + + @property + def headers(self): + return self._headers + + @property + def header(self): + pass + + @header.setter + def header(self, value: dict): + if isinstance(value, dict): + self._headers = {**self._headers, **value} + if self.api_session: + self.api_session.headers = self._headers + else: + raise ValueError("Bogus header given") + + @headers.setter + def headers(self, value: dict): + if isinstance(value, dict): + self._headers = value + if self.api_session: + self.api_session.headers = self._headers + else: + raise ValueError("Bogus header given") + + @property + def endpoint(self): + return self._endpoint + + @endpoint.setter + def endpoint(self, value: str): + if isinstance(value, str): + self._endpoint = "/" + value.strip().lstrip("/") + + @property + def allowed_models(self): + return self._allowed_models + + @allowed_models.setter + def allowed_models(self, value: List[str]): + if isinstance(value, list): + self._allowed_models = value + else: + raise ValueError("Bogus allowed models given") + + @property + def acls(self): + return self._acls + + @acls.setter + def acls(self, value: dict): + if isinstance(value, list): + for key, val in value: + if key not in self.allowed_models: + raise ValueError("ACL for non existant model given") + if val not in self._action_list: + raise ValueError("Not a valid action") + + self._acls = value + else: + raise ValueError("Bogus acls given") + + @property + def action_list(self) -> List[str]: + return self._action_list + + @property + def proxy(self) -> dict: + return self._proxy_dict + + @proxy.setter + def proxy(self, value: str): + if isinstance(value, str) and ( + value.startswith("http://") or value.startswith("https://") + ): + if value.startswith("http"): + self._proxy_dict = {"http": value.strip("http://")} + elif value.startswith("https"): + self._proxy_dict = {"https": value.strip("https://")} + else: + raise ValueError("Bogus proxy given") + + def _create_session(self, uri: str = "", authenticated: bool = False) -> bool: + """Needed for api calls that don't require api authentication""" + try: + api_session = requests.Session() + # Remove accept: application/json from headers since it will fail on non API calls + headers = self.headers + headers["Accept"] = "" + if authenticated: + api_session.auth = (self.username, self.password) + result = api_session.get(uri, verify=self.cert_verify, headers=headers) + try: + status_code = result.status_code + except AttributeError: + logger.error("Server did not return a status code") + status_code = None + + try: + text = result.text + except AttributeError: + logger.error("Server did not return any data") + text = None + if status_code == 200: + self.api_session = api_session + if authenticated: + # This has to be changed depending on the API + if text.lower().startswith("token"): + token = text.replace("Token", "").strip() + self.header = {"Authorization": "Bearer {}".format(token)} + return True + + logger.error("Cannot establish a session to server.") + logger.warning("Server return code: {}".format(status_code)) + try: + logger.debug( + "Error:\n{}".format(text.encode("utf-8", errors="backslashreplace")) + ) + except: + pass + except requests.exceptions.SSLError: + logger.error( + "Cannot establish a session: SSL/TLS error. Are your server & client certificates valid ?" + ) + logger.info("Trace:", exc_info=True) + except requests.exceptions.ConnectionError: + logger.error( + "Cannot establish a session. Looks like we cannot reach the server." + ) + logger.info("Trace:", exc_info=True) + except Exception as exc: # pylint: disable=W0703,broad-except + logger.error("Cannot establish a session, unknown reason: %s", exc) + logger.info("Trace:", exc_info=True) + return False + + def create_session(self, endpoint: str = None, authenticated: bool = False) -> bool: + """ + Tries every server in server list until one can be reached, and sets server_api + :return: + """ + auth_endpoint = None + for server in self.servers: + if endpoint: + auth_endpoint = server + endpoint.strip("/") + elif self._endpoint: + auth_endpoint = server + self._endpoint.strip("/") + else: + auth_endpoint = server + if ( + not self._create_session(auth_endpoint, authenticated) + and len(self.servers) > 1 + ): + logger.info("Contacting auth server failed. Trying fallback server.") + else: + self.connected_server = server + return True + return False + + def _base_requestor( + self, endpoint: str = None, action: str = "read", data: Any = None + ) -> requests.Request: + """ + simple request function that does handle all exceptions and will return a requeusts.Request object or False + """ + if not self.api_session: + logger.error("Cannot operate without proper session.") + return None + + if not self.connected_server: + logger.error( + "Currently not connected to any server. Do we have an open session ?" + ) + return False + + if endpoint: + url = self.connected_server + endpoint.strip("/") + else: + url = self.connected_server + try: + if action in ["update", "create"]: + result = getattr(self.api_session, self._action_requests_equiv[action])( + url, + headers=self.headers, + data=data, + proxies=self._proxy_dict, + verify=self.cert_verify, + ) + else: + result = getattr(self.api_session, self._action_requests_equiv[action])( + url, + headers=self.headers, + proxies=self._proxy_dict, + verify=self.cert_verify, + ) + status_code = result.status_code + if status_code in [200, 201, 202]: + if status_code == 200: + logger.debug(f"Succesful operation {action}:{endpoint}") + elif status_code == 201: + logger.debug(f"Created operation {action}:{endpoint}") + elif status_code == 202: + logger.debug(f"Accepted operation {action}:{endpoint}") + if action == "exists": + return True + return result + else: + if (status_code in [400, 404]) and action == "exists": + logger.debug(f"Exists operation{action}:{endpoint}: No.") + elif status_code == 401: + logger.error(f"Server denied operation for {action}:{endpoint}") + elif status_code == 404: + logger.debug(f"Server did not find {action}:{endpoint}") + else: + logger.error(f"Failed operation {action}:{endpoint}") + logger.error(f"Server return code: {status_code}.") + try: + logger.error( + f'Error:\n{result.text.encode("utf-8", errors="backslashreplace")}' + ) + except Exception: # pylint: disable=W0703,broad-except + logger.error("No other info given by server.") + return False + except requests.exceptions.SSLError: + logger.error( + "Cannot establish a session: SSL/TLS error. Are your server & client certificates valid ?" + ) + logger.error("Trace:", exc_info=True) + except requests.exceptions.ConnectionError: + logger.error( + "Cannot establish a session. Looks like we cannot reach the server." + ) + logger.error("Trace:", exc_info=True) + except Exception as exc: # pylint: disable=W0703,broad-except + logger.error("Cannot establish a session, unknown reason: %s", exc) + logger.error("Trace:", exc_info=True) + return False + + def requestor( + self, + endpoint: str = None, + action: str = "read", + data: Any = None, + json_output=False, + raw=False, + ) -> Union[dict, bytes, bool, str]: + """ + simple request function that does handle all exceptions and will return content or False + """ + if action not in self.action_list: + logger.error("Unknown action %s", action) + + result = self._base_requestor(endpoint, action, data) + if not result: + return False + + if json_output: + try: + return json.loads(result.text) + except json.JSONDecodeError as exc: + logger.error("Cannot decode json output: {}".format(exc)) + return None + if raw: + return result.content + return result.text + + def get_raw(self, endpoint: str) -> Union[bytes, bool]: + """ + Shorthand essentially used to download binary files + """ + result = self._base_requestor(endpoint, action="read") + if result: + return result.content + return result + + def data_model( + self, + model: str = None, + id_record: Optional[Union[int, str]] = None, + action: str = "read", + data: Any = None, + json_output: bool = True, + ) -> Optional[Union[bool, str, dict]]: + """ + CRUD(E) model handler for APIs + + Example: update users in /users + payload = { + 'username': 'Hello', + 'password': 'somepass' + } + res = requestor.handle_data_model(model='users', action='update', id=34, data=payload) + This will call a PUT /users/34 + """ + if self.allowed_models and model not in self.allowed_models: + logger.error("Model %s is not allowed", model) + + if action not in self.action_list: + logger.error("Unknown action %s", action) + + if self.allowed_models and self.acls: + if action not in self.acls[model]: + logger.error("ACLS don't allow action %s for model %s", action, model) + + # Sanitize model and id_record + if model: + model = model.strip().strip("/") + + if action == "create": + id_record = None + if id_record: + id_record = id_record.strip().strip("/") + + if isinstance(id_record, str) and id_record.startswith("#"): + raise ValueError( + "id may not start with [#] sign since it is reserved for pagination." + ) + + if action != "create" and model and id_record: + # Action is read, exists, update or delete + model_endpoint = "{}/{}".format(model, id_record) + else: + model_endpoint = model + + result = self.requestor(model_endpoint, action, data, json_output) + return result