From cd8338fbd07d746fc7b3f1c36122c613e26985d5 Mon Sep 17 00:00:00 2001 From: Adam Toy Date: Sat, 6 Nov 2021 16:52:23 -0400 Subject: [PATCH] Add capability for advanced sync of other elements (#4) * Add capability for advanced sync of other elements * Revert default refresh interval * Add fix for secondary_blocked_services * Add logging for modified entry * Update language * Bug fix --- .gitignore | 6 +- Dockerfile | 6 +- README.md | 4 + VERSION | 2 +- src/app.py | 105 ++++----------- src/block_allow_lists/__init__.py | 204 ++++++++++++++++++++++++++++++ src/blocked_services/__init__.py | 69 ++++++++++ src/custom_rules/__init__.py | 63 +++++++++ src/entries/__init__.py | 87 +++++++++++++ src/exceptions.py | 2 + 10 files changed, 462 insertions(+), 86 deletions(-) create mode 100644 src/block_allow_lists/__init__.py create mode 100644 src/blocked_services/__init__.py create mode 100644 src/custom_rules/__init__.py create mode 100644 src/entries/__init__.py create mode 100644 src/exceptions.py diff --git a/.gitignore b/.gitignore index 4296422..d1fcaea 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,8 @@ venv/ *.pyc .idea/ -*.iml \ No newline at end of file +*.iml +.venv/ +.vscode/ +.env +__pycache__ diff --git a/Dockerfile b/Dockerfile index 7ad8275..f0067d3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,8 +12,10 @@ COPY requirements.txt /tmp/requirements.txt RUN pip3 install -r /tmp/requirements.txt && \ rm -f /tmp/requirements.txt -COPY src/app.py /opt/app.py +COPY src /opt/app + +WORKDIR /opt/app ENTRYPOINT ["python3"] -CMD ["/opt/app.py"] +CMD ["app.py"] diff --git a/README.md b/README.md index 9b4fd79..32b949b 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,10 @@ AdGuard Sync is packaged as a Docker image and can be ran anywhere with access t | SECONDARY_ADGUARD_USER | No | Username to log into your secondary AdGuard instance. Only necessary if credentials are different between primary and secondary | Value of 'ADGUARD_USER' | | SECONDARY_ADGUARD_PASS | No | Password to log into your secondary AdGuard instance. Only necessary if credentials are different between primary and secondary | Value of 'ADGUARD_PASS' | | REFRESH_INTERVAL_SECS | No | Frequency in seconds to refresh entries. | 60 | +| SYNC_ENTRIES | No | If 'true', will sync rewrite entries. | true | +| SYNC_BLOCKED_SERVICES | No | If 'true', will sync blocked services. | true | +| SYNC_BLOCK_ALLOW_LISTS | No | If 'true', will sync block/allow lists. | true | +| SYNC_CUSTOM_RULES | No | If 'true', will sync custom rules. | true | Once you've updated the file and ensure you have `docker` and `docker-compose` installed, run the following in the root directory: diff --git a/VERSION b/VERSION index b123147..415b19f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1 \ No newline at end of file +2.0 \ No newline at end of file diff --git a/src/app.py b/src/app.py index 94d58ed..942673c 100644 --- a/src/app.py +++ b/src/app.py @@ -2,6 +2,11 @@ import requests import os import json import time +import entries +import blocked_services +import block_allow_lists +import custom_rules +from exceptions import UnauthenticatedError ADGUARD_PRIMARY = os.environ['ADGUARD_PRIMARY'] ADGUARD_SECONDARY = os.environ['ADGUARD_SECONDARY'] @@ -13,13 +18,15 @@ ADGUARD_PASS = os.environ['ADGUARD_PASS'] SECONDARY_ADGUARD_USER = os.environ.get('SECONDARY_ADGUARD_USER', ADGUARD_USER) SECONDARY_ADGUARD_PASS = os.environ.get('SECONDARY_ADGUARD_PASS', ADGUARD_PASS) +# By default, sync all +SYNC_ENTRIES = os.environ.get('SYNC_ENTRIES', 'true').lower() == 'true' +SYNC_BLOCKED_SERVICES = os.environ.get('SYNC_BLOCKED_SERVICES', 'true').lower() == 'true' +SYNC_BLOCK_ALLOW_LISTS = os.environ.get('SYNC_BLOCK_ALLOW_LISTS', 'true').lower() == 'true' +SYNC_CUSTOM_RULES = os.environ.get('SYNC_CUSTOM_RULES', 'true').lower() == 'true' + REFRESH_INTERVAL_SECS = int(os.environ.get('REFRESH_INTERVAL_SECS', '60')) -class UnauthenticatedError(Exception): - pass - - def get_login_cookie(url, user, passwd): """ Logs into AdGuard URL using username/password and returns a valid session cookie. @@ -44,65 +51,6 @@ def get_login_cookie(url, user, passwd): return response.cookies['agh_session'] -def get_entries(url, cookie): - """ - Retrieves all existing entries from AdGuard. - :param url: Base AdGuard URL - :param cookie: Session token - :return: List of Entries - """ - cookies = { - 'agh_session': cookie - } - response = requests.get('{}/control/rewrite/list'.format(url), cookies=cookies) - - if response.status_code == 403: - raise UnauthenticatedError - - entry_array = json.loads(response.text) - - return entry_array - - -def update_entries(url, cookie, sync_entries): - """ - Update entries from your primary to secondary AdGuard. - - ADD: Will add the entry with the domain pointing to IP. - UPDATE: Will update existing entry to point the domain to the new IP. - DEL: Will delete the existing entry from secondary AdGuard. - :param url: URL of the Secondary AdGuard - :param cookie: Secondary AdGuard Auth Cookie. - :param sync_entries: Array of entries to be sync. - :return: None - """ - - cookies = { - 'agh_session': cookie - } - - for entry in sync_entries: - if entry['action'] == 'ADD': - print(" - Adding entry ({} => {})".format(entry['domain'], entry['answer'])) - data = { - 'domain': entry['domain'], - 'answer': entry['answer'] - } - response = requests.post('{}/control/rewrite/add'.format(url), cookies=cookies, data=json.dumps(data)) - if response.status_code == 403: - raise UnauthenticatedError - - elif entry['action'] == 'DEL': - print(" - Deleting entry ({} => {})".format(entry['domain'], entry['answer'])) - data = { - 'domain': entry['domain'], - 'answer': entry['answer'] - } - response = requests.post('{}/control/rewrite/delete'.format(url), cookies=cookies, data=json.dumps(data)) - if response.status_code == 403: - raise UnauthenticatedError - - if __name__ == '__main__': print("Running Adguard Sync for '{}' => '{}'..".format(ADGUARD_PRIMARY, ADGUARD_SECONDARY)) @@ -115,28 +63,21 @@ if __name__ == '__main__': while True: try: - primary_entries = get_entries(ADGUARD_PRIMARY, primary_cookie) - secondary_entries = get_entries(ADGUARD_SECONDARY, secondary_cookie) + # Reconcile entries + if SYNC_ENTRIES: + entries.reconcile(ADGUARD_PRIMARY, ADGUARD_SECONDARY, primary_cookie, secondary_cookie) - sync_entries = [] + # Reconcile blocked services + if SYNC_BLOCKED_SERVICES: + blocked_services.reconcile(ADGUARD_PRIMARY, ADGUARD_SECONDARY, primary_cookie, secondary_cookie) - for e in primary_entries: - if e not in secondary_entries: - sync_entries.append({ - 'action': 'ADD', - 'domain': e['domain'], - 'answer': e['answer'] - }) + # Reconcile block/allow lists + if SYNC_BLOCK_ALLOW_LISTS: + block_allow_lists.reconcile(ADGUARD_PRIMARY, ADGUARD_SECONDARY, primary_cookie, secondary_cookie) - for s in secondary_entries: - if s not in primary_entries: - sync_entries.append({ - 'action': 'DEL', - 'domain': s['domain'], - 'answer': s['answer'] - }) - - update_entries(ADGUARD_SECONDARY, secondary_cookie, sync_entries) + # Reconcile custom rules + if SYNC_CUSTOM_RULES: + custom_rules.reconcile(ADGUARD_PRIMARY, ADGUARD_SECONDARY, primary_cookie, secondary_cookie) except UnauthenticatedError: primary_cookie = get_login_cookie(ADGUARD_PRIMARY, ADGUARD_USER, ADGUARD_PASS) diff --git a/src/block_allow_lists/__init__.py b/src/block_allow_lists/__init__.py new file mode 100644 index 0000000..63aa051 --- /dev/null +++ b/src/block_allow_lists/__init__.py @@ -0,0 +1,204 @@ +import requests +import os +import json +import time +from exceptions import UnauthenticatedError + + +def _get_block_allow_lists(url, cookie): + """ + Retrieves all existing blocklists from AdGuard. + :param url: Base AdGuard URL + :param cookie: Session token + :return: List of Entries + """ + cookies = { + 'agh_session': cookie + } + + formatted_block_allow_lists = { + 'blocklists': {}, + 'allowlists': {} + } + + response = requests.get('{}/control/filtering/status'.format(url), cookies=cookies) + + if response.status_code == 403: + raise UnauthenticatedError + + resp_obj = json.loads(response.text) + blocklist_array = resp_obj['filters'] + + if blocklist_array is not None: + for blocklist in blocklist_array: + formatted_block_allow_lists['blocklists'][blocklist['url']] = { + 'id': blocklist['id'], + 'name': blocklist['name'], + 'url': blocklist['url'], + 'enabled': blocklist['enabled'] + } + + allowlist_array = resp_obj['whitelist_filters'] + + if allowlist_array is not None: + for allowlist in allowlist_array: + formatted_block_allow_lists['allowlists'][allowlist['url']] = { + 'id': allowlist['id'], + 'name': allowlist['name'], + 'url': allowlist['url'], + 'enabled': allowlist['enabled'] + } + + return formatted_block_allow_lists + + +def _update_block_allow_lists(url, cookie, sync_block_allow_lists): + """ + Update blocked services from your primary to secondary AdGuard. + :param url: URL of the Secondary AdGuard + :param cookie: Secondary AdGuard Auth Cookie. + :param sync_blocked_services: Array of entries to be sync. + :return: None + """ + + cookies = { + 'agh_session': cookie + } + + # Perform deletes first to avoid any conflicts since URLs cannot exist in both. + for del_allowlist in sync_block_allow_lists['allowlists']['del']: + print(" - Deleting allowlist entry ({})".format(del_allowlist['url'])) + data = { + 'url': del_allowlist['url'], + 'allowlist': True + } + response = requests.post('{}/control/filtering/remove_url'.format(url), cookies=cookies, data=json.dumps(data)) + + if response.status_code == 403: + raise UnauthenticatedError + + for del_blocklist in sync_block_allow_lists['blocklists']['del']: + print(" - Deleting blocklist entry ({})".format(del_blocklist['url'])) + data = { + 'url': del_blocklist['url'], + 'allowlist': False + } + response = requests.post('{}/control/filtering/remove_url'.format(url), cookies=cookies, data=json.dumps(data)) + + if response.status_code == 403: + raise UnauthenticatedError + + # Perform adds second + for add_allowlist in sync_block_allow_lists['allowlists']['add']: + print(" - Adding allowlist entry ({})".format(add_allowlist['url'])) + data = { + 'name': add_allowlist['name'], + 'url': add_allowlist['url'], + 'allowlist': True + } + response = requests.post('{}/control/filtering/add_url'.format(url), cookies=cookies, data=json.dumps(data)) + + if response.status_code == 403: + raise UnauthenticatedError + + for add_blocklist in sync_block_allow_lists['blocklists']['add']: + print(" - Adding blocklist entry ({})".format(add_blocklist['url'])) + data = { + 'name': add_blocklist['name'], + 'url': add_blocklist['url'], + 'allowlist': False + } + response = requests.post('{}/control/filtering/add_url'.format(url), cookies=cookies, data=json.dumps(data)) + + if response.status_code == 403: + raise UnauthenticatedError + + # Modify any existing out of sync entry + for mod in sync_block_allow_lists['mods']: + data = { + 'url': mod['url'], + 'data': { + 'name': mod['name'], + 'url': mod['url'], + 'enabled': mod['enabled'] + }, + 'allowlist': mod['allowlist'] + } + + print(" - Updating modified entry ({})".format(mod['url'])) + response = requests.post('{}/control/filtering/set_url'.format(url), cookies=cookies, data=json.dumps(data)) + + if response.status_code == 403: + raise UnauthenticatedError + + +def reconcile(adguard_primary, adguard_secondary, primary_cookie, secondary_cookie): + """ + Reconcile blocklists from primary to secondary Adguards. + Uses the URL as the unique identifier between instances. + :param adguard_primary: URL of primary Adguard. + :param adguard_secondary: URL of secondardy Adguard. + :param primary_cookie: Auth cookie for primary Adguard. + :param secondary_cookie: Auth cookie for secondary Adguard. + """ + primary_block_allow_lists = _get_block_allow_lists(adguard_primary, primary_cookie) + secondary_block_allow_lists = _get_block_allow_lists(adguard_secondary, secondary_cookie) + + sync_block_allow_lists = { + 'blocklists': { + 'add': [], + 'del': [] + }, + 'allowlists': { + 'add': [], + 'del': [] + }, + 'mods': [] + } + + + for k,v in primary_block_allow_lists['blocklists'].items(): + if k not in secondary_block_allow_lists['blocklists']: + sync_block_allow_lists['blocklists']['add'].append({ + 'url': v['url'], + 'name': v['name'], + 'enabled': v['enabled'] + }) + else: + if primary_block_allow_lists['blocklists'][k]['enabled'] != secondary_block_allow_lists['blocklists'][k]['enabled'] or primary_block_allow_lists['blocklists'][k]['name'] != secondary_block_allow_lists['blocklists'][k]['name']: + sync_block_allow_lists['mods'].append({ + 'enabled': primary_block_allow_lists['blocklists'][k]['enabled'], + 'name': primary_block_allow_lists['blocklists'][k]['name'], + 'url': k, + 'allowlist': False + }) + + for k,v in secondary_block_allow_lists['blocklists'].items(): + if k not in primary_block_allow_lists['blocklists']: + sync_block_allow_lists['blocklists']['del'].append({ + 'url': v['url'] + }) + + for k,v in primary_block_allow_lists['allowlists'].items(): + if k not in secondary_block_allow_lists['allowlists']: + sync_block_allow_lists['allowlists']['add'].append({ + 'url': v['url'], + 'name': v['name'], + 'enabled': v['enabled'] + }) + else: + if primary_block_allow_lists['allowlists'][k]['enabled'] != secondary_block_allow_lists['allowlists'][k]['enabled'] or primary_block_allow_lists['allowlists'][k]['name'] != secondary_block_allow_lists['allowlists'][k]['name']: + sync_block_allow_lists['mods'].append({ + 'enabled': primary_block_allow_lists['allowlists'][k]['enabled'], + 'name': primary_block_allow_lists['allowlists'][k]['name'], + 'url': k, + 'allowlist': True + }) + + for k,v in secondary_block_allow_lists['allowlists'].items(): + if k not in primary_block_allow_lists['allowlists']: + sync_block_allow_lists['allowlists']['del'].append({ + 'url': v['url'] + }) + + _update_block_allow_lists(adguard_secondary, secondary_cookie, sync_block_allow_lists) \ No newline at end of file diff --git a/src/blocked_services/__init__.py b/src/blocked_services/__init__.py new file mode 100644 index 0000000..90e27c1 --- /dev/null +++ b/src/blocked_services/__init__.py @@ -0,0 +1,69 @@ +import requests +import os +import json +import time +from exceptions import UnauthenticatedError + + +def _get_blocked_services(url, cookie): + """ + Retrieves all existing blocked services from AdGuard. + :param url: Base AdGuard URL + :param cookie: Session token + :return: List of Entries + """ + cookies = { + 'agh_session': cookie + } + + response = requests.get('{}/control/blocked_services/list'.format(url), cookies=cookies) + + if response.status_code == 403: + raise UnauthenticatedError + + blocked_service_array = json.loads(response.text) + + return blocked_service_array + + +def _update_blocked_services(url, cookie, sync_blocked_services): + """ + Update blocked services from your primary to secondary AdGuard. + :param url: URL of the Secondary AdGuard + :param cookie: Secondary AdGuard Auth Cookie. + :param sync_blocked_services: Array of entries to be sync. + :return: None + """ + + cookies = { + 'agh_session': cookie + } + + print(" - Syncing blocked services") + response = requests.post('{}/control/blocked_services/set'.format(url), cookies=cookies, data=json.dumps(sync_blocked_services)) + + if response.status_code == 403: + raise UnauthenticatedError + + +def reconcile(adguard_primary, adguard_secondary, primary_cookie, secondary_cookie): + """ + Reconcile blocked services from primary to secondary Adguards. + :param adguard_primary: URL of primary Adguard. + :param adguard_secondary: URL of secondardy Adguard. + :param primary_cookie: Auth cookie for primary Adguard. + :param secondary_cookie: Auth cookie for secondary Adguard. + """ + primary_blocked_services = _get_blocked_services(adguard_primary, primary_cookie) + secondary_blocked_services = _get_blocked_services(adguard_secondary, secondary_cookie) + + for bs in primary_blocked_services: + if bs not in secondary_blocked_services: + _update_blocked_services(adguard_secondary, secondary_cookie, primary_blocked_services) + break + + for bs in secondary_blocked_services: + if bs not in primary_blocked_services: + _update_blocked_services(adguard_secondary, secondary_cookie, primary_blocked_services) + break + \ No newline at end of file diff --git a/src/custom_rules/__init__.py b/src/custom_rules/__init__.py new file mode 100644 index 0000000..bac92c4 --- /dev/null +++ b/src/custom_rules/__init__.py @@ -0,0 +1,63 @@ +import requests +import os +import json +import time +from exceptions import UnauthenticatedError + + +def _get_custom_rules(url, cookie): + """ + Retrieves all existing blocked services from AdGuard. + :param url: Base AdGuard URL + :param cookie: Session token + :return: List of Entries + """ + cookies = { + 'agh_session': cookie + } + + response = requests.get('{}/control/filtering/status'.format(url), cookies=cookies) + + if response.status_code == 403: + raise UnauthenticatedError + + resp = json.loads(response.text) + custom_rules_array = resp['user_rules'] + custom_rules_str = '\n'.join(custom_rules_array) + + return custom_rules_str + + +def _update_custom_rules(url, cookie, custom_rules): + """ + Update blocked services from your primary to secondary AdGuard. + :param url: URL of the Secondary AdGuard + :param cookie: Secondary AdGuard Auth Cookie. + :param sync_blocked_services: Array of entries to be sync. + :return: None + """ + + cookies = { + 'agh_session': cookie + } + + print(" - Syncing custom rules") + response = requests.post('{}/control/filtering/set_rules'.format(url), cookies=cookies, data=custom_rules) + + if response.status_code == 403: + raise UnauthenticatedError + + +def reconcile(adguard_primary, adguard_secondary, primary_cookie, secondary_cookie): + """ + Reconcile blocked services from primary to secondary Adguards. + :param adguard_primary: URL of primary Adguard. + :param adguard_secondary: URL of secondardy Adguard. + :param primary_cookie: Auth cookie for primary Adguard. + :param secondary_cookie: Auth cookie for secondary Adguard. + """ + primary_custom_rules = _get_custom_rules(adguard_primary, primary_cookie) + secondary_custom_rules = _get_custom_rules(adguard_secondary, secondary_cookie) + + if primary_custom_rules != secondary_custom_rules: + _update_custom_rules(adguard_secondary, secondary_cookie, primary_custom_rules) \ No newline at end of file diff --git a/src/entries/__init__.py b/src/entries/__init__.py new file mode 100644 index 0000000..faf64f0 --- /dev/null +++ b/src/entries/__init__.py @@ -0,0 +1,87 @@ +import requests +import os +import json +import time +from exceptions import UnauthenticatedError + + +def _get_entries(url, cookie): + """ + Retrieves all existing entries from AdGuard. + :param url: Base AdGuard URL + :param cookie: Session token + :return: List of Entries + """ + cookies = { + 'agh_session': cookie + } + response = requests.get('{}/control/rewrite/list'.format(url), cookies=cookies) + + if response.status_code == 403: + raise UnauthenticatedError + + entry_array = json.loads(response.text) + + return entry_array + +def _update_entries(url, cookie, sync_entries): + """ + Update entries from your primary to secondary AdGuard. + + ADD: Will add the entry with the domain pointing to IP. + UPDATE: Will update existing entry to point the domain to the new IP. + DEL: Will delete the existing entry from secondary AdGuard. + :param url: URL of the Secondary AdGuard + :param cookie: Secondary AdGuard Auth Cookie. + :param sync_entries: Array of entries to be sync. + :return: None + """ + + cookies = { + 'agh_session': cookie + } + + for entry in sync_entries: + if entry['action'] == 'ADD': + print(" - Adding entry ({} => {})".format(entry['domain'], entry['answer'])) + data = { + 'domain': entry['domain'], + 'answer': entry['answer'] + } + response = requests.post('{}/control/rewrite/add'.format(url), cookies=cookies, data=json.dumps(data)) + if response.status_code == 403: + raise UnauthenticatedError + + elif entry['action'] == 'DEL': + print(" - Deleting entry ({} => {})".format(entry['domain'], entry['answer'])) + data = { + 'domain': entry['domain'], + 'answer': entry['answer'] + } + response = requests.post('{}/control/rewrite/delete'.format(url), cookies=cookies, data=json.dumps(data)) + if response.status_code == 403: + raise UnauthenticatedError + +def reconcile(adguard_primary, adguard_secondary, primary_cookie, secondary_cookie): + primary_entries = _get_entries(adguard_primary, primary_cookie) + secondary_entries = _get_entries(adguard_secondary, secondary_cookie) + + sync_entries = [] + + for e in primary_entries: + if e not in secondary_entries: + sync_entries.append({ + 'action': 'ADD', + 'domain': e['domain'], + 'answer': e['answer'] + }) + + for s in secondary_entries: + if s not in primary_entries: + sync_entries.append({ + 'action': 'DEL', + 'domain': s['domain'], + 'answer': s['answer'] + }) + + _update_entries(adguard_secondary, secondary_cookie, sync_entries) \ No newline at end of file diff --git a/src/exceptions.py b/src/exceptions.py new file mode 100644 index 0000000..3d5cb06 --- /dev/null +++ b/src/exceptions.py @@ -0,0 +1,2 @@ +class UnauthenticatedError(Exception): + pass