Sync DNS/encryption/general settings and make it DRY

README.md

@@ -24,6 +24,9 @@ AdGuard Sync is packaged as a Docker image and can be ran anywhere with access t
 | SYNC_BLOCKED_SERVICES | No | If 'true', will sync blocked services. | true |
 | SYNC_BLOCK_ALLOW_LISTS | No | If 'true', will sync block/allow lists. | true |
 | SYNC_CUSTOM_RULES | No | If 'true', will sync custom rules. | true |
+| SYNC_GENERAL_SETTINGS | No | If 'true', will sync general settings. | true |
+| SYNC_DNS_SETTINGS | No | If 'true', will sync DNS settings. | true |
+| SYNC_ENCRYPTION_SETTINGS | No | If 'true', will sync encrypt settings. | false |
 
 Once you've updated the file and ensure you have `docker` and `docker-compose` installed, run the following in the root directory:
 
@@ -38,3 +41,7 @@ docker-compose logs
 ```
 
 **NOTE:** The container is set to automatically restart when the docker daemon restarts.
+
+### Encryption Syncing with Certifications/Keys
+
+If you plan to sync encryption settings across environments and you're using paths for certificates/keys, you *must make sure the files exist in both primary and secondary AdGuard instances*! Given this, `SYNC_ENCRYPTION_SETTINGS` is defaulted to `false` as a safety measure.

src/app.py

@@ -6,7 +6,10 @@ import entries
 import blocked_services
 import block_allow_lists
 import custom_rules
-from exceptions import UnauthenticatedError
+import filtering
+from exceptions import UnauthenticatedError, SystemError
+from settings import general, dns, encryption
+import common
 
 ADGUARD_PRIMARY = os.environ['ADGUARD_PRIMARY']
 ADGUARD_SECONDARY = os.environ['ADGUARD_SECONDARY']
@@ -23,6 +26,9 @@ SYNC_ENTRIES = os.environ.get('SYNC_ENTRIES', 'true').lower() == 'true'
 SYNC_BLOCKED_SERVICES = os.environ.get('SYNC_BLOCKED_SERVICES', 'true').lower() == 'true'
 SYNC_BLOCK_ALLOW_LISTS = os.environ.get('SYNC_BLOCK_ALLOW_LISTS', 'true').lower() == 'true'
 SYNC_CUSTOM_RULES = os.environ.get('SYNC_CUSTOM_RULES', 'true').lower() == 'true'
+SYNC_GENERAL_SETTINGS = os.environ.get('SYNC_GENERAL_SETTINGS', 'true').lower() == 'true'
+SYNC_DNS_SETTINGS = os.environ.get('SYNC_DNS_SETTINGS', 'true').lower() == 'true'
+SYNC_ENCRYPTION_SETTINGS = os.environ.get('SYNC_ENCRYPTION_SETTINGS', 'false').lower() == 'true'
 
 REFRESH_INTERVAL_SECS = int(os.environ.get('REFRESH_INTERVAL_SECS', '60'))
 
@@ -63,6 +69,10 @@ if __name__ == '__main__':
 
     while True:
         try:
+            # Since a bunch of things use filtering status, only retrieve it once per loop to reduce API calls
+            primary_filtering_status = common.get_response('{}/control/filtering/status'.format(ADGUARD_PRIMARY), primary_cookie)
+            secondary_filtering_status = common.get_response('{}/control/filtering/status'.format(ADGUARD_SECONDARY), secondary_cookie)
+
             # Reconcile entries
             if SYNC_ENTRIES:
                 entries.reconcile(ADGUARD_PRIMARY, ADGUARD_SECONDARY, primary_cookie, secondary_cookie)
@@ -73,11 +83,23 @@ if __name__ == '__main__':
 
             # Reconcile block/allow lists
             if SYNC_BLOCK_ALLOW_LISTS:
-                block_allow_lists.reconcile(ADGUARD_PRIMARY, ADGUARD_SECONDARY, primary_cookie, secondary_cookie)
+                block_allow_lists.reconcile(primary_filtering_status, secondary_filtering_status, ADGUARD_SECONDARY, secondary_cookie)
 
             # Reconcile custom rules
             if SYNC_CUSTOM_RULES:
-                custom_rules.reconcile(ADGUARD_PRIMARY, ADGUARD_SECONDARY, primary_cookie, secondary_cookie)
+                custom_rules.reconcile(primary_filtering_status, secondary_filtering_status, ADGUARD_SECONDARY, secondary_cookie)
+
+            # Reconcile general settings
+            if SYNC_GENERAL_SETTINGS:
+                general.reconcile(primary_filtering_status, secondary_filtering_status, ADGUARD_PRIMARY, primary_cookie, ADGUARD_SECONDARY, secondary_cookie)
+
+            # Reconcile DNS settings
+            if SYNC_DNS_SETTINGS:
+                dns.reconcile(ADGUARD_PRIMARY, primary_cookie, ADGUARD_SECONDARY, secondary_cookie)
+
+            # Reconcile encrypting settings
+            if SYNC_ENCRYPTION_SETTINGS:
+                encryption.reconcile(ADGUARD_PRIMARY, primary_cookie, ADGUARD_SECONDARY, secondary_cookie)
 
         except UnauthenticatedError:
             primary_cookie = get_login_cookie(ADGUARD_PRIMARY, ADGUARD_USER, ADGUARD_PASS)
@@ -86,4 +108,7 @@ if __name__ == '__main__':
             if primary_cookie is None or secondary_cookie is None:
                 exit(1)
 
+        except SystemError:
+            print('ERROR: Not able to reach AdGuard. Is it running?')
+
         time.sleep(REFRESH_INTERVAL_SECS)

src/block_allow_lists/__init__.py

@@ -1,33 +1,21 @@
 import requests
-import os
 import json
-import time
-from exceptions import UnauthenticatedError
+from exceptions import UnauthenticatedError, SystemError
 
 
-def _get_block_allow_lists(url, cookie):
+def _get_block_allow_lists(filtering_status):
     """
     Retrieves all existing blocklists from AdGuard.
     :param url: Base AdGuard URL
     :param cookie: Session token
     :return: List of Entries
     """
-    cookies = {
-        'agh_session': cookie
-    }
-
     formatted_block_allow_lists = {
         'blocklists': {},
         'allowlists': {}
     }
 
-    response = requests.get('{}/control/filtering/status'.format(url), cookies=cookies)
-
-    if response.status_code == 403:
-        raise UnauthenticatedError
-
-    resp_obj = json.loads(response.text)
-    blocklist_array = resp_obj['filters']
+    blocklist_array = filtering_status['filters']
 
     if blocklist_array is not None:
         for blocklist in blocklist_array:
@@ -38,7 +26,7 @@ def _get_block_allow_lists(url, cookie):
                 'enabled': blocklist['enabled']
             }
 
-    allowlist_array = resp_obj['whitelist_filters']
+    allowlist_array = filtering_status['whitelist_filters']
 
     if allowlist_array is not None:
         for allowlist in allowlist_array:
@@ -76,6 +64,8 @@ def _update_block_allow_lists(url, cookie, sync_block_allow_lists):
         
         if response.status_code == 403:
             raise UnauthenticatedError
+        elif response.status_code != 200:
+            raise SystemError
 
     for del_blocklist in sync_block_allow_lists['blocklists']['del']:
         print("  - Deleting blocklist entry ({})".format(del_blocklist['url']))
@@ -87,6 +77,8 @@ def _update_block_allow_lists(url, cookie, sync_block_allow_lists):
         
         if response.status_code == 403:
             raise UnauthenticatedError
+        elif response.status_code != 200:
+            raise SystemError
 
     # Perform adds second
     for add_allowlist in sync_block_allow_lists['allowlists']['add']:
@@ -100,6 +92,8 @@ def _update_block_allow_lists(url, cookie, sync_block_allow_lists):
         
         if response.status_code == 403:
             raise UnauthenticatedError
+        elif response.status_code != 200:
+            raise SystemError
 
     for add_blocklist in sync_block_allow_lists['blocklists']['add']:
         print("  - Adding blocklist entry ({})".format(add_blocklist['url']))
@@ -112,6 +106,8 @@ def _update_block_allow_lists(url, cookie, sync_block_allow_lists):
         
         if response.status_code == 403:
             raise UnauthenticatedError
+        elif response.status_code != 200:
+            raise SystemError
     
     # Modify any existing out of sync entry
     for mod in sync_block_allow_lists['mods']:
@@ -130,9 +126,11 @@ def _update_block_allow_lists(url, cookie, sync_block_allow_lists):
         
         if response.status_code == 403:
             raise UnauthenticatedError
+        elif response.status_code != 200:
+            raise SystemError
 
 
-def reconcile(adguard_primary, adguard_secondary, primary_cookie, secondary_cookie):
+def reconcile(primary_filtering_status, secondary_filtering_status, adguard_secondary, secondary_cookie):
     """
     Reconcile blocklists from primary to secondary Adguards.
     Uses the URL as the unique identifier between instances.
@@ -141,8 +139,8 @@ def reconcile(adguard_primary, adguard_secondary, primary_cookie, secondary_cook
     :param primary_cookie: Auth cookie for primary Adguard.
     :param secondary_cookie: Auth cookie for secondary Adguard.
     """
-    primary_block_allow_lists = _get_block_allow_lists(adguard_primary, primary_cookie)
-    secondary_block_allow_lists = _get_block_allow_lists(adguard_secondary, secondary_cookie)
+    primary_block_allow_lists = _get_block_allow_lists(primary_filtering_status)
+    secondary_block_allow_lists = _get_block_allow_lists(secondary_filtering_status)
 
     sync_block_allow_lists = {
         'blocklists': {

src/blocked_services/__init__.py

@@ -1,8 +1,7 @@
 import requests
-import os
 import json
-import time
-from exceptions import UnauthenticatedError
+import common
+from exceptions import UnauthenticatedError, SystemError
 
 
 def _get_blocked_services(url, cookie):
@@ -12,18 +11,8 @@ def _get_blocked_services(url, cookie):
     :param cookie: Session token
     :return: List of Entries
     """
-    cookies = {
-        'agh_session': cookie
-    }
 
-    response = requests.get('{}/control/blocked_services/list'.format(url), cookies=cookies)
-
-    if response.status_code == 403:
-        raise UnauthenticatedError
-
-    blocked_service_array = json.loads(response.text)
-
-    return blocked_service_array
+    return common.get_response('{}/control/blocked_services/list'.format(url), cookie)
 
 
 def _update_blocked_services(url, cookie, sync_blocked_services):
@@ -44,6 +33,8 @@ def _update_blocked_services(url, cookie, sync_blocked_services):
     
     if response.status_code == 403:
         raise UnauthenticatedError
+    elif response.status_code != 200:
+        raise SystemError
 
 
 def reconcile(adguard_primary, adguard_secondary, primary_cookie, secondary_cookie):

src/common.py

@@ -0,0 +1,45 @@
+import requests
+import os
+import json
+import time
+from exceptions import UnauthenticatedError, SystemError
+
+def get_response(url, cookie):
+    """
+    Helper function to handle errors and keep it DRY
+    """
+    cookies = {
+        'agh_session': cookie
+    }
+
+    response = requests.get(url, cookies=cookies)
+
+    if response.status_code == 403:
+        raise UnauthenticatedError
+    elif response.status_code != 200:
+        raise SystemError
+    
+    return json.loads(response.text)
+
+
+def update_settings(setting, primary_settings, secondary_settings, url, cookie):
+    """
+    Update main DNS settings on secondary AdGuard if necessary
+    :param setting: Name of the setting to change.
+    :param primary_settings: Primary settings for primary AdGuard.
+    :param secondary_settings: Secondary settings for secondary AdGuard.
+    :param url: Base URL for updating settings.
+    :param cookie: Auth cookie.
+    """
+    cookies = {
+        'agh_session': cookie
+    }
+
+    if primary_settings != secondary_settings:
+        print("  - Updating {} settings".format(setting))
+        response = requests.post(url, cookies=cookies, data=json.dumps(primary_settings))
+
+        if response.status_code == 403:
+            raise UnauthenticatedError
+        elif response.status_code != 200:
+            raise SystemError

src/custom_rules/__init__.py

@@ -2,27 +2,18 @@ import requests
 import os
 import json
 import time
-from exceptions import UnauthenticatedError
+from exceptions import UnauthenticatedError, SystemError
 
 
-def _get_custom_rules(url, cookie):
+def _get_custom_rules(filtering_status):
     """
     Retrieves all existing blocked services from AdGuard.
     :param url: Base AdGuard URL
     :param cookie: Session token
     :return: List of Entries
     """
-    cookies = {
-        'agh_session': cookie
-    }
-
-    response = requests.get('{}/control/filtering/status'.format(url), cookies=cookies)
-
-    if response.status_code == 403:
-        raise UnauthenticatedError
-
-    resp = json.loads(response.text)
-    custom_rules_array = resp['user_rules']
+    
+    custom_rules_array = filtering_status['user_rules']
     custom_rules_str = '\n'.join(custom_rules_array)
 
     return custom_rules_str
@@ -46,9 +37,11 @@ def _update_custom_rules(url, cookie, custom_rules):
     
     if response.status_code == 403:
         raise UnauthenticatedError
+    elif response.status_code != 200:
+        raise SystemError
 
 
-def reconcile(adguard_primary, adguard_secondary, primary_cookie, secondary_cookie):
+def reconcile(primary_filtering_status, secondary_filtering_status, adguard_secondary, secondary_cookie):
     """
     Reconcile blocked services from primary to secondary Adguards.
     :param adguard_primary: URL of primary Adguard.
@@ -56,8 +49,8 @@ def reconcile(adguard_primary, adguard_secondary, primary_cookie, secondary_cook
     :param primary_cookie: Auth cookie for primary Adguard.
     :param secondary_cookie: Auth cookie for secondary Adguard.
     """
-    primary_custom_rules = _get_custom_rules(adguard_primary, primary_cookie)
-    secondary_custom_rules = _get_custom_rules(adguard_secondary, secondary_cookie)
+    primary_custom_rules = _get_custom_rules(primary_filtering_status)
+    secondary_custom_rules = _get_custom_rules(secondary_filtering_status)
 
     if primary_custom_rules != secondary_custom_rules:
         _update_custom_rules(adguard_secondary, secondary_cookie, primary_custom_rules)

src/entries/__init__.py

@@ -1,8 +1,7 @@
 import requests
-import os
 import json
-import time
-from exceptions import UnauthenticatedError
+import common
+from exceptions import UnauthenticatedError, SystemError
 
 
 def _get_entries(url, cookie):
@@ -12,17 +11,9 @@ def _get_entries(url, cookie):
     :param cookie: Session token
     :return: List of Entries
     """
-    cookies = {
-        'agh_session': cookie
-    }
-    response = requests.get('{}/control/rewrite/list'.format(url), cookies=cookies)
 
-    if response.status_code == 403:
-        raise UnauthenticatedError
+    return common.get_response('{}/control/rewrite/list'.format(url), cookie)
 
-    entry_array = json.loads(response.text)
-
-    return entry_array
 
 def _update_entries(url, cookie, sync_entries):
     """
@@ -51,6 +42,8 @@ def _update_entries(url, cookie, sync_entries):
             response = requests.post('{}/control/rewrite/add'.format(url), cookies=cookies, data=json.dumps(data))
             if response.status_code == 403:
                 raise UnauthenticatedError
+            elif response.status_code != 200:
+                raise SystemError
 
         elif entry['action'] == 'DEL':
             print("  - Deleting entry ({} => {})".format(entry['domain'], entry['answer']))
@@ -61,6 +54,8 @@ def _update_entries(url, cookie, sync_entries):
             response = requests.post('{}/control/rewrite/delete'.format(url), cookies=cookies, data=json.dumps(data))
             if response.status_code == 403:
                 raise UnauthenticatedError
+            elif response.status_code != 200:
+                raise SystemError
 
 def reconcile(adguard_primary, adguard_secondary, primary_cookie, secondary_cookie):
     primary_entries = _get_entries(adguard_primary, primary_cookie)

src/exceptions.py

@@ -1,2 +1,5 @@
 class UnauthenticatedError(Exception):
     pass
+
+class SystemError(Exception):
+    pass

src/settings/dns.py

@@ -0,0 +1,59 @@
+import common
+
+def _get_dns_settings(url, cookie):
+    """
+    Retrieves all existing blocked services from AdGuard.
+    :param url: Base AdGuard URL
+    :param cookie: Session token
+    :return: List of Entries
+    """
+
+    settings = {
+        'upstream': {},
+        'server': {},
+        'cache': {},
+        'access': {}
+    }
+
+    # Retrieve DNS/cache setting
+    response = common.get_response('{}/control/dns_info'.format(url), cookie)
+    settings['upstream']['upstream_dns'] = response['upstream_dns']
+    settings['upstream']['bootstrap_dns'] = response['bootstrap_dns']
+    settings['upstream']['local_ptr_upstreams'] = response['local_ptr_upstreams']
+    settings['upstream']['resolve_clients'] = response['resolve_clients']
+    settings['upstream']['upstream_mode'] = response['upstream_mode']
+
+    settings['server']['blocking_ipv4'] = response['blocking_ipv4']
+    settings['server']['blocking_ipv6'] = response['blocking_ipv6']
+    settings['server']['blocking_mode'] = response['blocking_mode']
+    settings['server']['disable_ipv6'] = response['disable_ipv6']
+    settings['server']['dnssec_enabled'] = response['dnssec_enabled']
+    settings['server']['edns_cs_enabled'] = response['edns_cs_enabled']
+    settings['server']['ratelimit'] = response['ratelimit']
+
+    settings['cache']['cache_size'] =response['cache_size']
+    settings['cache']['cache_ttl_max'] = response['cache_ttl_max']
+    settings['cache']['cache_ttl_min'] = response['cache_ttl_min']
+
+    # Retrieve safesearch setting
+    response = common.get_response('{}/control/access/list'.format(url), cookie)
+    settings['access'] = response
+
+    return settings
+
+
+def reconcile(adguard_primary, primary_cookie, adguard_secondary, secondary_cookie):
+    """
+    Reconcile blocked services from primary to secondary Adguards.
+    :param adguard_primary: URL of primary Adguard.
+    :param adguard_secondary: URL of secondardy Adguard.
+    :param primary_cookie: Auth cookie for primary Adguard.
+    :param secondary_cookie: Auth cookie for secondary Adguard.
+    """
+    primary_dns_settings = _get_dns_settings(adguard_primary, primary_cookie)
+    secondary_dns_settings = _get_dns_settings(adguard_secondary, secondary_cookie)
+
+    common.update_settings('DNS upstream', primary_dns_settings['upstream'], secondary_dns_settings['upstream'], '{}/control/dns_config'.format(adguard_secondary), secondary_cookie)
+    common.update_settings('DNS server', primary_dns_settings['server'], secondary_dns_settings['server'], '{}/control/dns_config'.format(adguard_secondary), secondary_cookie) 
+    common.update_settings('DNS cache', primary_dns_settings['cache'], secondary_dns_settings['cache'], '{}/control/dns_config'.format(adguard_secondary), secondary_cookie)
+    common.update_settings('access', primary_dns_settings['access'], secondary_dns_settings['access'], '{}/control/access/set'.format(adguard_secondary), secondary_cookie)

src/settings/encryption.py

@@ -0,0 +1,26 @@
+import common
+
+def _get_encryption_settings(url, cookie):
+    """
+    Retrieves all existing encryption settings from AdGuard.
+    :param url: Base AdGuard URL
+    :param cookie: Session token
+    :return: List of Entries
+    """
+
+    # Retrieve encryption setting
+    return common.get_response('{}/control/tls/status'.format(url), cookie)
+
+
+def reconcile(adguard_primary, primary_cookie, adguard_secondary, secondary_cookie):
+    """
+    Reconcile encryption settings from primary to secondary Adguards.
+    :param adguard_primary: URL of primary Adguard.
+    :param adguard_secondary: URL of secondardy Adguard.
+    :param primary_cookie: Auth cookie for primary Adguard.
+    :param secondary_cookie: Auth cookie for secondary Adguard.
+    """
+    primary_encryption_settings = _get_encryption_settings(adguard_primary, primary_cookie)
+    secondary_encryption_settings = _get_encryption_settings(adguard_secondary, secondary_cookie)
+
+    common.update_settings('encryption', primary_encryption_settings, secondary_encryption_settings, '{}/control/tls/configure'.format(adguard_secondary), secondary_cookie)

src/settings/general.py

@@ -0,0 +1,96 @@
+import requests
+from exceptions import UnauthenticatedError, SystemError
+import common
+
+
+def _get_general_settings(filtering_status, url, cookie):
+    """
+    Retrieves all general settings from AdGuard.
+    :param url: Base AdGuard URL
+    :param cookie: Session token
+    :return: List of Entries
+    """
+
+    settings = {}
+
+    # Retrieve safebrowsing setting
+    response = common.get_response('{}/control/safebrowsing/status'.format(url), cookie)
+    settings['safebrowsing'] = response['enabled']
+
+    # Retrieve safesearch setting
+    response = common.get_response('{}/control/safesearch/status'.format(url), cookie)
+    settings['safesearch'] = response['enabled']
+
+    # Retrieve parental setting
+    response = common.get_response('{}/control/parental/status'.format(url), cookie)
+    settings['parental'] = response['enabled']
+
+    # Retrieve querylog setting
+    response = common.get_response('{}/control/querylog_info'.format(url), cookie)
+    settings['querylog_info'] = response
+
+    # Retrieve stats setting
+    response = common.get_response('{}/control/stats_info'.format(url), cookie)
+    settings['stats_info'] = response
+
+    # Set relevant filtering status
+    settings['filtering'] = {
+        'enabled': filtering_status['enabled'],
+        'interval': filtering_status['interval']
+    }
+
+    return settings
+
+
+def _update_enable_setting(setting, enabled, url, cookie):
+    """
+    Update enable/disable setting in secondary AdGuard.
+    :param setting: Name of the setting to be added to URL
+    :param enabled: Bool if the setting should be enabled/disabled
+    :param url: URL of the Secondary AdGuard
+    :param cookie: Secondary AdGuard Auth Cookie.
+    :return: None
+    """
+    cookies = {
+        'agh_session': cookie
+    }
+
+    print("  - Updating {} setting".format(setting))
+    if enabled:
+        response = requests.post('{}/control/{}/enable'.format(url, setting), cookies=cookies)
+    else:
+        response = requests.post('{}/control/{}/disable'.format(url, setting), cookies=cookies)
+
+    if response.status_code == 403:
+        raise UnauthenticatedError
+    elif response.status_code != 200:
+        raise SystemError
+
+
+def reconcile(primary_filtering_status, secondary_filtering_status, adguard_primary, primary_cookie, adguard_secondary, secondary_cookie):
+    """
+    Reconcile blocked services from primary to secondary Adguards.
+    :param adguard_primary: URL of primary Adguard.
+    :param adguard_secondary: URL of secondardy Adguard.
+    :param primary_cookie: Auth cookie for primary Adguard.
+    :param secondary_cookie: Auth cookie for secondary Adguard.
+    """
+    primary_general_settings = _get_general_settings(primary_filtering_status, adguard_primary, primary_cookie)
+    secondary_general_settings = _get_general_settings(secondary_filtering_status, adguard_secondary, secondary_cookie)
+
+    # Safesearch Update
+    if primary_general_settings['safesearch'] != secondary_general_settings['safesearch']:
+        _update_enable_setting('safesearch', primary_general_settings['safesearch'], adguard_secondary, secondary_cookie)
+
+    # Safebrowsing Update
+    if primary_general_settings['safebrowsing'] != secondary_general_settings['safebrowsing']:
+        _update_enable_setting('safebrowsing', primary_general_settings['safebrowsing'], adguard_secondary, secondary_cookie)
+
+    # Parental Update
+    if primary_general_settings['parental'] != secondary_general_settings['parental']:
+        _update_enable_setting('parental', primary_general_settings['parental'], adguard_secondary, secondary_cookie)
+
+    # Updating other settings, a little more complicated so passing all logic to function
+    common.update_settings('filtering', primary_general_settings['filtering'], secondary_general_settings['filtering'], '{}/control/filtering/config'.format(adguard_secondary), secondary_cookie)
+    common.update_settings('querylog', primary_general_settings['querylog_info'], secondary_general_settings['querylog_info'], '{}/control/querylog_config'.format(adguard_secondary), secondary_cookie)
+    common.update_settings('status', primary_general_settings['stats_info'], secondary_general_settings['stats_info'], '{}/control/stats_config'.format(adguard_secondary), secondary_cookie)