atoy3731/adguard-sync
1
1
Fork 0
mirror of https://github.com/atoy3731/adguard-sync.git synced 2024-09-20 06:56:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add capability for advanced sync of other elements (#4)

Browse source 
* Add capability for advanced sync of other elements

* Revert default refresh interval

* Add fix for secondary_blocked_services

* Add logging for modified entry

* Update language

* Bug fix
This commit is contained in:
Adam Toy 2021-11-06 16:52:23 -04:00 committed by GitHub
parent 6c1a1fcd2e
commit cd8338fbd0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 462 additions and 86 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
.gitignore vendored
View file

@ -1,4 +1,8 @@
venv/
*.pyc
.idea/
*.iml
*.iml
.venv/
.vscode/
.env
__pycache__

6
Dockerfile
View file

@ -12,8 +12,10 @@ COPY requirements.txt /tmp/requirements.txt
RUN pip3 install -r /tmp/requirements.txt && \
rm -f /tmp/requirements.txt
COPY src/app.py /opt/app.py
COPY src /opt/app
WORKDIR /opt/app
ENTRYPOINT ["python3"]
CMD ["/opt/app.py"]
CMD ["app.py"]

4
README.md
View file

@ -20,6 +20,10 @@ AdGuard Sync is packaged as a Docker image and can be ran anywhere with access t
| SECONDARY_ADGUARD_USER | No | Username to log into your secondary AdGuard instance. Only necessary if credentials are different between primary and secondary | Value of 'ADGUARD_USER' |
| SECONDARY_ADGUARD_PASS | No | Password to log into your secondary AdGuard instance. Only necessary if credentials are different between primary and secondary | Value of 'ADGUARD_PASS' |
| REFRESH_INTERVAL_SECS | No | Frequency in seconds to refresh entries. | 60 |
| SYNC_ENTRIES | No | If 'true', will sync rewrite entries. | true |
| SYNC_BLOCKED_SERVICES | No | If 'true', will sync blocked services. | true |
| SYNC_BLOCK_ALLOW_LISTS | No | If 'true', will sync block/allow lists. | true |
| SYNC_CUSTOM_RULES | No | If 'true', will sync custom rules. | true |
Once you've updated the file and ensure you have `docker` and `docker-compose` installed, run the following in the root directory:

2
VERSION
View file

@ -1 +1 @@
1.1
2.0

105
src/app.py
View file

@ -2,6 +2,11 @@ import requests
import os
import json
import time
import entries
import blocked_services
import block_allow_lists
import custom_rules
from exceptions import UnauthenticatedError
ADGUARD_PRIMARY = os.environ['ADGUARD_PRIMARY']
ADGUARD_SECONDARY = os.environ['ADGUARD_SECONDARY']
@ -13,13 +18,15 @@ ADGUARD_PASS = os.environ['ADGUARD_PASS']
SECONDARY_ADGUARD_USER = os.environ.get('SECONDARY_ADGUARD_USER', ADGUARD_USER)
SECONDARY_ADGUARD_PASS = os.environ.get('SECONDARY_ADGUARD_PASS', ADGUARD_PASS)
# By default, sync all
SYNC_ENTRIES = os.environ.get('SYNC_ENTRIES', 'true').lower() == 'true'
SYNC_BLOCKED_SERVICES = os.environ.get('SYNC_BLOCKED_SERVICES', 'true').lower() == 'true'
SYNC_BLOCK_ALLOW_LISTS = os.environ.get('SYNC_BLOCK_ALLOW_LISTS', 'true').lower() == 'true'
SYNC_CUSTOM_RULES = os.environ.get('SYNC_CUSTOM_RULES', 'true').lower() == 'true'
REFRESH_INTERVAL_SECS = int(os.environ.get('REFRESH_INTERVAL_SECS', '60'))
class UnauthenticatedError(Exception):
pass
def get_login_cookie(url, user, passwd):
"""
Logs into AdGuard URL using username/password and returns a valid session cookie.
@ -44,65 +51,6 @@ def get_login_cookie(url, user, passwd):
return response.cookies['agh_session']
def get_entries(url, cookie):
"""
Retrieves all existing entries from AdGuard.
:param url: Base AdGuard URL
:param cookie: Session token
:return: List of Entries
"""
cookies = {
'agh_session': cookie
}
response = requests.get('{}/control/rewrite/list'.format(url), cookies=cookies)
if response.status_code == 403:
raise UnauthenticatedError
entry_array = json.loads(response.text)
return entry_array
def update_entries(url, cookie, sync_entries):
"""
Update entries from your primary to secondary AdGuard.
ADD: Will add the entry with the domain pointing to IP.
UPDATE: Will update existing entry to point the domain to the new IP.
DEL: Will delete the existing entry from secondary AdGuard.
:param url: URL of the Secondary AdGuard
:param cookie: Secondary AdGuard Auth Cookie.
:param sync_entries: Array of entries to be sync.
:return: None
"""
cookies = {
'agh_session': cookie
}
for entry in sync_entries:
if entry['action'] == 'ADD':
print(" - Adding entry ({} => {})".format(entry['domain'], entry['answer']))
data = {
'domain': entry['domain'],
'answer': entry['answer']
}
response = requests.post('{}/control/rewrite/add'.format(url), cookies=cookies, data=json.dumps(data))
if response.status_code == 403:
raise UnauthenticatedError
elif entry['action'] == 'DEL':
print(" - Deleting entry ({} => {})".format(entry['domain'], entry['answer']))
data = {
'domain': entry['domain'],
'answer': entry['answer']
}
response = requests.post('{}/control/rewrite/delete'.format(url), cookies=cookies, data=json.dumps(data))
if response.status_code == 403:
raise UnauthenticatedError
if __name__ == '__main__':
print("Running Adguard Sync for '{}' => '{}'..".format(ADGUARD_PRIMARY, ADGUARD_SECONDARY))
@ -115,28 +63,21 @@ if __name__ == '__main__':
while True:
try:
primary_entries = get_entries(ADGUARD_PRIMARY, primary_cookie)
secondary_entries = get_entries(ADGUARD_SECONDARY, secondary_cookie)
# Reconcile entries
if SYNC_ENTRIES:
entries.reconcile(ADGUARD_PRIMARY, ADGUARD_SECONDARY, primary_cookie, secondary_cookie)
sync_entries = []
# Reconcile blocked services
if SYNC_BLOCKED_SERVICES:
blocked_services.reconcile(ADGUARD_PRIMARY, ADGUARD_SECONDARY, primary_cookie, secondary_cookie)
for e in primary_entries:
if e not in secondary_entries:
sync_entries.append({
'action': 'ADD',
'domain': e['domain'],
'answer': e['answer']
})
# Reconcile block/allow lists
if SYNC_BLOCK_ALLOW_LISTS:
block_allow_lists.reconcile(ADGUARD_PRIMARY, ADGUARD_SECONDARY, primary_cookie, secondary_cookie)
for s in secondary_entries:
if s not in primary_entries:
sync_entries.append({
'action': 'DEL',
'domain': s['domain'],
'answer': s['answer']
})
update_entries(ADGUARD_SECONDARY, secondary_cookie, sync_entries)
# Reconcile custom rules
if SYNC_CUSTOM_RULES:
custom_rules.reconcile(ADGUARD_PRIMARY, ADGUARD_SECONDARY, primary_cookie, secondary_cookie)
except UnauthenticatedError:
primary_cookie = get_login_cookie(ADGUARD_PRIMARY, ADGUARD_USER, ADGUARD_PASS)

204
src/block_allow_lists/__init__.py Normal file
View file

@ -0,0 +1,204 @@
import requests
import os
import json
import time
from exceptions import UnauthenticatedError
def _get_block_allow_lists(url, cookie):
"""
Retrieves all existing blocklists from AdGuard.
:param url: Base AdGuard URL
:param cookie: Session token
:return: List of Entries
"""
cookies = {
'agh_session': cookie
}
formatted_block_allow_lists = {
'blocklists': {},
'allowlists': {}
}
response = requests.get('{}/control/filtering/status'.format(url), cookies=cookies)
if response.status_code == 403:
raise UnauthenticatedError
resp_obj = json.loads(response.text)
blocklist_array = resp_obj['filters']
if blocklist_array is not None:
for blocklist in blocklist_array:
formatted_block_allow_lists['blocklists'][blocklist['url']] = {
'id': blocklist['id'],
'name': blocklist['name'],
'url': blocklist['url'],
'enabled': blocklist['enabled']
}
allowlist_array = resp_obj['whitelist_filters']
if allowlist_array is not None:
for allowlist in allowlist_array:
formatted_block_allow_lists['allowlists'][allowlist['url']] = {
'id': allowlist['id'],
'name': allowlist['name'],
'url': allowlist['url'],
'enabled': allowlist['enabled']
}
return formatted_block_allow_lists
def _update_block_allow_lists(url, cookie, sync_block_allow_lists):
"""
Update blocked services from your primary to secondary AdGuard.
:param url: URL of the Secondary AdGuard
:param cookie: Secondary AdGuard Auth Cookie.
:param sync_blocked_services: Array of entries to be sync.
:return: None
"""
cookies = {
'agh_session': cookie
}
# Perform deletes first to avoid any conflicts since URLs cannot exist in both.
for del_allowlist in sync_block_allow_lists['allowlists']['del']:
print(" - Deleting allowlist entry ({})".format(del_allowlist['url']))
data = {
'url': del_allowlist['url'],
'allowlist': True
}
response = requests.post('{}/control/filtering/remove_url'.format(url), cookies=cookies, data=json.dumps(data))
if response.status_code == 403:
raise UnauthenticatedError
for del_blocklist in sync_block_allow_lists['blocklists']['del']:
print(" - Deleting blocklist entry ({})".format(del_blocklist['url']))
data = {
'url': del_blocklist['url'],
'allowlist': False
}
response = requests.post('{}/control/filtering/remove_url'.format(url), cookies=cookies, data=json.dumps(data))
if response.status_code == 403:
raise UnauthenticatedError
# Perform adds second
for add_allowlist in sync_block_allow_lists['allowlists']['add']:
print(" - Adding allowlist entry ({})".format(add_allowlist['url']))
data = {
'name': add_allowlist['name'],
'url': add_allowlist['url'],
'allowlist': True
}
response = requests.post('{}/control/filtering/add_url'.format(url), cookies=cookies, data=json.dumps(data))
if response.status_code == 403:
raise UnauthenticatedError
for add_blocklist in sync_block_allow_lists['blocklists']['add']:
print(" - Adding blocklist entry ({})".format(add_blocklist['url']))
data = {
'name': add_blocklist['name'],
'url': add_blocklist['url'],
'allowlist': False
}
response = requests.post('{}/control/filtering/add_url'.format(url), cookies=cookies, data=json.dumps(data))
if response.status_code == 403:
raise UnauthenticatedError
# Modify any existing out of sync entry
for mod in sync_block_allow_lists['mods']:
data = {
'url': mod['url'],
'data': {
'name': mod['name'],
'url': mod['url'],
'enabled': mod['enabled']
},
'allowlist': mod['allowlist']
}
print(" - Updating modified entry ({})".format(mod['url']))
response = requests.post('{}/control/filtering/set_url'.format(url), cookies=cookies, data=json.dumps(data))
if response.status_code == 403:
raise UnauthenticatedError
def reconcile(adguard_primary, adguard_secondary, primary_cookie, secondary_cookie):
"""
Reconcile blocklists from primary to secondary Adguards.
Uses the URL as the unique identifier between instances.
:param adguard_primary: URL of primary Adguard.
:param adguard_secondary: URL of secondardy Adguard.
:param primary_cookie: Auth cookie for primary Adguard.
:param secondary_cookie: Auth cookie for secondary Adguard.
"""
primary_block_allow_lists = _get_block_allow_lists(adguard_primary, primary_cookie)
secondary_block_allow_lists = _get_block_allow_lists(adguard_secondary, secondary_cookie)
sync_block_allow_lists = {
'blocklists': {
'add': [],
'del': []
},
'allowlists': {
'add': [],
'del': []
},
'mods': []
}
for k,v in primary_block_allow_lists['blocklists'].items():
if k not in secondary_block_allow_lists['blocklists']:
sync_block_allow_lists['blocklists']['add'].append({
'url': v['url'],
'name': v['name'],
'enabled': v['enabled']
})
else:
if primary_block_allow_lists['blocklists'][k]['enabled'] != secondary_block_allow_lists['blocklists'][k]['enabled'] or primary_block_allow_lists['blocklists'][k]['name'] != secondary_block_allow_lists['blocklists'][k]['name']:
sync_block_allow_lists['mods'].append({
'enabled': primary_block_allow_lists['blocklists'][k]['enabled'],
'name': primary_block_allow_lists['blocklists'][k]['name'],
'url': k,
'allowlist': False
})
for k,v in secondary_block_allow_lists['blocklists'].items():
if k not in primary_block_allow_lists['blocklists']:
sync_block_allow_lists['blocklists']['del'].append({
'url': v['url']
})
for k,v in primary_block_allow_lists['allowlists'].items():
if k not in secondary_block_allow_lists['allowlists']:
sync_block_allow_lists['allowlists']['add'].append({
'url': v['url'],
'name': v['name'],
'enabled': v['enabled']
})
else:
if primary_block_allow_lists['allowlists'][k]['enabled'] != secondary_block_allow_lists['allowlists'][k]['enabled'] or primary_block_allow_lists['allowlists'][k]['name'] != secondary_block_allow_lists['allowlists'][k]['name']:
sync_block_allow_lists['mods'].append({
'enabled': primary_block_allow_lists['allowlists'][k]['enabled'],
'name': primary_block_allow_lists['allowlists'][k]['name'],
'url': k,
'allowlist': True
})
for k,v in secondary_block_allow_lists['allowlists'].items():
if k not in primary_block_allow_lists['allowlists']:
sync_block_allow_lists['allowlists']['del'].append({
'url': v['url']
})
_update_block_allow_lists(adguard_secondary, secondary_cookie, sync_block_allow_lists)

69
src/blocked_services/__init__.py Normal file
View file

@ -0,0 +1,69 @@
import requests
import os
import json
import time
from exceptions import UnauthenticatedError
def _get_blocked_services(url, cookie):
"""
Retrieves all existing blocked services from AdGuard.
:param url: Base AdGuard URL
:param cookie: Session token
:return: List of Entries
"""
cookies = {
'agh_session': cookie
}
response = requests.get('{}/control/blocked_services/list'.format(url), cookies=cookies)
if response.status_code == 403:
raise UnauthenticatedError
blocked_service_array = json.loads(response.text)
return blocked_service_array
def _update_blocked_services(url, cookie, sync_blocked_services):
"""
Update blocked services from your primary to secondary AdGuard.
:param url: URL of the Secondary AdGuard
:param cookie: Secondary AdGuard Auth Cookie.
:param sync_blocked_services: Array of entries to be sync.
:return: None
"""
cookies = {
'agh_session': cookie
}
print(" - Syncing blocked services")
response = requests.post('{}/control/blocked_services/set'.format(url), cookies=cookies, data=json.dumps(sync_blocked_services))
if response.status_code == 403:
raise UnauthenticatedError
def reconcile(adguard_primary, adguard_secondary, primary_cookie, secondary_cookie):
"""
Reconcile blocked services from primary to secondary Adguards.
:param adguard_primary: URL of primary Adguard.
:param adguard_secondary: URL of secondardy Adguard.
:param primary_cookie: Auth cookie for primary Adguard.
:param secondary_cookie: Auth cookie for secondary Adguard.
"""
primary_blocked_services = _get_blocked_services(adguard_primary, primary_cookie)
secondary_blocked_services = _get_blocked_services(adguard_secondary, secondary_cookie)
for bs in primary_blocked_services:
if bs not in secondary_blocked_services:
_update_blocked_services(adguard_secondary, secondary_cookie, primary_blocked_services)
break
for bs in secondary_blocked_services:
if bs not in primary_blocked_services:
_update_blocked_services(adguard_secondary, secondary_cookie, primary_blocked_services)
break

63
src/custom_rules/__init__.py Normal file
View file

@ -0,0 +1,63 @@
import requests
import os
import json
import time
from exceptions import UnauthenticatedError
def _get_custom_rules(url, cookie):
"""
Retrieves all existing blocked services from AdGuard.
:param url: Base AdGuard URL
:param cookie: Session token
:return: List of Entries
"""
cookies = {
'agh_session': cookie
}
response = requests.get('{}/control/filtering/status'.format(url), cookies=cookies)
if response.status_code == 403:
raise UnauthenticatedError
resp = json.loads(response.text)
custom_rules_array = resp['user_rules']
custom_rules_str = '\n'.join(custom_rules_array)
return custom_rules_str
def _update_custom_rules(url, cookie, custom_rules):
"""
Update blocked services from your primary to secondary AdGuard.
:param url: URL of the Secondary AdGuard
:param cookie: Secondary AdGuard Auth Cookie.
:param sync_blocked_services: Array of entries to be sync.
:return: None
"""
cookies = {
'agh_session': cookie
}
print(" - Syncing custom rules")
response = requests.post('{}/control/filtering/set_rules'.format(url), cookies=cookies, data=custom_rules)
if response.status_code == 403:
raise UnauthenticatedError
def reconcile(adguard_primary, adguard_secondary, primary_cookie, secondary_cookie):
"""
Reconcile blocked services from primary to secondary Adguards.
:param adguard_primary: URL of primary Adguard.
:param adguard_secondary: URL of secondardy Adguard.
:param primary_cookie: Auth cookie for primary Adguard.
:param secondary_cookie: Auth cookie for secondary Adguard.
"""
primary_custom_rules = _get_custom_rules(adguard_primary, primary_cookie)
secondary_custom_rules = _get_custom_rules(adguard_secondary, secondary_cookie)
if primary_custom_rules != secondary_custom_rules:
_update_custom_rules(adguard_secondary, secondary_cookie, primary_custom_rules)

87
src/entries/__init__.py Normal file
View file

@ -0,0 +1,87 @@
import requests
import os
import json
import time
from exceptions import UnauthenticatedError
def _get_entries(url, cookie):
"""
Retrieves all existing entries from AdGuard.
:param url: Base AdGuard URL
:param cookie: Session token
:return: List of Entries
"""
cookies = {
'agh_session': cookie
}
response = requests.get('{}/control/rewrite/list'.format(url), cookies=cookies)
if response.status_code == 403:
raise UnauthenticatedError
entry_array = json.loads(response.text)
return entry_array
def _update_entries(url, cookie, sync_entries):
"""
Update entries from your primary to secondary AdGuard.
ADD: Will add the entry with the domain pointing to IP.
UPDATE: Will update existing entry to point the domain to the new IP.
DEL: Will delete the existing entry from secondary AdGuard.
:param url: URL of the Secondary AdGuard
:param cookie: Secondary AdGuard Auth Cookie.
:param sync_entries: Array of entries to be sync.
:return: None
"""
cookies = {
'agh_session': cookie
}
for entry in sync_entries:
if entry['action'] == 'ADD':
print(" - Adding entry ({} => {})".format(entry['domain'], entry['answer']))
data = {
'domain': entry['domain'],
'answer': entry['answer']
}
response = requests.post('{}/control/rewrite/add'.format(url), cookies=cookies, data=json.dumps(data))
if response.status_code == 403:
raise UnauthenticatedError
elif entry['action'] == 'DEL':
print(" - Deleting entry ({} => {})".format(entry['domain'], entry['answer']))
data = {
'domain': entry['domain'],
'answer': entry['answer']
}
response = requests.post('{}/control/rewrite/delete'.format(url), cookies=cookies, data=json.dumps(data))
if response.status_code == 403:
raise UnauthenticatedError
def reconcile(adguard_primary, adguard_secondary, primary_cookie, secondary_cookie):
primary_entries = _get_entries(adguard_primary, primary_cookie)
secondary_entries = _get_entries(adguard_secondary, secondary_cookie)
sync_entries = []
for e in primary_entries:
if e not in secondary_entries:
sync_entries.append({
'action': 'ADD',
'domain': e['domain'],
'answer': e['answer']
})
for s in secondary_entries:
if s not in primary_entries:
sync_entries.append({
'action': 'DEL',
'domain': s['domain'],
'answer': s['answer']
})
_update_entries(adguard_secondary, secondary_cookie, sync_entries)

2
src/exceptions.py Normal file
View file

@ -0,0 +1,2 @@
class UnauthenticatedError(Exception):
pass