From 5d1dccde995138e631916dcef44a7527b7cdc6b6 Mon Sep 17 00:00:00 2001 From: morpheus65535 Date: Sat, 18 Dec 2021 09:41:48 -0500 Subject: [PATCH] Added vip and daily download limit to addic7ed and try to slow down a little to prevent being ip blocked by the provider. --- bazarr/config.py | 3 +- bazarr/get_providers.py | 1 + frontend/src/Settings/Providers/list.ts | 4 ++ libs/subliminal_patch/providers/addic7ed.py | 66 +++++++++++++++++---- 4 files changed, 62 insertions(+), 12 deletions(-) diff --git a/bazarr/config.py b/bazarr/config.py index 1982ebaf9..d74990316 100644 --- a/bazarr/config.py +++ b/bazarr/config.py @@ -132,7 +132,8 @@ defaults = { }, 'addic7ed': { 'username': '', - 'password': '' + 'password': '', + 'vip': 'False' }, 'podnapisi': { 'verify_ssl': 'True' diff --git a/bazarr/get_providers.py b/bazarr/get_providers.py index fd6f75dea..692818e0a 100644 --- a/bazarr/get_providers.py +++ b/bazarr/get_providers.py @@ -127,6 +127,7 @@ def get_providers_auth(): 'addic7ed' : { 'username': settings.addic7ed.username, 'password': settings.addic7ed.password, + 'is_vip': settings.addic7ed.getboolean('vip'), }, 'opensubtitles' : { 'username' : settings.opensubtitles.username, diff --git a/frontend/src/Settings/Providers/list.ts b/frontend/src/Settings/Providers/list.ts index 60b0df56d..e1e24a96c 100644 --- a/frontend/src/Settings/Providers/list.ts +++ b/frontend/src/Settings/Providers/list.ts @@ -21,6 +21,10 @@ export const ProviderList: Readonly = [ defaultKey: { username: "", password: "", + vip: false, + }, + keyNameOverride: { + vip: "VIP", }, }, { key: "argenteam", description: "LATAM Spanish Subtitles Provider" }, diff --git a/libs/subliminal_patch/providers/addic7ed.py b/libs/subliminal_patch/providers/addic7ed.py index 01e312b31..66e29070c 100644 --- a/libs/subliminal_patch/providers/addic7ed.py +++ b/libs/subliminal_patch/providers/addic7ed.py @@ -23,6 +23,8 @@ from subzero.language import Language logger = logging.getLogger(__name__) +show_cells_re = re.compile(b'.*?', re.DOTALL) + #: Series header parsing regex series_year_re = re.compile(r'^(?P[ \w\'.:(),*&!?-]+?)(?: \((?P\d{4})\))?$') @@ -68,6 +70,7 @@ class Addic7edProvider(_Addic7edProvider): languages.update(set(Language.rebuild(l, hi=True) for l in languages)) video_types = (Episode, Movie) + vip = False USE_ADDICTED_RANDOM_AGENTS = False hearing_impaired_verifiable = True subtitle_class = Addic7edSubtitle @@ -76,9 +79,10 @@ class Addic7edProvider(_Addic7edProvider): sanitize_characters = {'-', ':', '(', ')', '.', '/'} last_show_ids_fetch_key = "addic7ed_last_id_fetch" - def __init__(self, username=None, password=None, use_random_agents=False): + def __init__(self, username=None, password=None, use_random_agents=False, is_vip=False): super(Addic7edProvider, self).__init__(username=username, password=password) self.USE_ADDICTED_RANDOM_AGENTS = use_random_agents + self.vip = is_vip if not all((username, password)): raise ConfigurationError('Username and password must be specified') @@ -95,7 +99,7 @@ class Addic7edProvider(_Addic7edProvider): # login if self.username and self.password: def check_verification(cache_region): - rr = self.session.get(self.server_url + 'panel.php', allow_redirects=False, timeout=60, + rr = self.session.get(self.server_url + 'panel.php', allow_redirects=False, timeout=10, headers={"Referer": self.server_url}) if rr.status_code == 302: logger.info('Addic7ed: Login expired') @@ -115,7 +119,7 @@ class Addic7edProvider(_Addic7edProvider): tries = 0 while tries <= 3: tries += 1 - r = self.session.get(self.server_url + 'login.php', timeout=60, headers={"Referer": self.server_url}) + r = self.session.get(self.server_url + 'login.php', timeout=10, headers={"Referer": self.server_url}) if "g-recaptcha" in r.text or "grecaptcha" in r.text: logger.info('Addic7ed: Solving captcha. This might take a couple of minutes, but should only ' 'happen once every so often') @@ -139,11 +143,13 @@ class Addic7edProvider(_Addic7edProvider): if tries >= 3: raise Exception("Addic7ed: Couldn't solve captcha!") logger.info("Addic7ed: Couldn't solve captcha! Retrying") + time.sleep(4) continue data[g] = result - r = self.session.post(self.server_url + 'dologin.php', data, allow_redirects=False, timeout=60, + time.sleep(1) + r = self.session.post(self.server_url + 'dologin.php', data, allow_redirects=False, timeout=10, headers={"Referer": self.server_url + "login.php"}) if "relax, slow down" in r.text: @@ -157,6 +163,7 @@ class Addic7edProvider(_Addic7edProvider): logger.error("Addic7ed: Something went wrong when logging in") raise AuthenticationError(self.username) logger.info("Addic7ed: Something went wrong when logging in; retrying") + time.sleep(4) continue break @@ -165,6 +172,8 @@ class Addic7edProvider(_Addic7edProvider): logger.debug('Addic7ed: Logged in') self.logged_in = True + time.sleep(2) + def terminate(self): self.session.close() @@ -238,7 +247,7 @@ class Addic7edProvider(_Addic7edProvider): # get the movie id logger.info('Getting movie id') - r = self.session.get(self.server_url + 'search.php?search=' + quote_plus(movie), timeout=60) + r = self.session.get(self.server_url + 'search.php?search=' + quote_plus(movie), timeout=10) r.raise_for_status() soup = ParserBeautifulSoup(r.content.decode('utf-8', 'ignore'), ['lxml', 'html.parser']) @@ -285,10 +294,18 @@ class Addic7edProvider(_Addic7edProvider): logger.info('Getting show ids') region.set(self.last_show_ids_fetch_key, datetime.datetime.now()) - r = self.session.get(self.server_url + 'shows.php', timeout=60) + r = self.session.get(self.server_url + 'shows.php', timeout=10) r.raise_for_status() - soup = ParserBeautifulSoup(r.content.decode('utf-8', 'ignore'), ['lxml', 'html.parser']) + # LXML parser seems to fail when parsing Addic7ed.com HTML markup. + # Last known version to work properly is 3.6.4 (next version, 3.7.0, fails) + # Assuming the site's markup is bad, and stripping it down to only contain what's needed. + show_cells = re.findall(show_cells_re, r.content) + if show_cells: + soup = ParserBeautifulSoup(''.join(show_cells).decode('utf-8', 'ignore'), ['lxml', 'html.parser']) + else: + # If RegEx fails, fall back to original r.content and use 'html.parser' + soup = ParserBeautifulSoup(r.content, ['html.parser']) # populate the show ids show_ids = {} @@ -345,7 +362,7 @@ class Addic7edProvider(_Addic7edProvider): headers = { "referer": self.server_url + "srch.php" } - r = self.session.get(self.server_url + endpoint, params=params, timeout=60, headers=headers) + r = self.session.get(self.server_url + endpoint, params=params, timeout=10, headers=headers) r.raise_for_status() if r.text and "Sorry, your search" not in r.text: @@ -386,7 +403,7 @@ class Addic7edProvider(_Addic7edProvider): logger.info('Getting the page of show id %d, season %d', show_id, season) r = self.session.get(self.server_url + 'ajax_loadShow.php', params={'show': show_id, 'season': season}, - timeout=60, + timeout=10, headers={ "referer": "%sshow/%s" % (self.server_url, show_id), "X-Requested-With": "XMLHttpRequest" @@ -446,7 +463,7 @@ class Addic7edProvider(_Addic7edProvider): # get the page of the movie logger.info('Getting the page of movie id %d', movie_id) r = self.session.get(self.server_url + 'movie/' + movie_id, - timeout=60, + timeout=10, headers={ "referer": self.server_url, "X-Requested-With": "XMLHttpRequest" @@ -550,9 +567,30 @@ class Addic7edProvider(_Addic7edProvider): return [] def download_subtitle(self, subtitle): + last_dls = region.get("addic7ed_dls") + now = datetime.datetime.now() + one_day = datetime.timedelta(hours=24) + + def raise_limit(): + logger.info("Addic7ed: Downloads per day exceeded (%s)", cap) + raise DownloadLimitPerDayExceeded + + if type(last_dls) is not list: + last_dls = [] + else: + # filter all non-expired DLs + last_dls = list(filter(lambda t: t + one_day > now, last_dls)) + region.set("addic7ed_dls", last_dls) + + cap = self.vip and 80 or 40 + amount = len(last_dls) + + if amount >= cap: + raise_limit() + # download the subtitle r = self.session.get(self.server_url + subtitle.download_link, headers={'Referer': subtitle.page_link}, - timeout=60) + timeout=10) r.raise_for_status() if r.status_code == 304: @@ -569,3 +607,9 @@ class Addic7edProvider(_Addic7edProvider): raise DownloadLimitExceeded subtitle.content = fix_line_ending(r.content) + last_dls.append(datetime.datetime.now()) + region.set("addic7ed_dls", last_dls) + logger.info("Addic7ed: Used %s/%s downloads", amount + 1, cap) + + if amount + 1 >= cap: + raise_limit()