mirror of
https://github.com/morpheus65535/bazarr.git
synced 2025-01-01 04:22:07 +08:00
Improved search speed by reusing providers pools
This commit is contained in:
morpheus65535
GitHub
parent
01e1723325
commit
d8f14560e3
14 changed files with 627 additions and 351 deletions
|
|
@ -10,7 +10,7 @@ from database import TableEpisodes, get_audio_profile_languages, get_profile_id
|
|||
from ..utils import authenticate
|
||||
from helper import path_mappings
|
||||
from get_providers import get_providers, get_providers_auth
|
||||
from get_subtitle import download_subtitle, manual_upload_subtitle
|
||||
from get_subtitle import generate_subtitles, manual_upload_subtitle
|
||||
from utils import history_log, delete_subtitles
|
||||
from notifier import send_notifications
|
||||
from list_subtitles import store_subtitles
|
||||
|
|
@ -44,9 +44,6 @@ class EpisodesSubtitles(Resource):
|
|||
hi = request.form.get('hi').capitalize()
|
||||
forced = request.form.get('forced').capitalize()
|
||||
|
||||
providers_list = get_providers()
|
||||
providers_auth = get_providers_auth()
|
||||
|
||||
audio_language_list = get_audio_profile_languages(episode_id=sonarrEpisodeId)
|
||||
if len(audio_language_list) > 0:
|
||||
audio_language = audio_language_list[0]['name']
|
||||
|
|
@ -54,10 +51,10 @@ class EpisodesSubtitles(Resource):
|
|||
audio_language = None
|
||||
|
||||
try:
|
||||
result = download_subtitle(episodePath, language, audio_language, hi, forced, providers_list,
|
||||
providers_auth, sceneName, title, 'series',
|
||||
profile_id=get_profile_id(episode_id=sonarrEpisodeId))
|
||||
if result is not None:
|
||||
result = list(generate_subtitles(episodePath, [(language, hi, forced)], audio_language, sceneName,
|
||||
title, 'series', profile_id=get_profile_id(episode_id=sonarrEpisodeId)))
|
||||
if result:
|
||||
result = result[0]
|
||||
message = result[0]
|
||||
path = result[1]
|
||||
forced = result[5]
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from database import TableMovies, get_audio_profile_languages, get_profile_id
|
|||
from ..utils import authenticate
|
||||
from helper import path_mappings
|
||||
from get_providers import get_providers, get_providers_auth
|
||||
from get_subtitle import download_subtitle, manual_upload_subtitle
|
||||
from get_subtitle import manual_upload_subtitle, generate_subtitles
|
||||
from utils import history_log_movie, delete_subtitles
|
||||
from notifier import send_notifications_movie
|
||||
from list_subtitles import store_subtitles_movie
|
||||
|
|
@ -56,10 +56,10 @@ class MoviesSubtitles(Resource):
|
|||
audio_language = None
|
||||
|
||||
try:
|
||||
result = download_subtitle(moviePath, language, audio_language, hi, forced, providers_list,
|
||||
providers_auth, sceneName, title, 'movie',
|
||||
profile_id=get_profile_id(movie_id=radarrId))
|
||||
if result is not None:
|
||||
result = list(generate_subtitles(moviePath, [(language, hi, forced)], audio_language,
|
||||
sceneName, title, 'movie', profile_id=get_profile_id(movie_id=radarrId)))
|
||||
if result:
|
||||
result = result[0]
|
||||
message = result[0]
|
||||
path = result[1]
|
||||
forced = result[5]
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
# coding=utf-8
|
||||
# fmt: off
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
|
@ -12,15 +13,19 @@ import re
|
|||
import subliminal
|
||||
import copy
|
||||
import operator
|
||||
import time
|
||||
|
||||
from functools import reduce
|
||||
from inspect import getfullargspec
|
||||
from peewee import fn
|
||||
from datetime import datetime, timedelta
|
||||
from subzero.language import Language
|
||||
from subzero.video import parse_video
|
||||
from subliminal import region, score as subliminal_scores, \
|
||||
list_subtitles, Episode, Movie
|
||||
from subliminal_patch.core import SZAsyncProviderPool, download_best_subtitles, save_subtitles, download_subtitles, \
|
||||
list_all_subtitles, get_subtitle_path
|
||||
from subliminal_patch.core import SZAsyncProviderPool, save_subtitles, get_subtitle_path
|
||||
|
||||
from subliminal_patch.core_persistent import download_best_subtitles, list_all_subtitles, download_subtitles
|
||||
from subliminal_patch.score import compute_score
|
||||
from subliminal_patch.subtitle import Subtitle
|
||||
from get_languages import language_from_alpha3, alpha2_from_alpha3, alpha3_from_alpha2, language_from_alpha2, \
|
||||
|
|
@ -44,7 +49,6 @@ from analytics import track_event
|
|||
from locale import getpreferredencoding
|
||||
from score import movie_score, series_score
|
||||
|
||||
|
||||
def get_video(path, title, sceneName, providers=None, media_type="movie"):
|
||||
"""
|
||||
Construct `Video` instance
|
||||
|
|
@ -83,43 +87,122 @@ def get_video(path, title, sceneName, providers=None, media_type="movie"):
|
|||
logging.exception("BAZARR Error trying to get video information for this file: " + original_path)
|
||||
|
||||
|
||||
def download_subtitle(path, language, audio_language, hi, forced, providers, providers_auth, sceneName, title,
|
||||
media_type, forced_minimum_score=None, is_upgrade=False, profile_id=None):
|
||||
# fixme: supply all missing languages, not only one, to hit providers only once who support multiple languages in
|
||||
# one query
|
||||
# fmt: on
|
||||
def _init_pool(media_type, profile_id=None, providers=None):
|
||||
pool = provider_pool()
|
||||
return pool(
|
||||
providers=providers or get_providers(),
|
||||
provider_configs=get_providers_auth(),
|
||||
blacklist=get_blacklist(media_type),
|
||||
throttle_callback=provider_throttle,
|
||||
ban_list=get_ban_list(profile_id),
|
||||
language_hook=None,
|
||||
)
|
||||
|
||||
|
||||
_pools = {}
|
||||
|
||||
|
||||
def _get_pool(media_type, profile_id=None):
|
||||
try:
|
||||
return _pools[f'{media_type}_{profile_id or ""}']
|
||||
except KeyError:
|
||||
_update_pool(media_type, profile_id)
|
||||
|
||||
return _pools[f'{media_type}_{profile_id or ""}']
|
||||
|
||||
|
||||
def _update_pool(media_type, profile_id=None):
|
||||
pool_key = f'{media_type}_{profile_id or ""}'
|
||||
logging.debug("BAZARR updating pool: %s", pool_key)
|
||||
|
||||
# Init a new pool if not present
|
||||
if pool_key not in _pools:
|
||||
logging.debug("BAZARR pool not initialized: %s. Initializing", pool_key)
|
||||
_pools[pool_key] = _init_pool(media_type, profile_id)
|
||||
|
||||
pool = _pools[pool_key]
|
||||
if pool is None:
|
||||
return False
|
||||
|
||||
return pool.update(
|
||||
get_providers(),
|
||||
get_providers_auth(),
|
||||
get_blacklist(media_type),
|
||||
get_ban_list(profile_id),
|
||||
)
|
||||
|
||||
|
||||
def update_pools(f):
|
||||
"""Decorator that ensures all pools are updated on each function run.
|
||||
It will detect any config changes in Bazarr"""
|
||||
|
||||
def decorated(*args, **kwargs):
|
||||
logging.debug("BAZARR updating pools: %s", _pools)
|
||||
|
||||
start = time.time()
|
||||
args_spec = getfullargspec(f).args
|
||||
|
||||
try:
|
||||
profile_id = args[args_spec.index("profile_id")]
|
||||
except (IndexError, ValueError):
|
||||
profile_id = None
|
||||
|
||||
updated = _update_pool(args[args_spec.index("media_type")], profile_id)
|
||||
|
||||
if updated:
|
||||
logging.info("BAZARR pools update elapsed time: %s", time.time() - start)
|
||||
|
||||
return f(*args, **kwargs)
|
||||
|
||||
return decorated
|
||||
|
||||
|
||||
# fmt: off
|
||||
|
||||
@update_pools
|
||||
def generate_subtitles(path, languages, audio_language, sceneName, title, media_type,
|
||||
forced_minimum_score=None, is_upgrade=False, profile_id=None):
|
||||
if not languages:
|
||||
return None
|
||||
|
||||
if settings.general.getboolean('utf8_encode'):
|
||||
os.environ["SZ_KEEP_ENCODING"] = ""
|
||||
else:
|
||||
os.environ["SZ_KEEP_ENCODING"] = "True"
|
||||
|
||||
logging.debug('BAZARR Searching subtitles for this file: ' + path)
|
||||
if hi == "True":
|
||||
hi = "force HI"
|
||||
else:
|
||||
hi = "force non-HI"
|
||||
|
||||
if forced == "True":
|
||||
providers_auth['podnapisi']['only_foreign'] = True ## fixme: This is also in get_providers_auth()
|
||||
providers_auth['subscene']['only_foreign'] = True ## fixme: This is also in get_providers_auth()
|
||||
providers_auth['opensubtitles']['only_foreign'] = True ## fixme: This is also in get_providers_auth()
|
||||
else:
|
||||
providers_auth['podnapisi']['only_foreign'] = False
|
||||
providers_auth['subscene']['only_foreign'] = False
|
||||
providers_auth['opensubtitles']['only_foreign'] = False
|
||||
|
||||
language_set = set()
|
||||
|
||||
if not isinstance(language, list):
|
||||
language = [language]
|
||||
if not isinstance(languages, (set, list)):
|
||||
languages = [languages]
|
||||
|
||||
pool = _get_pool(media_type, profile_id)
|
||||
providers = pool.providers
|
||||
|
||||
for l in languages:
|
||||
l, hi_item, forced_item = l
|
||||
logging.debug('BAZARR Searching subtitles for this file: ' + path)
|
||||
if hi_item == "True":
|
||||
hi = "force HI"
|
||||
else:
|
||||
hi = "force non-HI"
|
||||
|
||||
# Fixme: This block should be updated elsewhere
|
||||
if forced_item == "True":
|
||||
pool.provider_configs['podnapisi']['only_foreign'] = True
|
||||
pool.provider_configs['subscene']['only_foreign'] = True
|
||||
pool.provider_configs['opensubtitles']['only_foreign'] = True
|
||||
else:
|
||||
pool.provider_configs['podnapisi']['only_foreign'] = False
|
||||
pool.provider_configs['subscene']['only_foreign'] = False
|
||||
pool.provider_configs['opensubtitles']['only_foreign'] = False
|
||||
|
||||
for l in language:
|
||||
# Always use alpha2 in API Request
|
||||
l = alpha3_from_alpha2(l)
|
||||
|
||||
lang_obj = _get_lang_obj(l)
|
||||
|
||||
if forced == "True":
|
||||
if forced_item == "True":
|
||||
lang_obj = Language.rebuild(lang_obj, forced=True)
|
||||
if hi == "force HI":
|
||||
lang_obj = Language.rebuild(lang_obj, hi=True)
|
||||
|
|
@ -144,6 +227,7 @@ def download_subtitle(path, language, audio_language, hi, forced, providers, pro
|
|||
"""
|
||||
video = get_video(force_unicode(path), title, sceneName, providers=providers,
|
||||
media_type=media_type)
|
||||
|
||||
if video:
|
||||
handler = series_score if media_type == "series" else movie_score
|
||||
min_score, max_score, scores = _get_scores(media_type, minimum_score_movie, minimum_score)
|
||||
|
|
@ -151,19 +235,11 @@ def download_subtitle(path, language, audio_language, hi, forced, providers, pro
|
|||
if providers:
|
||||
if forced_minimum_score:
|
||||
min_score = int(forced_minimum_score) + 1
|
||||
downloaded_subtitles = download_best_subtitles({video}, language_set, int(min_score), hi,
|
||||
providers=providers,
|
||||
provider_configs=providers_auth,
|
||||
pool_class=provider_pool(),
|
||||
downloaded_subtitles = download_best_subtitles({video}, language_set, pool,
|
||||
int(min_score), hi,
|
||||
compute_score=compute_score,
|
||||
throttle_time=None, # fixme
|
||||
blacklist=get_blacklist(media_type=media_type),
|
||||
ban_list=get_ban_list(profile_id),
|
||||
throttle_callback=provider_throttle,
|
||||
score_obj=handler,
|
||||
pre_download_hook=None, # fixme
|
||||
post_download_hook=None, # fixme
|
||||
language_hook=None) # fixme
|
||||
score_obj=handler)
|
||||
else:
|
||||
downloaded_subtitles = None
|
||||
logging.info("BAZARR All providers are throttled")
|
||||
|
|
@ -287,7 +363,7 @@ def download_subtitle(path, language, audio_language, hi, forced, providers, pro
|
|||
|
||||
track_event(category=downloaded_provider, action=action, label=downloaded_language)
|
||||
|
||||
return message, reversed_path, downloaded_language_code2, downloaded_provider, subtitle.score, \
|
||||
yield message, reversed_path, downloaded_language_code2, downloaded_provider, subtitle.score, \
|
||||
subtitle.language.forced, subtitle.id, reversed_subtitles_path, subtitle.language.hi
|
||||
|
||||
if not saved_any:
|
||||
|
|
@ -299,7 +375,8 @@ def download_subtitle(path, language, audio_language, hi, forced, providers, pro
|
|||
logging.debug('BAZARR Ended searching Subtitles for file: ' + path)
|
||||
|
||||
|
||||
def manual_search(path, profileId, providers, providers_auth, sceneName, title, media_type):
|
||||
@update_pools
|
||||
def manual_search(path, profile_id, providers, providers_auth, sceneName, title, media_type):
|
||||
logging.debug('BAZARR Manually searching subtitles for this file: ' + path)
|
||||
|
||||
final_subtitles = []
|
||||
|
|
@ -308,7 +385,8 @@ def manual_search(path, profileId, providers, providers_auth, sceneName, title,
|
|||
language_set = set()
|
||||
|
||||
# where [3] is items list of dict(id, lang, forced, hi)
|
||||
language_items = get_profiles_list(profile_id=int(profileId))['items']
|
||||
language_items = get_profiles_list(profile_id=int(profile_id))['items']
|
||||
pool = _get_pool(media_type, profile_id)
|
||||
|
||||
for language in language_items:
|
||||
forced = language['forced']
|
||||
|
|
@ -323,8 +401,8 @@ def manual_search(path, profileId, providers, providers_auth, sceneName, title,
|
|||
if forced == "True":
|
||||
lang_obj = Language.rebuild(lang_obj, forced=True)
|
||||
|
||||
providers_auth['podnapisi']['also_foreign'] = True
|
||||
providers_auth['opensubtitles']['also_foreign'] = True
|
||||
pool.provider_configs['podnapisi']['also_foreign'] = True
|
||||
pool.provider_configs['opensubtitles']['also_foreign'] = True
|
||||
|
||||
if hi == "True":
|
||||
lang_obj = Language.rebuild(lang_obj, hi=True)
|
||||
|
|
@ -358,29 +436,20 @@ def manual_search(path, profileId, providers, providers_auth, sceneName, title,
|
|||
|
||||
try:
|
||||
if providers:
|
||||
subtitles = list_all_subtitles([video], language_set,
|
||||
providers=providers,
|
||||
provider_configs=providers_auth,
|
||||
blacklist=get_blacklist(media_type=media_type),
|
||||
ban_list=get_ban_list(profileId),
|
||||
throttle_callback=provider_throttle,
|
||||
language_hook=None) # fixme
|
||||
subtitles = list_all_subtitles([video], language_set, pool)
|
||||
|
||||
if 'subscene' in providers:
|
||||
s_pool = _init_pool("movie", profile_id, {"subscene"})
|
||||
|
||||
subscene_language_set = set()
|
||||
for language in language_set:
|
||||
if language.forced:
|
||||
subscene_language_set.add(language)
|
||||
if len(subscene_language_set):
|
||||
providers_auth['subscene']['only_foreign'] = True
|
||||
subtitles_subscene = list_all_subtitles([video], subscene_language_set,
|
||||
providers=['subscene'],
|
||||
provider_configs=providers_auth,
|
||||
blacklist=get_blacklist(media_type=media_type),
|
||||
ban_list=get_ban_list(profileId),
|
||||
throttle_callback=provider_throttle,
|
||||
language_hook=None) # fixme
|
||||
providers_auth['subscene']['only_foreign'] = False
|
||||
s_pool.provider_configs['subscene'] = {}
|
||||
s_pool.provider_configs['subscene']['only_foreign'] = True
|
||||
subtitles_subscene = list_all_subtitles([video], subscene_language_set, s_pool)
|
||||
s_pool.provider_configs['subscene']['only_foreign'] = False
|
||||
subtitles[video] += subtitles_subscene[video]
|
||||
else:
|
||||
subtitles = []
|
||||
|
|
@ -407,6 +476,7 @@ def manual_search(path, profileId, providers, providers_auth, sceneName, title,
|
|||
logging.debug(u"BAZARR Skipping %s, because it doesn't match our series/episode", s)
|
||||
continue
|
||||
|
||||
initial_hi = None
|
||||
initial_hi_match = False
|
||||
for language in initial_language_set:
|
||||
if s.language.basename == language.basename and \
|
||||
|
|
@ -468,6 +538,7 @@ def manual_search(path, profileId, providers, providers_auth, sceneName, title,
|
|||
return final_subtitles
|
||||
|
||||
|
||||
@update_pools
|
||||
def manual_download_subtitle(path, language, audio_language, hi, forced, subtitle, provider, providers_auth, sceneName,
|
||||
title, media_type, profile_id):
|
||||
logging.debug('BAZARR Manually downloading Subtitles for this file: ' + path)
|
||||
|
|
@ -496,13 +567,7 @@ def manual_download_subtitle(path, language, audio_language, hi, forced, subtitl
|
|||
min_score, max_score, scores = _get_scores(media_type)
|
||||
try:
|
||||
if provider:
|
||||
download_subtitles([subtitle],
|
||||
providers={provider},
|
||||
provider_configs=providers_auth,
|
||||
pool_class=provider_pool(),
|
||||
blacklist=get_blacklist(media_type=media_type),
|
||||
ban_list=get_ban_list(profile_id),
|
||||
throttle_callback=provider_throttle)
|
||||
download_subtitles([subtitle], _get_pool(media_type, profile_id))
|
||||
logging.debug('BAZARR Subtitles file downloaded for this file:' + path)
|
||||
else:
|
||||
logging.info("BAZARR All providers are throttled")
|
||||
|
|
@ -765,8 +830,6 @@ def series_download_subtitles(no):
|
|||
"ignored because of monitored status, series type or series tags: {}".format(no))
|
||||
return
|
||||
|
||||
providers_auth = get_providers_auth()
|
||||
|
||||
count_episodes_details = len(episodes_details)
|
||||
|
||||
for i, episode in enumerate(episodes_details):
|
||||
|
|
@ -782,6 +845,13 @@ def series_download_subtitles(no):
|
|||
value=i,
|
||||
count=count_episodes_details)
|
||||
|
||||
audio_language_list = get_audio_profile_languages(episode_id=episode['sonarrEpisodeId'])
|
||||
if len(audio_language_list) > 0:
|
||||
audio_language = audio_language_list[0]['name']
|
||||
else:
|
||||
audio_language = 'None'
|
||||
|
||||
languages = []
|
||||
for language in ast.literal_eval(episode['missing_subtitles']):
|
||||
# confirm if language is still missing or if cutoff have been reached
|
||||
confirmed_missing_subs = TableEpisodes.select(TableEpisodes.missing_subtitles) \
|
||||
|
|
@ -792,40 +862,36 @@ def series_download_subtitles(no):
|
|||
continue
|
||||
|
||||
if language is not None:
|
||||
audio_language_list = get_audio_profile_languages(episode_id=episode['sonarrEpisodeId'])
|
||||
if len(audio_language_list) > 0:
|
||||
audio_language = audio_language_list[0]['name']
|
||||
else:
|
||||
audio_language = 'None'
|
||||
hi_ = "True" if language.endswith(':hi') else "False"
|
||||
forced_ ="True" if language.endswith(':forced') else "False"
|
||||
languages.append((language.split(":")[0], hi_, forced_))
|
||||
|
||||
result = download_subtitle(path_mappings.path_replace(episode['path']),
|
||||
language.split(':')[0],
|
||||
audio_language,
|
||||
"True" if language.endswith(':hi') else "False",
|
||||
"True" if language.endswith(':forced') else "False",
|
||||
providers_list,
|
||||
providers_auth,
|
||||
str(episode['scene_name']),
|
||||
episode['title'],
|
||||
'series')
|
||||
if result is not None:
|
||||
message = result[0]
|
||||
path = result[1]
|
||||
forced = result[5]
|
||||
if result[8]:
|
||||
language_code = result[2] + ":hi"
|
||||
elif forced:
|
||||
language_code = result[2] + ":forced"
|
||||
else:
|
||||
language_code = result[2]
|
||||
provider = result[3]
|
||||
score = result[4]
|
||||
subs_id = result[6]
|
||||
subs_path = result[7]
|
||||
store_subtitles(episode['path'], path_mappings.path_replace(episode['path']))
|
||||
history_log(1, no, episode['sonarrEpisodeId'], message, path, language_code, provider, score,
|
||||
subs_id, subs_path)
|
||||
send_notifications(no, episode['sonarrEpisodeId'], message)
|
||||
if not languages:
|
||||
continue
|
||||
|
||||
for result in generate_subtitles(path_mappings.path_replace(episode['path']),
|
||||
languages,
|
||||
audio_language,
|
||||
str(episode['scene_name']),
|
||||
episode['title'], 'series'):
|
||||
if result:
|
||||
message = result[0]
|
||||
path = result[1]
|
||||
forced = result[5]
|
||||
if result[8]:
|
||||
language_code = result[2] + ":hi"
|
||||
elif forced:
|
||||
language_code = result[2] + ":forced"
|
||||
else:
|
||||
language_code = result[2]
|
||||
provider = result[3]
|
||||
score = result[4]
|
||||
subs_id = result[6]
|
||||
subs_path = result[7]
|
||||
store_subtitles(episode['path'], path_mappings.path_replace(episode['path']))
|
||||
history_log(1, no, episode['sonarrEpisodeId'], message, path, language_code, provider, score,
|
||||
subs_id, subs_path)
|
||||
send_notifications(no, episode['sonarrEpisodeId'], message)
|
||||
else:
|
||||
logging.info("BAZARR All providers are throttled")
|
||||
break
|
||||
|
|
@ -871,6 +937,14 @@ def episode_download_subtitles(no, send_progress=False):
|
|||
episode['episodeTitle']),
|
||||
value=0,
|
||||
count=1)
|
||||
|
||||
audio_language_list = get_audio_profile_languages(episode_id=episode['sonarrEpisodeId'])
|
||||
if len(audio_language_list) > 0:
|
||||
audio_language = audio_language_list[0]['name']
|
||||
else:
|
||||
audio_language = 'None'
|
||||
|
||||
languages = []
|
||||
for language in ast.literal_eval(episode['missing_subtitles']):
|
||||
# confirm if language is still missing or if cutoff have been reached
|
||||
confirmed_missing_subs = TableEpisodes.select(TableEpisodes.missing_subtitles) \
|
||||
|
|
@ -881,40 +955,38 @@ def episode_download_subtitles(no, send_progress=False):
|
|||
continue
|
||||
|
||||
if language is not None:
|
||||
audio_language_list = get_audio_profile_languages(episode_id=episode['sonarrEpisodeId'])
|
||||
if len(audio_language_list) > 0:
|
||||
audio_language = audio_language_list[0]['name']
|
||||
else:
|
||||
audio_language = 'None'
|
||||
hi_ = "True" if language.endswith(':hi') else "False"
|
||||
forced_ ="True" if language.endswith(':forced') else "False"
|
||||
languages.append((language.split(":")[0], hi_, forced_))
|
||||
|
||||
if not languages:
|
||||
continue
|
||||
|
||||
for result in generate_subtitles(path_mappings.path_replace(episode['path']),
|
||||
languages,
|
||||
audio_language,
|
||||
str(episode['scene_name']),
|
||||
episode['title'],
|
||||
'series'):
|
||||
if result:
|
||||
message = result[0]
|
||||
path = result[1]
|
||||
forced = result[5]
|
||||
if result[8]:
|
||||
language_code = result[2] + ":hi"
|
||||
elif forced:
|
||||
language_code = result[2] + ":forced"
|
||||
else:
|
||||
language_code = result[2]
|
||||
provider = result[3]
|
||||
score = result[4]
|
||||
subs_id = result[6]
|
||||
subs_path = result[7]
|
||||
store_subtitles(episode['path'], path_mappings.path_replace(episode['path']))
|
||||
history_log(1, episode['sonarrSeriesId'], episode['sonarrEpisodeId'], message, path,
|
||||
language_code, provider, score, subs_id, subs_path)
|
||||
send_notifications(episode['sonarrSeriesId'], episode['sonarrEpisodeId'], message)
|
||||
|
||||
result = download_subtitle(path_mappings.path_replace(episode['path']),
|
||||
language.split(':')[0],
|
||||
audio_language,
|
||||
"True" if language.endswith(':hi') else "False",
|
||||
"True" if language.endswith(':forced') else "False",
|
||||
providers_list,
|
||||
providers_auth,
|
||||
str(episode['scene_name']),
|
||||
episode['title'],
|
||||
'series')
|
||||
if result is not None:
|
||||
message = result[0]
|
||||
path = result[1]
|
||||
forced = result[5]
|
||||
if result[8]:
|
||||
language_code = result[2] + ":hi"
|
||||
elif forced:
|
||||
language_code = result[2] + ":forced"
|
||||
else:
|
||||
language_code = result[2]
|
||||
provider = result[3]
|
||||
score = result[4]
|
||||
subs_id = result[6]
|
||||
subs_path = result[7]
|
||||
store_subtitles(episode['path'], path_mappings.path_replace(episode['path']))
|
||||
history_log(1, episode['sonarrSeriesId'], episode['sonarrEpisodeId'], message, path,
|
||||
language_code, provider, score, subs_id, subs_path)
|
||||
send_notifications(episode['sonarrSeriesId'], episode['sonarrEpisodeId'], message)
|
||||
if send_progress:
|
||||
hide_progress(id='episode_search_progress_{}'.format(no))
|
||||
else:
|
||||
|
|
@ -941,16 +1013,28 @@ def movies_download_subtitles(no):
|
|||
else:
|
||||
movie = movies[0]
|
||||
|
||||
providers_auth = get_providers_auth()
|
||||
|
||||
if ast.literal_eval(movie['missing_subtitles']):
|
||||
count_movie = len(ast.literal_eval(movie['missing_subtitles']))
|
||||
else:
|
||||
count_movie = 0
|
||||
|
||||
audio_language_list = get_audio_profile_languages(movie_id=movie['radarrId'])
|
||||
if len(audio_language_list) > 0:
|
||||
audio_language = audio_language_list[0]['name']
|
||||
else:
|
||||
audio_language = 'None'
|
||||
|
||||
languages = []
|
||||
providers_list = None
|
||||
|
||||
for i, language in enumerate(ast.literal_eval(movie['missing_subtitles'])):
|
||||
providers_list = get_providers()
|
||||
|
||||
if language is not None:
|
||||
hi_ = "True" if language.endswith(':hi') else "False"
|
||||
forced_ ="True" if language.endswith(':forced') else "False"
|
||||
languages.append((language.split(":")[0], hi_, forced_))
|
||||
|
||||
if providers_list:
|
||||
# confirm if language is still missing or if cutoff have been reached
|
||||
confirmed_missing_subs = TableMovies.select(TableMovies.missing_subtitles) \
|
||||
|
|
@ -966,47 +1050,100 @@ def movies_download_subtitles(no):
|
|||
value=i,
|
||||
count=count_movie)
|
||||
|
||||
if language is not None:
|
||||
audio_language_list = get_audio_profile_languages(movie_id=movie['radarrId'])
|
||||
if len(audio_language_list) > 0:
|
||||
audio_language = audio_language_list[0]['name']
|
||||
else:
|
||||
audio_language = 'None'
|
||||
if providers_list:
|
||||
for result in generate_subtitles(path_mappings.path_replace_movie(movie['path']),
|
||||
languages,
|
||||
audio_language,
|
||||
str(movie['sceneName']),
|
||||
movie['title'],
|
||||
'movie'):
|
||||
|
||||
result = download_subtitle(path_mappings.path_replace_movie(movie['path']),
|
||||
language.split(':')[0],
|
||||
audio_language,
|
||||
"True" if language.endswith(':hi') else "False",
|
||||
"True" if language.endswith(':forced') else "False",
|
||||
providers_list,
|
||||
providers_auth,
|
||||
str(movie['sceneName']),
|
||||
movie['title'],
|
||||
'movie')
|
||||
if result is not None:
|
||||
message = result[0]
|
||||
path = result[1]
|
||||
forced = result[5]
|
||||
if result[8]:
|
||||
language_code = result[2] + ":hi"
|
||||
elif forced:
|
||||
language_code = result[2] + ":forced"
|
||||
else:
|
||||
language_code = result[2]
|
||||
provider = result[3]
|
||||
score = result[4]
|
||||
subs_id = result[6]
|
||||
subs_path = result[7]
|
||||
store_subtitles_movie(movie['path'], path_mappings.path_replace_movie(movie['path']))
|
||||
history_log_movie(1, no, message, path, language_code, provider, score, subs_id, subs_path)
|
||||
send_notifications_movie(no, message)
|
||||
else:
|
||||
logging.info("BAZARR All providers are throttled")
|
||||
break
|
||||
if result:
|
||||
message = result[0]
|
||||
path = result[1]
|
||||
forced = result[5]
|
||||
if result[8]:
|
||||
language_code = result[2] + ":hi"
|
||||
elif forced:
|
||||
language_code = result[2] + ":forced"
|
||||
else:
|
||||
language_code = result[2]
|
||||
provider = result[3]
|
||||
score = result[4]
|
||||
subs_id = result[6]
|
||||
subs_path = result[7]
|
||||
store_subtitles_movie(movie['path'], path_mappings.path_replace_movie(movie['path']))
|
||||
history_log_movie(1, no, message, path, language_code, provider, score, subs_id, subs_path)
|
||||
send_notifications_movie(no, message)
|
||||
else:
|
||||
logging.info("BAZARR All providers are throttled")
|
||||
|
||||
hide_progress(id='movie_search_progress_{}'.format(no))
|
||||
|
||||
|
||||
def _wanted_episode(episode):
|
||||
audio_language_list = get_audio_profile_languages(episode_id=episode['sonarrEpisodeId'])
|
||||
if len(audio_language_list) > 0:
|
||||
audio_language = audio_language_list[0]['name']
|
||||
else:
|
||||
audio_language = 'None'
|
||||
|
||||
languages = []
|
||||
for language in ast.literal_eval(episode['missing_subtitles']):
|
||||
|
||||
# confirm if language is still missing or if cutoff have been reached
|
||||
confirmed_missing_subs = TableEpisodes.select(TableEpisodes.missing_subtitles) \
|
||||
.where(TableEpisodes.sonarrEpisodeId == episode['sonarrEpisodeId']) \
|
||||
.dicts() \
|
||||
.get()
|
||||
if language not in ast.literal_eval(confirmed_missing_subs['missing_subtitles']):
|
||||
continue
|
||||
|
||||
if is_search_active(desired_language=language, attempt_string=episode['failedAttempts']):
|
||||
TableEpisodes.update({TableEpisodes.failedAttempts:
|
||||
updateFailedAttempts(desired_language=language,
|
||||
attempt_string=episode['failedAttempts'])}) \
|
||||
.where(TableEpisodes.sonarrEpisodeId == episode['sonarrEpisodeId']) \
|
||||
.execute()
|
||||
|
||||
|
||||
hi_ = "True" if language.endswith(':hi') else "False"
|
||||
forced_ ="True" if language.endswith(':forced') else "False"
|
||||
languages.append((language.split(":")[0], hi_, forced_))
|
||||
|
||||
else:
|
||||
logging.debug(
|
||||
f"BAZARR Search is throttled by adaptive search for this episode {episode['path']} and "
|
||||
f"language: {language}")
|
||||
|
||||
for result in generate_subtitles(path_mappings.path_replace(episode['path']),
|
||||
languages,
|
||||
audio_language,
|
||||
str(episode['scene_name']),
|
||||
episode['title'],
|
||||
'series'):
|
||||
if result:
|
||||
message = result[0]
|
||||
path = result[1]
|
||||
forced = result[5]
|
||||
if result[8]:
|
||||
language_code = result[2] + ":hi"
|
||||
elif forced:
|
||||
language_code = result[2] + ":forced"
|
||||
else:
|
||||
language_code = result[2]
|
||||
provider = result[3]
|
||||
score = result[4]
|
||||
subs_id = result[6]
|
||||
subs_path = result[7]
|
||||
store_subtitles(episode['path'], path_mappings.path_replace(episode['path']))
|
||||
history_log(1, episode['sonarrSeriesId'], episode['sonarrEpisodeId'], message, path,
|
||||
language_code, provider, score, subs_id, subs_path)
|
||||
event_stream(type='series', action='update', payload=episode['sonarrSeriesId'])
|
||||
event_stream(type='episode-wanted', action='delete', payload=episode['sonarrEpisodeId'])
|
||||
send_notifications(episode['sonarrSeriesId'], episode['sonarrEpisodeId'], message)
|
||||
|
||||
|
||||
def wanted_download_subtitles(sonarr_episode_id):
|
||||
episodes_details = TableEpisodes.select(TableEpisodes.path,
|
||||
TableEpisodes.missing_subtitles,
|
||||
|
|
@ -1021,73 +1158,76 @@ def wanted_download_subtitles(sonarr_episode_id):
|
|||
.dicts()
|
||||
episodes_details = list(episodes_details)
|
||||
|
||||
providers_auth = get_providers_auth()
|
||||
|
||||
for episode in episodes_details:
|
||||
providers_list = get_providers()
|
||||
|
||||
if providers_list:
|
||||
for language in ast.literal_eval(episode['missing_subtitles']):
|
||||
# confirm if language is still missing or if cutoff have been reached
|
||||
confirmed_missing_subs = TableEpisodes.select(TableEpisodes.missing_subtitles) \
|
||||
.where(TableEpisodes.sonarrEpisodeId == episode['sonarrEpisodeId']) \
|
||||
.dicts() \
|
||||
.get()
|
||||
if language not in ast.literal_eval(confirmed_missing_subs['missing_subtitles']):
|
||||
continue
|
||||
|
||||
if is_search_active(desired_language=language, attempt_string=episode['failedAttempts']):
|
||||
TableEpisodes.update({TableEpisodes.failedAttempts:
|
||||
updateFailedAttempts(desired_language=language,
|
||||
attempt_string=episode['failedAttempts'])}) \
|
||||
.where(TableEpisodes.sonarrEpisodeId == episode['sonarrEpisodeId']) \
|
||||
.execute()
|
||||
|
||||
audio_language_list = get_audio_profile_languages(episode_id=episode['sonarrEpisodeId'])
|
||||
if len(audio_language_list) > 0:
|
||||
audio_language = audio_language_list[0]['name']
|
||||
else:
|
||||
audio_language = 'None'
|
||||
|
||||
result = download_subtitle(path_mappings.path_replace(episode['path']),
|
||||
language.split(':')[0],
|
||||
audio_language,
|
||||
"True" if language.endswith(':hi') else "False",
|
||||
"True" if language.endswith(':forced') else "False",
|
||||
providers_list,
|
||||
providers_auth,
|
||||
str(episode['scene_name']),
|
||||
episode['title'],
|
||||
'series')
|
||||
if result is not None:
|
||||
message = result[0]
|
||||
path = result[1]
|
||||
forced = result[5]
|
||||
if result[8]:
|
||||
language_code = result[2] + ":hi"
|
||||
elif forced:
|
||||
language_code = result[2] + ":forced"
|
||||
else:
|
||||
language_code = result[2]
|
||||
provider = result[3]
|
||||
score = result[4]
|
||||
subs_id = result[6]
|
||||
subs_path = result[7]
|
||||
store_subtitles(episode['path'], path_mappings.path_replace(episode['path']))
|
||||
history_log(1, episode['sonarrSeriesId'], episode['sonarrEpisodeId'], message, path,
|
||||
language_code, provider, score, subs_id, subs_path)
|
||||
event_stream(type='series', action='update', payload=episode['sonarrSeriesId'])
|
||||
event_stream(type='episode-wanted', action='delete', payload=episode['sonarrEpisodeId'])
|
||||
send_notifications(episode['sonarrSeriesId'], episode['sonarrEpisodeId'], message)
|
||||
else:
|
||||
logging.debug(
|
||||
f"BAZARR Search is throttled by adaptive search for this episode {episode['path']} and "
|
||||
f"language: {language}")
|
||||
_wanted_episode(episode)
|
||||
else:
|
||||
logging.info("BAZARR All providers are throttled")
|
||||
break
|
||||
|
||||
|
||||
def _wanted_movie(movie):
|
||||
audio_language_list = get_audio_profile_languages(movie_id=movie['radarrId'])
|
||||
if len(audio_language_list) > 0:
|
||||
audio_language = audio_language_list[0]['name']
|
||||
else:
|
||||
audio_language = 'None'
|
||||
|
||||
languages = []
|
||||
|
||||
for language in ast.literal_eval(movie['missing_subtitles']):
|
||||
# confirm if language is still missing or if cutoff have been reached
|
||||
confirmed_missing_subs = TableMovies.select(TableMovies.missing_subtitles) \
|
||||
.where(TableMovies.radarrId == movie['radarrId']) \
|
||||
.dicts() \
|
||||
.get()
|
||||
if language not in ast.literal_eval(confirmed_missing_subs['missing_subtitles']):
|
||||
continue
|
||||
|
||||
if is_search_active(desired_language=language, attempt_string=movie['failedAttempts']):
|
||||
TableMovies.update({TableMovies.failedAttempts:
|
||||
updateFailedAttempts(desired_language=language,
|
||||
attempt_string=movie['failedAttempts'])}) \
|
||||
.where(TableMovies.radarrId == movie['radarrId']) \
|
||||
.execute()
|
||||
|
||||
hi_ = "True" if language.endswith(':hi') else "False"
|
||||
forced_ ="True" if language.endswith(':forced') else "False"
|
||||
languages.append((language.split(":")[0], hi_, forced_))
|
||||
|
||||
else:
|
||||
logging.info(f"BAZARR Search is throttled by adaptive search for this movie {movie['path']} and "
|
||||
f"language: {language}")
|
||||
|
||||
for result in generate_subtitles(path_mappings.path_replace_movie(movie['path']),
|
||||
languages,
|
||||
audio_language,
|
||||
str(movie['sceneName']),
|
||||
movie['title'], 'movie'):
|
||||
|
||||
if result:
|
||||
message = result[0]
|
||||
path = result[1]
|
||||
forced = result[5]
|
||||
if result[8]:
|
||||
language_code = result[2] + ":hi"
|
||||
elif forced:
|
||||
language_code = result[2] + ":forced"
|
||||
else:
|
||||
language_code = result[2]
|
||||
provider = result[3]
|
||||
score = result[4]
|
||||
subs_id = result[6]
|
||||
subs_path = result[7]
|
||||
store_subtitles_movie(movie['path'], path_mappings.path_replace_movie(movie['path']))
|
||||
history_log_movie(1, movie['radarrId'], message, path, language_code, provider, score,
|
||||
subs_id, subs_path)
|
||||
event_stream(type='movie-wanted', action='delete', payload=movie['radarrId'])
|
||||
send_notifications_movie(movie['radarrId'], message)
|
||||
|
||||
|
||||
def wanted_download_subtitles_movie(radarr_id):
|
||||
movies_details = TableMovies.select(TableMovies.path,
|
||||
TableMovies.missing_subtitles,
|
||||
|
|
@ -1100,66 +1240,11 @@ def wanted_download_subtitles_movie(radarr_id):
|
|||
.dicts()
|
||||
movies_details = list(movies_details)
|
||||
|
||||
providers_auth = get_providers_auth()
|
||||
|
||||
for movie in movies_details:
|
||||
providers_list = get_providers()
|
||||
|
||||
if providers_list:
|
||||
for language in ast.literal_eval(movie['missing_subtitles']):
|
||||
# confirm if language is still missing or if cutoff have been reached
|
||||
confirmed_missing_subs = TableMovies.select(TableMovies.missing_subtitles) \
|
||||
.where(TableMovies.radarrId == movie['radarrId']) \
|
||||
.dicts() \
|
||||
.get()
|
||||
if language not in ast.literal_eval(confirmed_missing_subs['missing_subtitles']):
|
||||
continue
|
||||
|
||||
if is_search_active(desired_language=language, attempt_string=movie['failedAttempts']):
|
||||
TableMovies.update({TableMovies.failedAttempts:
|
||||
updateFailedAttempts(desired_language=language,
|
||||
attempt_string=movie['failedAttempts'])}) \
|
||||
.where(TableMovies.radarrId == movie['radarrId']) \
|
||||
.execute()
|
||||
|
||||
audio_language_list = get_audio_profile_languages(movie_id=movie['radarrId'])
|
||||
if len(audio_language_list) > 0:
|
||||
audio_language = audio_language_list[0]['name']
|
||||
else:
|
||||
audio_language = 'None'
|
||||
|
||||
result = download_subtitle(path_mappings.path_replace_movie(movie['path']),
|
||||
language.split(':')[0],
|
||||
audio_language,
|
||||
"True" if language.endswith(':hi') else "False",
|
||||
"True" if language.endswith(':forced') else "False",
|
||||
providers_list,
|
||||
providers_auth,
|
||||
str(movie['sceneName']),
|
||||
movie['title'],
|
||||
'movie')
|
||||
if result is not None:
|
||||
message = result[0]
|
||||
path = result[1]
|
||||
forced = result[5]
|
||||
if result[8]:
|
||||
language_code = result[2] + ":hi"
|
||||
elif forced:
|
||||
language_code = result[2] + ":forced"
|
||||
else:
|
||||
language_code = result[2]
|
||||
provider = result[3]
|
||||
score = result[4]
|
||||
subs_id = result[6]
|
||||
subs_path = result[7]
|
||||
store_subtitles_movie(movie['path'], path_mappings.path_replace_movie(movie['path']))
|
||||
history_log_movie(1, movie['radarrId'], message, path, language_code, provider, score,
|
||||
subs_id, subs_path)
|
||||
event_stream(type='movie-wanted', action='delete', payload=movie['radarrId'])
|
||||
send_notifications_movie(movie['radarrId'], message)
|
||||
else:
|
||||
logging.info(f"BAZARR Search is throttled by adaptive search for this movie {movie['path']} and "
|
||||
f"language: {language}")
|
||||
_wanted_movie(movie)
|
||||
else:
|
||||
logging.info("BAZARR All providers are throttled")
|
||||
break
|
||||
|
|
@ -1471,8 +1556,6 @@ def upgrade_subtitles():
|
|||
|
||||
count_movie_to_upgrade = len(movies_to_upgrade)
|
||||
|
||||
providers_auth = get_providers_auth()
|
||||
|
||||
if settings.general.getboolean('use_sonarr'):
|
||||
for i, episode in enumerate(episodes_to_upgrade):
|
||||
providers_list = get_providers()
|
||||
|
|
@ -1508,19 +1591,17 @@ def upgrade_subtitles():
|
|||
else:
|
||||
audio_language = 'None'
|
||||
|
||||
result = download_subtitle(path_mappings.path_replace(episode['video_path']),
|
||||
language,
|
||||
result = list(generate_subtitles(path_mappings.path_replace(episode['video_path']),
|
||||
[(language, is_hi, is_forced)],
|
||||
audio_language,
|
||||
is_hi,
|
||||
is_forced,
|
||||
providers_list,
|
||||
providers_auth,
|
||||
str(episode['scene_name']),
|
||||
episode['title'],
|
||||
'series',
|
||||
forced_minimum_score=int(episode['score']),
|
||||
is_upgrade=True)
|
||||
if result is not None:
|
||||
is_upgrade=True))
|
||||
|
||||
if result:
|
||||
result = result[0]
|
||||
message = result[0]
|
||||
path = result[1]
|
||||
forced = result[5]
|
||||
|
|
@ -1573,19 +1654,16 @@ def upgrade_subtitles():
|
|||
else:
|
||||
audio_language = 'None'
|
||||
|
||||
result = download_subtitle(path_mappings.path_replace_movie(movie['video_path']),
|
||||
language,
|
||||
result = list(generate_subtitles(path_mappings.path_replace_movie(movie['video_path']),
|
||||
[(language, is_hi, is_forced)],
|
||||
audio_language,
|
||||
is_hi,
|
||||
is_forced,
|
||||
providers_list,
|
||||
providers_auth,
|
||||
str(movie['sceneName']),
|
||||
movie['title'],
|
||||
'movie',
|
||||
forced_minimum_score=int(movie['score']),
|
||||
is_upgrade=True)
|
||||
if result is not None:
|
||||
is_upgrade=True))
|
||||
if result:
|
||||
result = result[0]
|
||||
message = result[0]
|
||||
path = result[1]
|
||||
forced = result[5]
|
||||
|
|
|
|||
|
|
@ -161,5 +161,8 @@ class Provider(object):
|
|||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def ping(self):
|
||||
return True
|
||||
|
||||
def __repr__(self):
|
||||
return '<%s [%r]>' % (self.__class__.__name__, self.video_types)
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import json
|
|||
import re
|
||||
import os
|
||||
import logging
|
||||
import datetime
|
||||
import socket
|
||||
import traceback
|
||||
import time
|
||||
|
|
@ -55,6 +56,8 @@ REMOVE_CRAP_FROM_FILENAME = re.compile(r"(?i)(?:([\s_-]+(?:obfuscated|scrambled|
|
|||
|
||||
SUBTITLE_EXTENSIONS = ('.srt', '.sub', '.smi', '.txt', '.ssa', '.ass', '.mpl', '.vtt')
|
||||
|
||||
_POOL_LIFETIME = datetime.timedelta(hours=12)
|
||||
|
||||
|
||||
def remove_crap_from_fn(fn):
|
||||
# in case of the second regex part, the legit release group name will be in group(2), if it's followed by [string]
|
||||
|
|
@ -69,7 +72,7 @@ class SZProviderPool(ProviderPool):
|
|||
def __init__(self, providers=None, provider_configs=None, blacklist=None, ban_list=None, throttle_callback=None,
|
||||
pre_download_hook=None, post_download_hook=None, language_hook=None):
|
||||
#: Name of providers to use
|
||||
self.providers = providers
|
||||
self.providers = set(providers or [])
|
||||
|
||||
#: Provider configuration
|
||||
self.provider_configs = provider_configs or {}
|
||||
|
|
@ -91,9 +94,69 @@ class SZProviderPool(ProviderPool):
|
|||
self.post_download_hook = post_download_hook
|
||||
self.language_hook = language_hook
|
||||
|
||||
self._born = time.time()
|
||||
|
||||
if not self.throttle_callback:
|
||||
self.throttle_callback = lambda x, y: x
|
||||
|
||||
def update(self, providers, provider_configs, blacklist, ban_list):
|
||||
# Check if the pool was initialized enough hours ago
|
||||
self._check_lifetime()
|
||||
|
||||
# Check if any new provider has been added
|
||||
updated = set(providers) != self.providers or ban_list != self.ban_list
|
||||
removed_providers = list(sorted(self.providers - set(providers)))
|
||||
new_providers = list(sorted(set(providers) - self.providers))
|
||||
|
||||
# Terminate and delete removed providers from instance
|
||||
for removed in removed_providers:
|
||||
try:
|
||||
del self[removed]
|
||||
# If the user has updated the providers but hasn't made any
|
||||
# subtitle searches yet, the removed provider won't be in the
|
||||
# self dictionary
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
if updated:
|
||||
logger.debug("Removed providers: %s", removed_providers)
|
||||
logger.debug("New providers: %s", new_providers)
|
||||
|
||||
self.discarded_providers.difference_update(new_providers)
|
||||
self.providers.difference_update(removed_providers)
|
||||
self.providers.update(list(providers))
|
||||
|
||||
self.blacklist = blacklist
|
||||
|
||||
# Restart providers with new configs
|
||||
for key, val in provider_configs.items():
|
||||
# key: provider's name; val: config dict
|
||||
old_val = self.provider_configs.get(key)
|
||||
|
||||
if old_val == val:
|
||||
continue
|
||||
|
||||
logger.debug("Restarting provider: %s", key)
|
||||
try:
|
||||
provider = provider_registry[key](**val)
|
||||
provider.initialize()
|
||||
except Exception as error:
|
||||
self.throttle_callback(key, error)
|
||||
else:
|
||||
self.initialized_providers[key] = provider
|
||||
updated = True
|
||||
|
||||
self.provider_configs = provider_configs
|
||||
|
||||
return updated
|
||||
|
||||
def _check_lifetime(self):
|
||||
# This method is used to avoid possible memory leaks
|
||||
if abs(self._born - time.time()) > _POOL_LIFETIME.seconds:
|
||||
logger.info("%s elapsed. Terminating providers", _POOL_LIFETIME)
|
||||
self._born = time.time()
|
||||
self.terminate()
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
|
|
@ -170,17 +233,7 @@ class SZProviderPool(ProviderPool):
|
|||
logger.info('Listing subtitles with provider %r and languages %r', provider, provider_languages)
|
||||
results = []
|
||||
try:
|
||||
try:
|
||||
results = self[provider].list_subtitles(video, provider_languages)
|
||||
except ResponseNotReady:
|
||||
logger.error('Provider %r response error, reinitializing', provider)
|
||||
try:
|
||||
self[provider].terminate()
|
||||
self[provider].initialize()
|
||||
results = self[provider].list_subtitles(video, provider_languages)
|
||||
except:
|
||||
logger.error('Provider %r reinitialization error: %s', provider, traceback.format_exc())
|
||||
|
||||
results = self[provider].list_subtitles(video, provider_languages)
|
||||
seen = []
|
||||
out = []
|
||||
for s in results:
|
||||
|
|
@ -198,16 +251,13 @@ class SZProviderPool(ProviderPool):
|
|||
continue
|
||||
if s.id in seen:
|
||||
continue
|
||||
|
||||
s.plex_media_fps = float(video.fps) if video.fps else None
|
||||
out.append(s)
|
||||
seen.append(s.id)
|
||||
|
||||
return out
|
||||
|
||||
except (requests.Timeout, socket.timeout) as e:
|
||||
logger.error('Provider %r timed out', provider)
|
||||
self.throttle_callback(provider, e)
|
||||
|
||||
except Exception as e:
|
||||
logger.exception('Unexpected error in provider %r: %s', provider, traceback.format_exc())
|
||||
self.throttle_callback(provider, e)
|
||||
|
|
@ -289,16 +339,6 @@ class SZProviderPool(ProviderPool):
|
|||
logger.error('Provider %r connection error', subtitle.provider_name)
|
||||
self.throttle_callback(subtitle.provider_name, e)
|
||||
|
||||
except ResponseNotReady as e:
|
||||
logger.error('Provider %r response error, reinitializing', subtitle.provider_name)
|
||||
try:
|
||||
self[subtitle.provider_name].terminate()
|
||||
self[subtitle.provider_name].initialize()
|
||||
except:
|
||||
logger.error('Provider %r reinitialization error: %s', subtitle.provider_name,
|
||||
traceback.format_exc())
|
||||
self.throttle_callback(subtitle.provider_name, e)
|
||||
|
||||
except rarfile.BadRarFile:
|
||||
logger.error('Malformed RAR file from provider %r, skipping subtitle.', subtitle.provider_name)
|
||||
logger.debug("RAR Traceback: %s", traceback.format_exc())
|
||||
|
|
|
|||
92
libs/subliminal_patch/core_persistent.py
Normal file
92
libs/subliminal_patch/core_persistent.py
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
# coding=utf-8
|
||||
from __future__ import absolute_import
|
||||
|
||||
from collections import defaultdict
|
||||
import logging
|
||||
import time
|
||||
|
||||
from subliminal.core import check_video
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# list_all_subtitles, list_supported_languages, list_supported_video_types, download_subtitles, download_best_subtitles
|
||||
def list_all_subtitles(videos, languages, pool_instance):
|
||||
listed_subtitles = defaultdict(list)
|
||||
|
||||
# return immediatly if no video passed the checks
|
||||
if not videos:
|
||||
return listed_subtitles
|
||||
|
||||
for video in videos:
|
||||
logger.info("Listing subtitles for %r", video)
|
||||
subtitles = pool_instance.list_subtitles(
|
||||
video, languages - video.subtitle_languages
|
||||
)
|
||||
listed_subtitles[video].extend(subtitles)
|
||||
logger.info("Found %d subtitle(s)", len(subtitles))
|
||||
|
||||
return listed_subtitles
|
||||
|
||||
|
||||
def list_supported_languages(pool_instance):
|
||||
return pool_instance.list_supported_languages()
|
||||
|
||||
|
||||
def list_supported_video_types(pool_instance):
|
||||
return pool_instance.list_supported_video_types()
|
||||
|
||||
|
||||
def download_subtitles(subtitles, pool_instance):
|
||||
for subtitle in subtitles:
|
||||
logger.info("Downloading subtitle %r with score %s", subtitle, subtitle.score)
|
||||
pool_instance.download_subtitle(subtitle)
|
||||
|
||||
|
||||
def download_best_subtitles(
|
||||
videos,
|
||||
languages,
|
||||
pool_instance,
|
||||
min_score=0,
|
||||
hearing_impaired=False,
|
||||
only_one=False,
|
||||
compute_score=None,
|
||||
throttle_time=0,
|
||||
score_obj=None,
|
||||
):
|
||||
downloaded_subtitles = defaultdict(list)
|
||||
|
||||
# check videos
|
||||
checked_videos = []
|
||||
for video in videos:
|
||||
if not check_video(video, languages=languages, undefined=only_one):
|
||||
logger.info("Skipping video %r", video)
|
||||
continue
|
||||
checked_videos.append(video)
|
||||
|
||||
# return immediately if no video passed the checks
|
||||
if not checked_videos:
|
||||
return downloaded_subtitles
|
||||
|
||||
got_multiple = len(checked_videos) > 1
|
||||
|
||||
# download best subtitles
|
||||
for video in checked_videos:
|
||||
logger.info("Downloading best subtitles for %r", video)
|
||||
subtitles = pool_instance.download_best_subtitles(
|
||||
pool_instance.list_subtitles(video, languages - video.subtitle_languages),
|
||||
video,
|
||||
languages,
|
||||
min_score=min_score,
|
||||
hearing_impaired=hearing_impaired,
|
||||
only_one=only_one,
|
||||
compute_score=compute_score,
|
||||
score_obj=score_obj,
|
||||
)
|
||||
logger.info("Downloaded %d subtitle(s)", len(subtitles))
|
||||
downloaded_subtitles[video].extend(subtitles)
|
||||
|
||||
if got_multiple and throttle_time:
|
||||
logger.debug("Waiting %ss before continuing ...", throttle_time)
|
||||
time.sleep(throttle_time)
|
||||
|
||||
return downloaded_subtitles
|
||||
|
|
@ -1,8 +1,11 @@
|
|||
# coding=utf-8
|
||||
|
||||
from __future__ import absolute_import
|
||||
|
||||
import functools
|
||||
import importlib
|
||||
import os
|
||||
import logging
|
||||
import subliminal
|
||||
from subliminal.providers import Provider as _Provider
|
||||
from subliminal.subtitle import Subtitle as _Subtitle
|
||||
|
|
@ -14,11 +17,50 @@ from subzero.lib.io import get_viable_encoding
|
|||
import six
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Provider(_Provider):
|
||||
hash_verifiable = False
|
||||
hearing_impaired_verifiable = False
|
||||
skip_wrong_fps = True
|
||||
|
||||
def ping(self):
|
||||
"""Check if the provider is alive."""
|
||||
return True
|
||||
|
||||
|
||||
def reinitialize_on_error(exceptions: tuple, attempts=1):
|
||||
"""Method decorator for Provider class. It will reinitialize the instance
|
||||
and re-run the method in case of exceptions.
|
||||
|
||||
:param exceptions: tuple of expected exceptions
|
||||
:param attempts: number of attempts to call the method
|
||||
"""
|
||||
|
||||
def real_decorator(method):
|
||||
@functools.wraps(method)
|
||||
def wrapper(self, *args, **kwargs):
|
||||
inc = 1
|
||||
while True:
|
||||
try:
|
||||
return method(self, *args, **kwargs)
|
||||
except exceptions as error:
|
||||
if inc > attempts:
|
||||
raise
|
||||
|
||||
logger.exception(error)
|
||||
logger.debug("Reinitializing %s instance (%s attempt)", self, inc)
|
||||
|
||||
self.terminate()
|
||||
self.initialize()
|
||||
|
||||
inc += 1
|
||||
|
||||
return wrapper
|
||||
|
||||
return real_decorator
|
||||
|
||||
|
||||
# register providers
|
||||
# fixme: this is bad
|
||||
|
|
|
|||
|
|
@ -11,12 +11,14 @@ from urllib.parse import quote_plus
|
|||
import babelfish
|
||||
from dogpile.cache.api import NO_VALUE
|
||||
from requests import Session
|
||||
from requests.exceptions import RequestException
|
||||
from subliminal.cache import region
|
||||
from subliminal.video import Episode, Movie
|
||||
from subliminal.exceptions import DownloadLimitExceeded, AuthenticationError, ConfigurationError
|
||||
from subliminal.providers.addic7ed import Addic7edProvider as _Addic7edProvider, \
|
||||
Addic7edSubtitle as _Addic7edSubtitle, ParserBeautifulSoup
|
||||
from subliminal.subtitle import fix_line_ending
|
||||
from subliminal_patch.providers import reinitialize_on_error
|
||||
from subliminal_patch.utils import sanitize
|
||||
from subliminal_patch.exceptions import TooManyRequests
|
||||
from subliminal_patch.pitcher import pitchers, load_verification, store_verification
|
||||
|
|
@ -550,6 +552,7 @@ class Addic7edProvider(_Addic7edProvider):
|
|||
|
||||
return subtitles
|
||||
|
||||
@reinitialize_on_error((RequestException,), attempts=1)
|
||||
def list_subtitles(self, video, languages):
|
||||
if isinstance(video, Episode):
|
||||
# lookup show_id
|
||||
|
|
@ -586,6 +589,7 @@ class Addic7edProvider(_Addic7edProvider):
|
|||
|
||||
return []
|
||||
|
||||
@reinitialize_on_error((RequestException,), attempts=1)
|
||||
def download_subtitle(self, subtitle):
|
||||
last_dls = region.get("addic7ed_dls")
|
||||
now = datetime.datetime.now()
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ from requests.exceptions import HTTPError
|
|||
import rarfile
|
||||
|
||||
from guessit import guessit
|
||||
from requests.exceptions import RequestException
|
||||
from subliminal.cache import region
|
||||
from subliminal.exceptions import ConfigurationError, AuthenticationError, ServiceUnavailable, DownloadLimitExceeded
|
||||
from subliminal.providers import ParserBeautifulSoup
|
||||
|
|
@ -18,7 +19,7 @@ from subliminal.utils import sanitize, sanitize_release_group
|
|||
from subliminal.video import Episode, Movie
|
||||
from subliminal_patch.exceptions import TooManyRequests, IPAddressBlocked
|
||||
from subliminal_patch.http import RetryingCFSession
|
||||
from subliminal_patch.providers import Provider
|
||||
from subliminal_patch.providers import Provider, reinitialize_on_error
|
||||
from subliminal_patch.score import get_scores, framerate_equal
|
||||
from subliminal_patch.subtitle import Subtitle, guess_matches
|
||||
from subzero.language import Language
|
||||
|
|
@ -260,6 +261,7 @@ class LegendasdivxProvider(Provider):
|
|||
)
|
||||
return subtitles
|
||||
|
||||
@reinitialize_on_error((RequestException,), attempts=1)
|
||||
def query(self, video, languages):
|
||||
|
||||
_searchurl = self.searchurl
|
||||
|
|
@ -362,7 +364,8 @@ class LegendasdivxProvider(Provider):
|
|||
|
||||
def list_subtitles(self, video, languages):
|
||||
return self.query(video, languages)
|
||||
|
||||
|
||||
@reinitialize_on_error((RequestException,), attempts=1)
|
||||
def download_subtitle(self, subtitle):
|
||||
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -8,8 +8,10 @@ from subliminal.exceptions import ConfigurationError
|
|||
from subliminal.providers.legendastv import LegendasTVSubtitle as _LegendasTVSubtitle, \
|
||||
LegendasTVProvider as _LegendasTVProvider, Episode, Movie, guessit, sanitize, region, type_map, \
|
||||
raise_for_status, json, SHOW_EXPIRATION_TIME, title_re, season_re, datetime, pytz, NO_VALUE, releases_key, \
|
||||
SUBTITLE_EXTENSIONS, language_converters
|
||||
SUBTITLE_EXTENSIONS, language_converters, ServiceUnavailable
|
||||
|
||||
from requests.exceptions import RequestException
|
||||
from subliminal_patch.providers import reinitialize_on_error
|
||||
from subliminal_patch.subtitle import guess_matches
|
||||
from subzero.language import Language
|
||||
|
||||
|
|
@ -184,6 +186,7 @@ class LegendasTVProvider(_LegendasTVProvider):
|
|||
|
||||
return titles_found
|
||||
|
||||
@reinitialize_on_error((RequestException, ServiceUnavailable), attempts=1)
|
||||
def query(self, language, titles, season=None, episode=None, year=None, imdb_id=None):
|
||||
# search for titles
|
||||
titles_found = self.search_titles(titles, season, year, imdb_id)
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ from subliminal.providers.opensubtitles import OpenSubtitlesProvider as _OpenSub
|
|||
DownloadLimitReached, InvalidImdbid, UnknownUserAgent, DisabledUserAgent, OpenSubtitlesError
|
||||
from .mixins import ProviderRetryMixin
|
||||
from subliminal.subtitle import fix_line_ending
|
||||
from subliminal_patch.providers import reinitialize_on_error
|
||||
from subliminal_patch.http import SubZeroRequestsTransport
|
||||
from subliminal_patch.utils import sanitize, fix_inconsistent_naming
|
||||
from subliminal.cache import region
|
||||
|
|
@ -236,7 +237,7 @@ class OpenSubtitlesProvider(ProviderRetryMixin, _OpenSubtitlesProvider):
|
|||
def terminate(self):
|
||||
self.server = None
|
||||
self.token = None
|
||||
|
||||
|
||||
def list_subtitles(self, video, languages):
|
||||
"""
|
||||
:param video:
|
||||
|
|
@ -272,6 +273,7 @@ class OpenSubtitlesProvider(ProviderRetryMixin, _OpenSubtitlesProvider):
|
|||
use_tag_search=self.use_tag_search, only_foreign=self.only_foreign,
|
||||
also_foreign=self.also_foreign)
|
||||
|
||||
@reinitialize_on_error((NoSession, Unauthorized, OpenSubtitlesError, ServiceUnavailable), attempts=1)
|
||||
def query(self, video, languages, hash=None, size=None, imdb_id=None, query=None, season=None, episode=None,
|
||||
tag=None, use_tag_search=False, only_foreign=False, also_foreign=False):
|
||||
# fill the search criteria
|
||||
|
|
@ -377,6 +379,7 @@ class OpenSubtitlesProvider(ProviderRetryMixin, _OpenSubtitlesProvider):
|
|||
|
||||
return subtitles
|
||||
|
||||
@reinitialize_on_error((NoSession, Unauthorized, OpenSubtitlesError, ServiceUnavailable), attempts=1)
|
||||
def download_subtitle(self, subtitle):
|
||||
logger.info('Downloading subtitle %r', subtitle)
|
||||
response = self.use_token_or_login(
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
import datetime
|
||||
|
||||
from requests import Session, ConnectionError, Timeout, ReadTimeout
|
||||
|
|
@ -147,15 +148,18 @@ class OpenSubtitlesComProvider(ProviderRetryMixin, Provider):
|
|||
self.password = password
|
||||
self.video = None
|
||||
self.use_hash = use_hash
|
||||
self._started = None
|
||||
|
||||
def initialize(self):
|
||||
self.token = region.get("oscom_token", expiration_time=TOKEN_EXPIRATION_TIME)
|
||||
if self.token is NO_VALUE:
|
||||
self.login()
|
||||
self._started = time.time()
|
||||
self.login()
|
||||
|
||||
def terminate(self):
|
||||
self.session.close()
|
||||
|
||||
def ping(self):
|
||||
return self._started and (time.time() - self._started) < TOKEN_EXPIRATION_TIME
|
||||
|
||||
def login(self):
|
||||
try:
|
||||
r = self.session.post(self.server_url + 'login',
|
||||
|
|
|
|||
|
|
@ -20,13 +20,14 @@ import rarfile
|
|||
from babelfish import language_converters
|
||||
from guessit import guessit
|
||||
from dogpile.cache.api import NO_VALUE
|
||||
from requests.exceptions import RequestException
|
||||
from subliminal import Episode, ProviderError
|
||||
from subliminal.video import Episode, Movie
|
||||
from subliminal.exceptions import ConfigurationError, ServiceUnavailable
|
||||
from subliminal.utils import sanitize_release_group
|
||||
from subliminal.cache import region
|
||||
from subliminal_patch.http import RetryingCFSession
|
||||
from subliminal_patch.providers import Provider
|
||||
from subliminal_patch.providers import Provider, reinitialize_on_error
|
||||
from subliminal_patch.providers.mixins import ProviderSubtitleArchiveMixin
|
||||
from subliminal_patch.subtitle import Subtitle, guess_matches
|
||||
from subliminal_patch.converters.subscene import language_ids, supported_languages
|
||||
|
|
@ -315,7 +316,9 @@ class SubsceneProvider(Provider, ProviderSubtitleArchiveMixin):
|
|||
return search(*args, **kwargs)
|
||||
except requests.HTTPError:
|
||||
region.delete("subscene_cookies2")
|
||||
raise
|
||||
|
||||
@reinitialize_on_error((RequestException,), attempts=1)
|
||||
def query(self, video):
|
||||
subtitles = []
|
||||
if isinstance(video, Episode):
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import re
|
|||
from subzero.language import Language
|
||||
from guessit import guessit
|
||||
from requests import Session
|
||||
from requests.exceptions import RequestException
|
||||
|
||||
from subliminal.providers import ParserBeautifulSoup, Provider
|
||||
from subliminal import __short_version__
|
||||
|
|
@ -16,6 +17,7 @@ from subliminal.subtitle import Subtitle, fix_line_ending
|
|||
from subliminal.utils import sanitize, sanitize_release_group
|
||||
from subliminal.video import Episode
|
||||
from subliminal_patch.subtitle import guess_matches
|
||||
from subliminal_patch.providers import reinitialize_on_error
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
article_re = re.compile(r'^([A-Za-z]{1,3}) (.*)$')
|
||||
|
|
@ -189,7 +191,8 @@ class XSubsProvider(Provider):
|
|||
break
|
||||
|
||||
return int(show_id) if show_id else None
|
||||
|
||||
|
||||
@reinitialize_on_error((RequestException,), attempts=1)
|
||||
def query(self, show_id, series, season, year=None, country=None):
|
||||
# get the season list of the show
|
||||
logger.info('Getting the season list of show id %d', show_id)
|
||||
|
|
@ -291,6 +294,7 @@ class XSubsProvider(Provider):
|
|||
|
||||
return []
|
||||
|
||||
@reinitialize_on_error((RequestException,), attempts=1)
|
||||
def download_subtitle(self, subtitle):
|
||||
if isinstance(subtitle, XSubsSubtitle):
|
||||
# download the subtitle
|
||||
|
|
|
|||
Loading…
Reference in a new issue