mirror of
https://github.com/morpheus65535/bazarr.git
synced 2024-09-20 15:35:58 +08:00
Improved providers throttling and prevent hammering providers by updating throttled providers on each iteration of the loop.
This commit is contained in:
parent
5d1dccde99
commit
5ff3fe4684
|
@ -8,7 +8,6 @@ import pretty
|
||||||
import time
|
import time
|
||||||
import socket
|
import socket
|
||||||
import requests
|
import requests
|
||||||
import ast
|
|
||||||
|
|
||||||
from get_args import args
|
from get_args import args
|
||||||
from config import settings, get_array_from
|
from config import settings, get_array_from
|
||||||
|
@ -328,9 +327,11 @@ def get_throttled_providers():
|
||||||
if os.path.exists(os.path.join(args.config_dir, 'config', 'throttled_providers.dat')):
|
if os.path.exists(os.path.join(args.config_dir, 'config', 'throttled_providers.dat')):
|
||||||
with open(os.path.normpath(os.path.join(args.config_dir, 'config', 'throttled_providers.dat')), 'r') as \
|
with open(os.path.normpath(os.path.join(args.config_dir, 'config', 'throttled_providers.dat')), 'r') as \
|
||||||
handle:
|
handle:
|
||||||
providers = ast.literal_eval(handle.read())
|
providers = eval(handle.read())
|
||||||
except (OSError, ValueError):
|
except:
|
||||||
providers = {}
|
# set empty content in throttled_providers.dat
|
||||||
|
logging.error("Invalid content in throttled_providers.dat. Resetting")
|
||||||
|
set_throttled_providers(providers)
|
||||||
finally:
|
finally:
|
||||||
return providers
|
return providers
|
||||||
|
|
||||||
|
@ -340,12 +341,6 @@ def set_throttled_providers(data):
|
||||||
handle.write(data)
|
handle.write(data)
|
||||||
|
|
||||||
|
|
||||||
try:
|
tp = get_throttled_providers()
|
||||||
tp = get_throttled_providers()
|
if not isinstance(tp, dict):
|
||||||
if not isinstance(tp, dict):
|
raise ValueError('tp should be a dict')
|
||||||
raise ValueError('tp should be a dict')
|
|
||||||
except Exception:
|
|
||||||
logging.error("Invalid content in throttled_providers.dat. Resetting")
|
|
||||||
# set empty content in throttled_providers.dat
|
|
||||||
set_throttled_providers('')
|
|
||||||
tp = get_throttled_providers()
|
|
||||||
|
|
|
@ -765,12 +765,13 @@ def series_download_subtitles(no):
|
||||||
"ignored because of monitored status, series type or series tags: {}".format(no))
|
"ignored because of monitored status, series type or series tags: {}".format(no))
|
||||||
return
|
return
|
||||||
|
|
||||||
providers_list = get_providers()
|
|
||||||
providers_auth = get_providers_auth()
|
providers_auth = get_providers_auth()
|
||||||
|
|
||||||
count_episodes_details = len(episodes_details)
|
count_episodes_details = len(episodes_details)
|
||||||
|
|
||||||
for i, episode in enumerate(episodes_details):
|
for i, episode in enumerate(episodes_details):
|
||||||
|
providers_list = get_providers()
|
||||||
|
|
||||||
if providers_list:
|
if providers_list:
|
||||||
show_progress(id='series_search_progress_{}'.format(no),
|
show_progress(id='series_search_progress_{}'.format(no),
|
||||||
header='Searching missing subtitles...',
|
header='Searching missing subtitles...',
|
||||||
|
@ -855,10 +856,11 @@ def episode_download_subtitles(no, send_progress=False):
|
||||||
logging.debug("BAZARR no episode with that sonarrEpisodeId can be found in database:", str(no))
|
logging.debug("BAZARR no episode with that sonarrEpisodeId can be found in database:", str(no))
|
||||||
return
|
return
|
||||||
|
|
||||||
providers_list = get_providers()
|
|
||||||
providers_auth = get_providers_auth()
|
providers_auth = get_providers_auth()
|
||||||
|
|
||||||
for episode in episodes_details:
|
for episode in episodes_details:
|
||||||
|
providers_list = get_providers()
|
||||||
|
|
||||||
if providers_list:
|
if providers_list:
|
||||||
if send_progress:
|
if send_progress:
|
||||||
show_progress(id='episode_search_progress_{}'.format(no),
|
show_progress(id='episode_search_progress_{}'.format(no),
|
||||||
|
@ -939,7 +941,6 @@ def movies_download_subtitles(no):
|
||||||
else:
|
else:
|
||||||
movie = movies[0]
|
movie = movies[0]
|
||||||
|
|
||||||
providers_list = get_providers()
|
|
||||||
providers_auth = get_providers_auth()
|
providers_auth = get_providers_auth()
|
||||||
|
|
||||||
if ast.literal_eval(movie['missing_subtitles']):
|
if ast.literal_eval(movie['missing_subtitles']):
|
||||||
|
@ -948,15 +949,17 @@ def movies_download_subtitles(no):
|
||||||
count_movie = 0
|
count_movie = 0
|
||||||
|
|
||||||
for i, language in enumerate(ast.literal_eval(movie['missing_subtitles'])):
|
for i, language in enumerate(ast.literal_eval(movie['missing_subtitles'])):
|
||||||
# confirm if language is still missing or if cutoff have been reached
|
providers_list = get_providers()
|
||||||
confirmed_missing_subs = TableMovies.select(TableMovies.missing_subtitles)\
|
|
||||||
.where(TableMovies.radarrId == movie['radarrId'])\
|
|
||||||
.dicts()\
|
|
||||||
.get()
|
|
||||||
if language not in ast.literal_eval(confirmed_missing_subs['missing_subtitles']):
|
|
||||||
continue
|
|
||||||
|
|
||||||
if providers_list:
|
if providers_list:
|
||||||
|
# confirm if language is still missing or if cutoff have been reached
|
||||||
|
confirmed_missing_subs = TableMovies.select(TableMovies.missing_subtitles) \
|
||||||
|
.where(TableMovies.radarrId == movie['radarrId']) \
|
||||||
|
.dicts() \
|
||||||
|
.get()
|
||||||
|
if language not in ast.literal_eval(confirmed_missing_subs['missing_subtitles']):
|
||||||
|
continue
|
||||||
|
|
||||||
show_progress(id='movie_search_progress_{}'.format(no),
|
show_progress(id='movie_search_progress_{}'.format(no),
|
||||||
header='Searching missing subtitles...',
|
header='Searching missing subtitles...',
|
||||||
name=movie['title'],
|
name=movie['title'],
|
||||||
|
@ -1018,77 +1021,82 @@ def wanted_download_subtitles(sonarr_episode_id):
|
||||||
.dicts()
|
.dicts()
|
||||||
episodes_details = list(episodes_details)
|
episodes_details = list(episodes_details)
|
||||||
|
|
||||||
providers_list = get_providers()
|
|
||||||
providers_auth = get_providers_auth()
|
providers_auth = get_providers_auth()
|
||||||
|
|
||||||
for episode in episodes_details:
|
for episode in episodes_details:
|
||||||
attempt = episode['failedAttempts']
|
providers_list = get_providers()
|
||||||
if type(attempt) == str:
|
|
||||||
attempt = ast.literal_eval(attempt)
|
|
||||||
for language in ast.literal_eval(episode['missing_subtitles']):
|
|
||||||
# confirm if language is still missing or if cutoff have been reached
|
|
||||||
confirmed_missing_subs = TableEpisodes.select(TableEpisodes.missing_subtitles) \
|
|
||||||
.where(TableEpisodes.sonarrEpisodeId == episode['sonarrEpisodeId']) \
|
|
||||||
.dicts() \
|
|
||||||
.get()
|
|
||||||
if language not in ast.literal_eval(confirmed_missing_subs['missing_subtitles']):
|
|
||||||
continue
|
|
||||||
|
|
||||||
if attempt is None:
|
if providers_list:
|
||||||
attempt = []
|
attempt = episode['failedAttempts']
|
||||||
attempt.append([language, time.time()])
|
if type(attempt) == str:
|
||||||
else:
|
attempt = ast.literal_eval(attempt)
|
||||||
att = list(zip(*attempt))[0]
|
for language in ast.literal_eval(episode['missing_subtitles']):
|
||||||
if language not in att:
|
# confirm if language is still missing or if cutoff have been reached
|
||||||
|
confirmed_missing_subs = TableEpisodes.select(TableEpisodes.missing_subtitles) \
|
||||||
|
.where(TableEpisodes.sonarrEpisodeId == episode['sonarrEpisodeId']) \
|
||||||
|
.dicts() \
|
||||||
|
.get()
|
||||||
|
if language not in ast.literal_eval(confirmed_missing_subs['missing_subtitles']):
|
||||||
|
continue
|
||||||
|
|
||||||
|
if attempt is None:
|
||||||
|
attempt = []
|
||||||
attempt.append([language, time.time()])
|
attempt.append([language, time.time()])
|
||||||
|
else:
|
||||||
|
att = list(zip(*attempt))[0]
|
||||||
|
if language not in att:
|
||||||
|
attempt.append([language, time.time()])
|
||||||
|
|
||||||
TableEpisodes.update({TableEpisodes.failedAttempts: str(attempt)})\
|
TableEpisodes.update({TableEpisodes.failedAttempts: str(attempt)})\
|
||||||
.where(TableEpisodes.sonarrEpisodeId == episode['sonarrEpisodeId'])\
|
.where(TableEpisodes.sonarrEpisodeId == episode['sonarrEpisodeId'])\
|
||||||
.execute()
|
.execute()
|
||||||
|
|
||||||
for i in range(len(attempt)):
|
for i in range(len(attempt)):
|
||||||
if attempt[i][0] == language:
|
if attempt[i][0] == language:
|
||||||
if search_active(attempt[i][1]):
|
if search_active(attempt[i][1]):
|
||||||
audio_language_list = get_audio_profile_languages(episode_id=episode['sonarrEpisodeId'])
|
audio_language_list = get_audio_profile_languages(episode_id=episode['sonarrEpisodeId'])
|
||||||
if len(audio_language_list) > 0:
|
if len(audio_language_list) > 0:
|
||||||
audio_language = audio_language_list[0]['name']
|
audio_language = audio_language_list[0]['name']
|
||||||
else:
|
|
||||||
audio_language = 'None'
|
|
||||||
|
|
||||||
result = download_subtitle(path_mappings.path_replace(episode['path']),
|
|
||||||
language.split(':')[0],
|
|
||||||
audio_language,
|
|
||||||
"True" if language.endswith(':hi') else "False",
|
|
||||||
"True" if language.endswith(':forced') else "False",
|
|
||||||
providers_list,
|
|
||||||
providers_auth,
|
|
||||||
str(episode['scene_name']),
|
|
||||||
episode['title'],
|
|
||||||
'series')
|
|
||||||
if result is not None:
|
|
||||||
message = result[0]
|
|
||||||
path = result[1]
|
|
||||||
forced = result[5]
|
|
||||||
if result[8]:
|
|
||||||
language_code = result[2] + ":hi"
|
|
||||||
elif forced:
|
|
||||||
language_code = result[2] + ":forced"
|
|
||||||
else:
|
else:
|
||||||
language_code = result[2]
|
audio_language = 'None'
|
||||||
provider = result[3]
|
|
||||||
score = result[4]
|
result = download_subtitle(path_mappings.path_replace(episode['path']),
|
||||||
subs_id = result[6]
|
language.split(':')[0],
|
||||||
subs_path = result[7]
|
audio_language,
|
||||||
store_subtitles(episode['path'], path_mappings.path_replace(episode['path']))
|
"True" if language.endswith(':hi') else "False",
|
||||||
history_log(1, episode['sonarrSeriesId'], episode['sonarrEpisodeId'], message, path,
|
"True" if language.endswith(':forced') else "False",
|
||||||
language_code, provider, score, subs_id, subs_path)
|
providers_list,
|
||||||
event_stream(type='series', action='update', payload=episode['sonarrSeriesId'])
|
providers_auth,
|
||||||
event_stream(type='episode-wanted', action='delete', payload=episode['sonarrEpisodeId'])
|
str(episode['scene_name']),
|
||||||
send_notifications(episode['sonarrSeriesId'], episode['sonarrEpisodeId'], message)
|
episode['title'],
|
||||||
else:
|
'series')
|
||||||
logging.debug(
|
if result is not None:
|
||||||
'BAZARR Search is not active for episode ' + episode['path'] + ' Language: ' + attempt[i][
|
message = result[0]
|
||||||
0])
|
path = result[1]
|
||||||
|
forced = result[5]
|
||||||
|
if result[8]:
|
||||||
|
language_code = result[2] + ":hi"
|
||||||
|
elif forced:
|
||||||
|
language_code = result[2] + ":forced"
|
||||||
|
else:
|
||||||
|
language_code = result[2]
|
||||||
|
provider = result[3]
|
||||||
|
score = result[4]
|
||||||
|
subs_id = result[6]
|
||||||
|
subs_path = result[7]
|
||||||
|
store_subtitles(episode['path'], path_mappings.path_replace(episode['path']))
|
||||||
|
history_log(1, episode['sonarrSeriesId'], episode['sonarrEpisodeId'], message, path,
|
||||||
|
language_code, provider, score, subs_id, subs_path)
|
||||||
|
event_stream(type='series', action='update', payload=episode['sonarrSeriesId'])
|
||||||
|
event_stream(type='episode-wanted', action='delete', payload=episode['sonarrEpisodeId'])
|
||||||
|
send_notifications(episode['sonarrSeriesId'], episode['sonarrEpisodeId'], message)
|
||||||
|
else:
|
||||||
|
logging.debug(
|
||||||
|
'BAZARR Search is not active for episode ' + episode['path'] + ' Language: ' + attempt[i][
|
||||||
|
0])
|
||||||
|
else:
|
||||||
|
logging.info("BAZARR All providers are throttled")
|
||||||
|
break
|
||||||
|
|
||||||
|
|
||||||
def wanted_download_subtitles_movie(radarr_id):
|
def wanted_download_subtitles_movie(radarr_id):
|
||||||
|
@ -1103,76 +1111,81 @@ def wanted_download_subtitles_movie(radarr_id):
|
||||||
.dicts()
|
.dicts()
|
||||||
movies_details = list(movies_details)
|
movies_details = list(movies_details)
|
||||||
|
|
||||||
providers_list = get_providers()
|
|
||||||
providers_auth = get_providers_auth()
|
providers_auth = get_providers_auth()
|
||||||
|
|
||||||
for movie in movies_details:
|
for movie in movies_details:
|
||||||
attempt = movie['failedAttempts']
|
providers_list = get_providers()
|
||||||
if type(attempt) == str:
|
|
||||||
attempt = ast.literal_eval(attempt)
|
|
||||||
for language in ast.literal_eval(movie['missing_subtitles']):
|
|
||||||
# confirm if language is still missing or if cutoff have been reached
|
|
||||||
confirmed_missing_subs = TableMovies.select(TableMovies.missing_subtitles) \
|
|
||||||
.where(TableMovies.radarrId == movie['radarrId']) \
|
|
||||||
.dicts() \
|
|
||||||
.get()
|
|
||||||
if language not in ast.literal_eval(confirmed_missing_subs['missing_subtitles']):
|
|
||||||
continue
|
|
||||||
|
|
||||||
if attempt is None:
|
if providers_list:
|
||||||
attempt = []
|
attempt = movie['failedAttempts']
|
||||||
attempt.append([language, time.time()])
|
if type(attempt) == str:
|
||||||
else:
|
attempt = ast.literal_eval(attempt)
|
||||||
att = list(zip(*attempt))[0]
|
for language in ast.literal_eval(movie['missing_subtitles']):
|
||||||
if language not in att:
|
# confirm if language is still missing or if cutoff have been reached
|
||||||
|
confirmed_missing_subs = TableMovies.select(TableMovies.missing_subtitles) \
|
||||||
|
.where(TableMovies.radarrId == movie['radarrId']) \
|
||||||
|
.dicts() \
|
||||||
|
.get()
|
||||||
|
if language not in ast.literal_eval(confirmed_missing_subs['missing_subtitles']):
|
||||||
|
continue
|
||||||
|
|
||||||
|
if attempt is None:
|
||||||
|
attempt = []
|
||||||
attempt.append([language, time.time()])
|
attempt.append([language, time.time()])
|
||||||
|
else:
|
||||||
|
att = list(zip(*attempt))[0]
|
||||||
|
if language not in att:
|
||||||
|
attempt.append([language, time.time()])
|
||||||
|
|
||||||
TableMovies.update({TableMovies.failedAttempts: str(attempt)})\
|
TableMovies.update({TableMovies.failedAttempts: str(attempt)})\
|
||||||
.where(TableMovies.radarrId == movie['radarrId'])\
|
.where(TableMovies.radarrId == movie['radarrId'])\
|
||||||
.execute()
|
.execute()
|
||||||
|
|
||||||
for i in range(len(attempt)):
|
for i in range(len(attempt)):
|
||||||
if attempt[i][0] == language:
|
if attempt[i][0] == language:
|
||||||
if search_active(attempt[i][1]) is True:
|
if search_active(attempt[i][1]) is True:
|
||||||
audio_language_list = get_audio_profile_languages(movie_id=movie['radarrId'])
|
audio_language_list = get_audio_profile_languages(movie_id=movie['radarrId'])
|
||||||
if len(audio_language_list) > 0:
|
if len(audio_language_list) > 0:
|
||||||
audio_language = audio_language_list[0]['name']
|
audio_language = audio_language_list[0]['name']
|
||||||
else:
|
|
||||||
audio_language = 'None'
|
|
||||||
|
|
||||||
result = download_subtitle(path_mappings.path_replace_movie(movie['path']),
|
|
||||||
language.split(':')[0],
|
|
||||||
audio_language,
|
|
||||||
"True" if language.endswith(':hi') else "False",
|
|
||||||
"True" if language.endswith(':forced') else "False",
|
|
||||||
providers_list,
|
|
||||||
providers_auth,
|
|
||||||
str(movie['sceneName']),
|
|
||||||
movie['title'],
|
|
||||||
'movie')
|
|
||||||
if result is not None:
|
|
||||||
message = result[0]
|
|
||||||
path = result[1]
|
|
||||||
forced = result[5]
|
|
||||||
if result[8]:
|
|
||||||
language_code = result[2] + ":hi"
|
|
||||||
elif forced:
|
|
||||||
language_code = result[2] + ":forced"
|
|
||||||
else:
|
else:
|
||||||
language_code = result[2]
|
audio_language = 'None'
|
||||||
provider = result[3]
|
|
||||||
score = result[4]
|
result = download_subtitle(path_mappings.path_replace_movie(movie['path']),
|
||||||
subs_id = result[6]
|
language.split(':')[0],
|
||||||
subs_path = result[7]
|
audio_language,
|
||||||
store_subtitles_movie(movie['path'], path_mappings.path_replace_movie(movie['path']))
|
"True" if language.endswith(':hi') else "False",
|
||||||
history_log_movie(1, movie['radarrId'], message, path, language_code, provider, score,
|
"True" if language.endswith(':forced') else "False",
|
||||||
subs_id, subs_path)
|
providers_list,
|
||||||
event_stream(type='movie-wanted', action='delete', payload=movie['radarrId'])
|
providers_auth,
|
||||||
send_notifications_movie(movie['radarrId'], message)
|
str(movie['sceneName']),
|
||||||
else:
|
movie['title'],
|
||||||
logging.info(
|
'movie')
|
||||||
'BAZARR Search is not active for this Movie ' + movie['path'] + ' Language: ' + attempt[i][
|
if result is not None:
|
||||||
0])
|
message = result[0]
|
||||||
|
path = result[1]
|
||||||
|
forced = result[5]
|
||||||
|
if result[8]:
|
||||||
|
language_code = result[2] + ":hi"
|
||||||
|
elif forced:
|
||||||
|
language_code = result[2] + ":forced"
|
||||||
|
else:
|
||||||
|
language_code = result[2]
|
||||||
|
provider = result[3]
|
||||||
|
score = result[4]
|
||||||
|
subs_id = result[6]
|
||||||
|
subs_path = result[7]
|
||||||
|
store_subtitles_movie(movie['path'], path_mappings.path_replace_movie(movie['path']))
|
||||||
|
history_log_movie(1, movie['radarrId'], message, path, language_code, provider, score,
|
||||||
|
subs_id, subs_path)
|
||||||
|
event_stream(type='movie-wanted', action='delete', payload=movie['radarrId'])
|
||||||
|
send_notifications_movie(movie['radarrId'], message)
|
||||||
|
else:
|
||||||
|
logging.info(
|
||||||
|
'BAZARR Search is not active for this Movie ' + movie['path'] + ' Language: ' + attempt[i][
|
||||||
|
0])
|
||||||
|
else:
|
||||||
|
logging.info("BAZARR All providers are throttled")
|
||||||
|
break
|
||||||
|
|
||||||
|
|
||||||
def wanted_search_missing_subtitles_series():
|
def wanted_search_missing_subtitles_series():
|
||||||
|
@ -1500,11 +1513,12 @@ def upgrade_subtitles():
|
||||||
|
|
||||||
count_movie_to_upgrade = len(movies_to_upgrade)
|
count_movie_to_upgrade = len(movies_to_upgrade)
|
||||||
|
|
||||||
providers_list = get_providers()
|
|
||||||
providers_auth = get_providers_auth()
|
providers_auth = get_providers_auth()
|
||||||
|
|
||||||
if settings.general.getboolean('use_sonarr'):
|
if settings.general.getboolean('use_sonarr'):
|
||||||
for i, episode in enumerate(episodes_to_upgrade):
|
for i, episode in enumerate(episodes_to_upgrade):
|
||||||
|
providers_list = get_providers()
|
||||||
|
|
||||||
show_progress(id='upgrade_episodes_progress',
|
show_progress(id='upgrade_episodes_progress',
|
||||||
header='Upgrading episodes subtitles...',
|
header='Upgrading episodes subtitles...',
|
||||||
name='{0} - S{1:02d}E{2:02d} - {3}'.format(episode['seriesTitle'],
|
name='{0} - S{1:02d}E{2:02d} - {3}'.format(episode['seriesTitle'],
|
||||||
|
@ -1514,8 +1528,7 @@ def upgrade_subtitles():
|
||||||
value=i,
|
value=i,
|
||||||
count=count_episode_to_upgrade)
|
count=count_episode_to_upgrade)
|
||||||
|
|
||||||
providers = get_providers()
|
if not providers_list:
|
||||||
if not providers:
|
|
||||||
logging.info("BAZARR All providers are throttled")
|
logging.info("BAZARR All providers are throttled")
|
||||||
return
|
return
|
||||||
if episode['language'].endswith('forced'):
|
if episode['language'].endswith('forced'):
|
||||||
|
@ -1572,17 +1585,15 @@ def upgrade_subtitles():
|
||||||
|
|
||||||
if settings.general.getboolean('use_radarr'):
|
if settings.general.getboolean('use_radarr'):
|
||||||
for i, movie in enumerate(movies_to_upgrade):
|
for i, movie in enumerate(movies_to_upgrade):
|
||||||
|
providers_list = get_providers()
|
||||||
|
|
||||||
show_progress(id='upgrade_movies_progress',
|
show_progress(id='upgrade_movies_progress',
|
||||||
header='Upgrading movies subtitles...',
|
header='Upgrading movies subtitles...',
|
||||||
name=movie['title'],
|
name=movie['title'],
|
||||||
value=i,
|
value=i,
|
||||||
count=count_movie_to_upgrade)
|
count=count_movie_to_upgrade)
|
||||||
|
|
||||||
providers = get_providers()
|
if not providers_list:
|
||||||
if not providers:
|
|
||||||
logging.info("BAZARR All providers are throttled")
|
|
||||||
return
|
|
||||||
if not providers:
|
|
||||||
logging.info("BAZARR All providers are throttled")
|
logging.info("BAZARR All providers are throttled")
|
||||||
return
|
return
|
||||||
if movie['language'].endswith('forced'):
|
if movie['language'].endswith('forced'):
|
||||||
|
|
Loading…
Reference in a new issue