Merge branch 'development' into python3

# Conflicts:
#	bazarr/get_series.py
#	bazarr/list_subtitles.py
This commit is contained in:
Louis Vézina 2019-10-31 13:21:49 -04:00
commit 7f80b1de83
10 changed files with 261 additions and 44 deletions

View file

@ -124,6 +124,12 @@ def sync_episodes():
'audio_codec': audioCodec,
'episode_file_id': episode['episodeFile']['id']})
# Remove old episodes from DB
removed_episodes = list(set(current_episodes_db_list) - set(current_episodes_sonarr))
for removed_episode in removed_episodes:
database.execute("DELETE FROM table_episodes WHERE sonarrEpisodeId=?", (removed_episode,))
# Update existing episodes in DB
episode_in_db_list = []
episodes_in_db = database.execute("SELECT sonarrSeriesId, sonarrEpisodeId, title, path, season, episode, "
@ -146,17 +152,14 @@ def sync_episodes():
# Insert new episodes in DB
for added_episode in episodes_to_add:
query = dict_converter.convert(added_episode)
database.execute(
result = database.execute(
'''INSERT OR IGNORE INTO table_episodes(''' + query.keys_insert + ''') VALUES(''' + query.question_marks +
''')''', query.values)
altered_episodes.append([added_episode['sonarrEpisodeId'],
added_episode['path']])
# Remove old episodes from DB
removed_episodes = list(set(current_episodes_db_list) - set(current_episodes_sonarr))
for removed_episode in removed_episodes:
database.execute("DELETE FROM table_episodes WHERE sonarrEpisodeId=?", (removed_episode,))
if result:
altered_episodes.append([added_episode['sonarrEpisodeId'], added_episode['path']])
else:
logging.debug('BAZARR unable to insert this episode into the database:',
path_replace(added_episode['path']))
# Store subtitles for added or modified episodes
for i, altered_episode in enumerate(altered_episodes, 1):

View file

@ -206,8 +206,14 @@ def update_movies():
logging.error(
'BAZARR Radarr returned a movie without a file path: ' + movie["path"] + separator +
movie['movieFile']['relativePath'])
# Update or insert movies in DB
# Remove old movies from DB
removed_movies = list(set(current_movies_db_list) - set(current_movies_radarr))
for removed_movie in removed_movies:
database.execute("DELETE FROM table_movies WHERE tmdbId=?", (removed_movie,))
# Update movies in DB
movies_in_db_list = []
movies_in_db = database.execute("SELECT radarrId, title, path, tmdbId, overview, poster, fanart, "
"audio_language, sceneName, monitored, sortTitle, year, "
@ -231,19 +237,17 @@ def update_movies():
# Insert new movies in DB
for added_movie in movies_to_add:
query = dict_converter.convert(added_movie)
database.execute(
result = database.execute(
'''INSERT OR IGNORE INTO table_movies(''' + query.keys_insert + ''') VALUES(''' +
query.question_marks + ''')''', query.values)
altered_movies.append([added_movie['tmdbId'],
added_movie['path'],
added_movie['radarrId'],
added_movie['monitored']])
# Remove old movies from DB
removed_movies = list(set(current_movies_db_list) - set(current_movies_radarr))
for removed_movie in removed_movies:
database.execute("DELETE FROM table_movies WHERE tmdbId=?", (removed_movie,))
if result:
altered_movies.append([added_movie['tmdbId'],
added_movie['path'],
added_movie['radarrId'],
added_movie['monitored']])
else:
logging.debug('BAZARR unable to insert this movie into the database:',
path_replace_movie(added_movie['path']))
# Store subtitles for added or modified movies
for i, altered_movie in enumerate(altered_movies, 1):

View file

@ -15,6 +15,7 @@ from list_subtitles import list_missing_subtitles
from database import database, dict_converter
from utils import get_sonarr_version
import six
from helper import path_replace
def update_series():
@ -123,7 +124,13 @@ def update_series():
'sortTitle': show['sortTitle'],
'year': show['year'],
'alternateTitles': alternateTitles})
# Remove old series from DB
removed_series = list(set(current_shows_db_list) - set(current_shows_sonarr))
for series in removed_series:
database.execute("DELETE FROM table_shows WHERE tvdbId=?",(series,))
# Update existing series in DB
series_in_db_list = []
series_in_db = database.execute("SELECT title, path, tvdbId, sonarrSeriesId, overview, poster, fanart, "
@ -142,16 +149,14 @@ def update_series():
# Insert new series in DB
for added_series in series_to_add:
query = dict_converter.convert(added_series)
database.execute(
result = database.execute(
'''INSERT OR IGNORE INTO table_shows(''' + query.keys_insert + ''') VALUES(''' +
query.question_marks + ''')''', query.values)
list_missing_subtitles(no=added_series['sonarrSeriesId'])
# Remove old series from DB
removed_series = list(set(current_shows_db_list) - set(current_shows_sonarr))
for series in removed_series:
database.execute("DELETE FROM table_shows WHERE tvdbId=?",(series,))
if result:
list_missing_subtitles(no=added_series['sonarrSeriesId'])
else:
logging.debug('BAZARR unable to insert this series into the database:',
path_replace(added_series['path']))
logging.debug('BAZARR All series synced from Sonarr into database.')

View file

@ -181,6 +181,8 @@ def download_subtitle(path, language, hi, forced, providers, providers_auth, sce
pre_download_hook=None, # fixme
post_download_hook=None, # fixme
language_hook=None) # fixme
for provider in providers:
track_event(category=provider, action='search', label=language[0])
else:
downloaded_subtitles = None
logging.info("BAZARR All providers are throttled")
@ -337,6 +339,8 @@ def manual_search(path, language, hi, forced, providers, providers_auth, sceneNa
provider_configs=providers_auth,
throttle_callback=provider_throttle,
language_hook=None) # fixme
for provider in providers:
track_event(category=provider, action='search', label=language[0])
else:
subtitles = []
logging.info("BAZARR All providers are throttled")
@ -561,9 +565,15 @@ def series_download_subtitles(no):
episodes_details = database.execute("SELECT path, missing_subtitles, sonarrEpisodeId, scene_name "
"FROM table_episodes WHERE sonarrSeriesId=? and missing_subtitles!='[]'" +
episodes_details_clause, (no,))
if not episodes_details:
logging.debug("BAZARR no episode for that sonarrSeriesId can be found in database:", str(no))
return
series_details = database.execute("SELECT hearing_impaired, title, forced FROM table_shows WHERE sonarrSeriesId=?",
(no,), only_one=True)
if not series_details:
logging.debug("BAZARR no series with that sonarrSeriesId can be found in database:", str(no))
return
providers_list = get_providers()
providers_auth = get_providers_auth()
@ -617,6 +627,10 @@ def episode_download_subtitles(no):
"table_shows.forced FROM table_episodes LEFT JOIN table_shows on "
"table_episodes.sonarrSeriesId = table_shows.sonarrSeriesId "
"WHERE sonarrEpisodeId=?" + episodes_details_clause, (no,))
if not episodes_details:
logging.debug("BAZARR no episode with that sonarrEpisodeId can be found in database:", str(no))
return
providers_list = get_providers()
providers_auth = get_providers_auth()
@ -661,6 +675,10 @@ def movies_download_subtitles(no):
movie = database.execute("SELECT path, missing_subtitles, radarrId, sceneName, hearing_impaired, title, forced "
"FROM table_movies WHERE radarrId=?" + movie_details_clause, (no,), only_one=True)
if not movie:
logging.debug("BAZARR no movie with that radarrId can be found in database:", str(no))
return
providers_list = get_providers()
providers_auth = get_providers_auth()
@ -999,14 +1017,15 @@ def upgrade_subtitles():
"table_history.score, table_shows.hearing_impaired, "
"table_episodes.scene_name, table_episodes.title,"
"table_episodes.sonarrSeriesId, table_episodes.sonarrEpisodeId,"
"MAX(table_history.timestamp), table_shows.languages, table_shows.forced "
"MAX(table_history.timestamp) as timestamp, "
"table_shows.languages, table_shows.forced "
"FROM table_history INNER JOIN table_shows on "
"table_shows.sonarrSeriesId = table_history.sonarrSeriesId INNER JOIN "
"table_episodes on table_episodes.sonarrEpisodeId = "
"table_history.sonarrEpisodeId WHERE action IN "
"(" + ','.join(map(str, query_actions)) + ") AND timestamp > ? AND "
"score is not null" + series_monitored_only_query_string +
"GROUP BY table_history.video_path, table_history.language",
" GROUP BY table_history.video_path, table_history.language",
(minimum_timestamp,))
upgradable_episodes_not_perfect = []
@ -1031,7 +1050,7 @@ def upgrade_subtitles():
upgradable_movies = database.execute("SELECT table_history_movie.video_path, table_history_movie.language, "
"table_history_movie.score, table_movies.hearing_impaired, "
"table_movies.sceneName, table_movies.title, table_movies.radarrId, "
"MAX(table_history_movie.timestamp), table_movies.languages, "
"MAX(table_history_movie.timestamp) as timestamp, table_movies.languages, "
"table_movies.forced FROM table_history_movie INNER JOIN "
"table_movies on table_movies.radarrId = table_history_movie.radarrId "
"WHERE action IN (" + ','.join(map(str, query_actions)) +

View file

@ -78,7 +78,8 @@ def store_subtitles(original_path, reversed_path):
logging.debug("BAZARR external subtitles detected: " + "pb:forced")
actual_subtitles.append(
[str("pb:forced"), path_replace_reverse(subtitle_path)])
elif not language:
continue
elif str(language) != 'und':
logging.debug("BAZARR external subtitles detected: " + str(language))
actual_subtitles.append(
@ -110,7 +111,7 @@ def store_subtitles(original_path, reversed_path):
episode = database.execute("SELECT sonarrEpisodeId FROM table_episodes WHERE path=?",
(original_path,), only_one=True)
if len(episode):
if episode:
logging.debug("BAZARR storing those languages to DB: " + str(actual_subtitles))
list_missing_subtitles(epno=episode['sonarrEpisodeId'])
else:
@ -170,6 +171,8 @@ def store_subtitles_movie(original_path, reversed_path):
elif str(os.path.splitext(subtitle)[0]).lower().endswith(tuple(brazilian_portuguese_forced)):
logging.debug("BAZARR external subtitles detected: " + "pb:forced")
actual_subtitles.append([str("pb:forced"), path_replace_reverse_movie(subtitle_path)])
elif not language:
continue
elif str(language) != 'und':
logging.debug("BAZARR external subtitles detected: " + str(language))
actual_subtitles.append([str(language), path_replace_reverse_movie(subtitle_path)])
@ -197,11 +200,11 @@ def store_subtitles_movie(original_path, reversed_path):
database.execute("UPDATE table_movies SET subtitles=? WHERE path=?",
(str(actual_subtitles), original_path))
movie = database.execute("SELECT radarrId FROM table_movies WHERE path=?", (original_path,))
movie = database.execute("SELECT radarrId FROM table_movies WHERE path=?", (original_path,), only_one=True)
if len(movie):
if movie:
logging.debug("BAZARR storing those languages to DB: " + str(actual_subtitles))
list_missing_subtitles_movies(no=movie[0]['radarrId'])
list_missing_subtitles_movies(no=movie['radarrId'])
else:
logging.debug("BAZARR haven't been able to update existing subtitles to DB : " + str(actual_subtitles))
else:

View file

@ -93,7 +93,7 @@ class Sqlite3Worker(threading.Thread):
for token, query, values, only_one in iter(self.sql_queue.get, None):
LOGGER.debug("sql_queue: %s", self.sql_queue.qsize())
if token != self.exit_token:
LOGGER.debug("run: %s", query)
LOGGER.debug("run: %s, %s", query, values)
self.run_query(token, query, values, only_one)
execute_count += 1
# Let the executes build up a little before committing to disk
@ -188,9 +188,9 @@ class Sqlite3Worker(threading.Thread):
If it's a select query it will return the results of the query.
"""
if self.exit_set:
LOGGER.debug("Exit set, not running: %s", query)
LOGGER.debug("Exit set, not running: %s, %s", query, values)
return "Exit Called"
LOGGER.debug("execute: %s", query)
LOGGER.debug("execute: %s, %s", query, values)
values = values or []
# A token to track this query with.
token = str(uuid.uuid4())

View file

@ -0,0 +1,161 @@
# -*- coding: utf-8 -*-
import json
import logging
import os
import re
import io
from babelfish import language_converters
from guessit import guessit
from requests import Session
from subzero.language import Language
from subliminal import Movie, Episode, ProviderError, __short_version__
from subliminal.exceptions import AuthenticationError, ConfigurationError, DownloadLimitExceeded, ProviderError
from subliminal_patch.subtitle import Subtitle, guess_matches
from subliminal.subtitle import fix_line_ending, SUBTITLE_EXTENSIONS
from subliminal_patch.providers import Provider
logger = logging.getLogger(__name__)
server_url = 'https://subtitulamos.tv/'
class SubtitulamosTVSubtitle(Subtitle):
provider_name = 'subtitulamostv'
hash_verifiable = False
def __init__(self, language, page_link, download_link, description, title, matches, release_info):
super(SubtitulamosTVSubtitle, self).__init__(language, hearing_impaired=False,
page_link=page_link)
self.download_link = download_link
self.description = description.lower()
self.title = title
self.release_info = release_info
self.found_matches = matches
@property
def id(self):
return self.download_link
def get_matches(self, video):
matches = self.found_matches
# release_group
if video.release_group and video.release_group.lower() in self.description:
matches.add('release_group')
# resolution
if video.resolution and video.resolution.lower() in self.description:
matches.add('resolution')
# format
if video.format:
formats = [video.format.lower()]
if formats[0] == "web-dl":
formats.append("webdl")
formats.append("webrip")
formats.append("web ")
for frmt in formats:
if frmt.lower() in self.description:
matches.add('format')
break
# video_codec
if video.video_codec:
video_codecs = [video.video_codec.lower()]
if video_codecs[0] == "h264":
formats.append("x264")
elif video_codecs[0] == "h265":
formats.append("x265")
for vc in formats:
if vc.lower() in self.description:
matches.add('video_codec')
break
return matches
class SubtitulamosTVProvider(Provider):
"""Subtitulamostv Provider"""
languages = {Language.fromietf(l) for l in ['en','es']}
video_types = (Episode,)
def initialize(self):
self.session = Session()
self.session.headers = {
'User-Agent': os.environ.get("SZ_USER_AGENT", "Sub-Zero/2")}
def terminate(self):
self.session.close()
def query(self, languages, video):
# query the server
result = None
year = (" (%d)" % video.year) if video.year else ""
q = "%s%s %dx%02d" % (video.series, year, video.season, video.episode)
logger.debug('Searching subtitles "%s"', q)
res = self.session.get(
server_url + 'search/query', params={'q':q}, timeout=10)
res.raise_for_status()
result = res.json()
subtitles = []
for s in [s for s in result if len(s['episodes'])]:
for e in s['episodes']:
res = self.session.get(
server_url + 'episodes/%d' % e['id'], timeout=10)
res.raise_for_status()
html = res.text
for lang_m in re.finditer(r"<div class=\"subtitle_language\">(.*?)<\/div>.*?(?=<div class=\"subtitle_language\">|<div id=\"subtitle-actions\">)", html, re.S):
lang = lang_m.group(1)
language = "es"
if "English" in lang:
language = "en"
logger.debug('Found subtitles in "%s" language.', language)
for subt_m in re.finditer(r"<div class=\"version_name\">(.*?)</div>.*?<a href=\"/(subtitles/\d+/download)\" rel=\"nofollow\">(?:.*?<div class=\"version_comments ?\">.*?</i>(.*?)</p>)?", lang_m.group(0), re.S):
matches = set()
if video.alternative_series is None:
if video.series.lower() == s['name'].lower():
matches.add('series')
elif s['name'].lower() in [video.series.lower()]+map(lambda name: name.lower(), video.alternative_series):
matches.add('series')
if video.season == e['season']:
matches.add('season')
if video.episode == e['number']:
matches.add('episode')
if video.title == e['name']:
matches.add('title')
#if video.year is None or ("(%d)" % video.year) in s['name']:
matches.add('year')
subtitles.append(
SubtitulamosTVSubtitle(
Language.fromietf(language),
server_url + 'episodes/%d' % e['id'],
server_url + subt_m.group(2),
subt_m.group(1)+(subt_m.group(3) if not subt_m.group(3) is None else ""),
e['name'],
matches,
'%s %dx%d,%s,%s' % (s['name'], e['season'], e['number'], subt_m.group(1), lang_m.group(1)),
)
)
return subtitles
def list_subtitles(self, video, languages):
return self.query(languages, video)
def download_subtitle(self, subtitle):
logger.info('Downloading subtitle %r', subtitle)
r = self.session.get(subtitle.download_link, timeout=10)
r.raise_for_status()
r.encoding = "ISO-8859-1"
subtitle_content = r.text
if subtitle_content:
subtitle.content = fix_line_ending(subtitle_content)
else:
logger.debug('Could not download subtitle from %s', subtitle.download_link)

View file

@ -89,7 +89,7 @@
</div>
</td>
<td>
% upgradable_criteria = (row['timestamp'], row['video_path'], row['score'])
% upgradable_criteria = dict([('timestamp', row['timestamp']), ('video_path', row['video_path']), ('score', row['score'])])
% if upgradable_criteria in upgradable_movies:
% if row['languages'] != "None":
% desired_languages = ast.literal_eval(str(row['languages']))

View file

@ -104,7 +104,7 @@
</div>
</td>
<td>
% upgradable_criteria = (row['timestamp'], row['path'], row['score'])
% upgradable_criteria = dict([('timestamp', row['timestamp']), ('video_path', row['path']), ('score', row['score'])])
% if upgradable_criteria in upgradable_episodes:
% if row['languages'] != "None":
% desired_languages = ast.literal_eval(str(row['languages']))

View file

@ -628,6 +628,28 @@
</div>
<div class="middle aligned row">
<div class="right aligned four wide column">
<label>Subtitulamos.tv</label>
</div>
<div class="one wide column">
<div id="subtitulamostv" class="ui toggle checkbox provider">
<input type="checkbox">
<label></label>
</div>
</div>
<div class="collapsed column">
<div class="collapsed center aligned column">
<div class="ui basic icon" data-tooltip="Spanish subtitles provider." data-inverted="">
<i class="help circle large icon"></i>
</div>
</div>
</div>
</div>
<div id="subtitulamostv_option" class="ui grid container">
</div>
<div class="middle aligned row">
<div class="right aligned four wide column">
<label>Supersubtitles</label>