From f809e608e47644aadd010fbc62147fd376c23f64 Mon Sep 17 00:00:00 2001 From: Gonzalo Merciel Date: Fri, 9 Aug 2019 11:41:35 -0300 Subject: [PATCH 01/14] Added subtitulamos.tv provider. --- .../providers/subtitulamostv.py | 136 ++++++++++++++++++ views/providers.tpl | 22 +++ 2 files changed, 158 insertions(+) create mode 100644 libs/subliminal_patch/providers/subtitulamostv.py diff --git a/libs/subliminal_patch/providers/subtitulamostv.py b/libs/subliminal_patch/providers/subtitulamostv.py new file mode 100644 index 000000000..08239768c --- /dev/null +++ b/libs/subliminal_patch/providers/subtitulamostv.py @@ -0,0 +1,136 @@ +# -*- coding: utf-8 -*- +import json +import logging +import os +import re +import io +import rarfile +import zipfile + +from babelfish import language_converters +from guessit import guessit +from requests import Session +from subzero.language import Language + +from subliminal import Movie, Episode, ProviderError, __short_version__ +from subliminal.exceptions import AuthenticationError, ConfigurationError, DownloadLimitExceeded, ProviderError +from subliminal_patch.subtitle import Subtitle, guess_matches +from subliminal.subtitle import fix_line_ending, SUBTITLE_EXTENSIONS +from subliminal_patch.providers import Provider +from subzero.language import Language + +logger = logging.getLogger(__name__) + +server_url = 'https://subtitulamos.tv/' + + +class SubtitulamosTVSubtitle(Subtitle): + provider_name = 'subtitulamostv' + + def __init__(self, subtitle_id, language, release_group, url, matches): + super(SubtitulamosTVSubtitle, self).__init__(language, page_link=url) + self.subtitle_id = subtitle_id + self.release_group = release_group + self.download_url = url + self.matches = matches + + @property + def id(self): + return self.subtitle_id + + @property + def download_link(self): + return self.download_url + + def get_matches(self, video): + matches = self.matches + + if isinstance(video, Episode): + matches |= guess_matches(video, guessit( + self.release_group, {'type': 'episode'}), partial=True) + + return matches + + +class SubtitulamosTVProvider(Provider): + """Subtitulamostv Provider""" + languages = {Language.fromietf(l) for l in ['en','es-AR','es-ES']} + video_types = (Episode,) + + # def __init__(self): + # if not token: + # raise ConfigurationError('Token must be specified') + # self.token = token + + def initialize(self): + self.session = Session() + self.session.headers = { + 'User-Agent': os.environ.get("SZ_USER_AGENT", "Sub-Zero/2")} + + def terminate(self): + self.session.close() + + def query(self, languages, video): + # query the server + result = None + matches = set() + + q = "%s %dx%02d" % (video.series, video.season, video.episode) + logger.debug('Searching subtitles "%s"', q) + + res = self.session.get( + server_url + 'search/query', params={'q':q}, timeout=10) + res.raise_for_status() + result = res.json() + + subtitles = [] + for s in [s for s in result if len(s['episodes'])]: + for e in s['episodes']: + res = self.session.get( + server_url + 'episodes/%d' % e['id'], timeout=10) + res.raise_for_status() + html = res.text + for lang_m in re.finditer(r"
(.*?)<\/div>.*?(?=
|
)", html, re.S): + lang = lang_m.group(1) + language = "es" + if "Latino" in lang: + language = "es-AR" + elif "(Espa" in lang: + language = "es-ES" + elif "English" in lang: + language = "en" + logger.debug('Found subtitles in "%s" language.', language) + + for subt_m in re.finditer(r"
(.*?)
.*?", lang_m.group(0), re.S): + logger.debug('Found release "%s".', subt_m.group(1)) + if video.alternative_series is None: + if video.series == s['name']: + matches.add('series') + elif s['name'] in [video.series]+video.alternative_series: + matches.add('series') + if video.season == e['season']: + matches.add('season') + if video.episode == e['number']: + matches.add('episode') + if video.title == e['name']: + matches.add('title') + + subtitles.append(SubtitulamosTVSubtitle( + subt_m.group(3), Language.fromietf(language), subt_m.group(1), server_url + subt_m.group(2), matches)) + + return subtitles + + def list_subtitles(self, video, languages): + return self.query(languages, video) + + def download_subtitle(self, subtitle): + logger.info('Downloading subtitle %r', subtitle) + r = self.session.get(subtitle.download_link, timeout=10) + r.raise_for_status() + + subtitle_content = r.text + + if subtitle_content: + subtitle.content = fix_line_ending(subtitle_content) + else: + logger.debug('Could not download subtitle from %s', subtitle.download_link) diff --git a/views/providers.tpl b/views/providers.tpl index 89d929217..e8c694411 100644 --- a/views/providers.tpl +++ b/views/providers.tpl @@ -628,6 +628,28 @@
+
+
+ +
+
+
+ + +
+
+ +
+
+ +
+
From 6b91993545c66ad3813cfd79fdee46ff1f1998ed Mon Sep 17 00:00:00 2001 From: Gonzalo Merciel Date: Thu, 10 Oct 2019 16:25:03 -0300 Subject: [PATCH 02/14] Added encoding, realease_info and better matching. --- .../providers/subtitulamostv.py | 94 ++++++++++++------- 1 file changed, 60 insertions(+), 34 deletions(-) diff --git a/libs/subliminal_patch/providers/subtitulamostv.py b/libs/subliminal_patch/providers/subtitulamostv.py index 08239768c..065e4563d 100644 --- a/libs/subliminal_patch/providers/subtitulamostv.py +++ b/libs/subliminal_patch/providers/subtitulamostv.py @@ -4,8 +4,6 @@ import logging import os import re import io -import rarfile -import zipfile from babelfish import language_converters from guessit import guessit @@ -26,42 +24,63 @@ server_url = 'https://subtitulamos.tv/' class SubtitulamosTVSubtitle(Subtitle): provider_name = 'subtitulamostv' + hash_verifiable = False - def __init__(self, subtitle_id, language, release_group, url, matches): - super(SubtitulamosTVSubtitle, self).__init__(language, page_link=url) - self.subtitle_id = subtitle_id - self.release_group = release_group - self.download_url = url + def __init__(self, language, page_link, download_link, description, title, matches, release_info): + super(SubtitulamosTVSubtitle, self).__init__(language, hearing_impaired=False, + page_link=page_link, release_info=release_info, encoding="windows-1252") + self.download_link = download_link + self.description = description.lower() + self.title = title self.matches = matches @property def id(self): - return self.subtitle_id - - @property - def download_link(self): - return self.download_url + return self.download_link def get_matches(self, video): matches = self.matches - if isinstance(video, Episode): - matches |= guess_matches(video, guessit( - self.release_group, {'type': 'episode'}), partial=True) + # release_group + if video.release_group and video.release_group.lower() in self.description: + matches.add('release_group') + + # resolution + if video.resolution and video.resolution.lower() in self.description: + matches.add('resolution') + + # format + if video.format: + formats = [video.format.lower()] + if formats[0] == "web-dl": + formats.append("webdl") + formats.append("webrip") + formats.append("web ") + for frmt in formats: + if frmt.lower() in self.description: + matches.add('format') + break + + # video_codec + if video.video_codec: + video_codecs = [video.video_codec.lower()] + if video_codecs[0] == "h264": + formats.append("x264") + elif video_codecs[0] == "h265": + formats.append("x265") + for vc in formats: + if vc.lower() in self.description: + matches.add('video_codec') + break return matches class SubtitulamosTVProvider(Provider): """Subtitulamostv Provider""" - languages = {Language.fromietf(l) for l in ['en','es-AR','es-ES']} + languages = {Language.fromietf(l) for l in ['en','es']} video_types = (Episode,) - # def __init__(self): - # if not token: - # raise ConfigurationError('Token must be specified') - # self.token = token - def initialize(self): self.session = Session() self.session.headers = { @@ -74,8 +93,8 @@ class SubtitulamosTVProvider(Provider): # query the server result = None matches = set() - - q = "%s %dx%02d" % (video.series, video.season, video.episode) + year = (" (%d)" % video.year) if video.year else "" + q = "%s%s %dx%02d" % (video.series, year, video.season, video.episode) logger.debug('Searching subtitles "%s"', q) res = self.session.get( @@ -93,20 +112,16 @@ class SubtitulamosTVProvider(Provider): for lang_m in re.finditer(r"
(.*?)<\/div>.*?(?=
|
)", html, re.S): lang = lang_m.group(1) language = "es" - if "Latino" in lang: - language = "es-AR" - elif "(Espa" in lang: - language = "es-ES" - elif "English" in lang: + if "English" in lang: language = "en" logger.debug('Found subtitles in "%s" language.', language) - for subt_m in re.finditer(r"
(.*?)
.*?", lang_m.group(0), re.S): + for subt_m in re.finditer(r"
(.*?)
.*?(?:.*?
.*?(.*?)

)?", lang_m.group(0), re.S): logger.debug('Found release "%s".', subt_m.group(1)) if video.alternative_series is None: - if video.series == s['name']: + if video.series.lower() == s['name'].lower(): matches.add('series') - elif s['name'] in [video.series]+video.alternative_series: + elif s['name'].lower() in [video.series.lower()]+map(lambda name: name.lower(), video.alternative_series): matches.add('series') if video.season == e['season']: matches.add('season') @@ -114,9 +129,20 @@ class SubtitulamosTVProvider(Provider): matches.add('episode') if video.title == e['name']: matches.add('title') - - subtitles.append(SubtitulamosTVSubtitle( - subt_m.group(3), Language.fromietf(language), subt_m.group(1), server_url + subt_m.group(2), matches)) + #if video.year is None or ("(%d)" % video.year) in s['name']: + matches.add('year') + logger.debug('%d' % video.year) + subtitles.append( + SubtitulamosTVSubtitle( + Language.fromietf(language), + server_url + 'episodes/%d' % e['id'], + server_url + subt_m.group(2), + subt_m.group(1)+(subt_m.group(3) if not subt_m.group(3) is None else ""), + e['name'], + matches, + ('%s %dx%d' % (s['name'], e['season'], e['number']), subt_m.group(1), lang_m.group(1)) + ) + ) return subtitles From 3d492ab4d635d76c90aa16e3c3269eccfb06e459 Mon Sep 17 00:00:00 2001 From: Gonzalo Merciel Date: Thu, 10 Oct 2019 17:24:13 -0300 Subject: [PATCH 03/14] Corrected release_info property. Corrected bug with matches. --- libs/subliminal_patch/providers/subtitulamostv.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/libs/subliminal_patch/providers/subtitulamostv.py b/libs/subliminal_patch/providers/subtitulamostv.py index 065e4563d..789ecd51e 100644 --- a/libs/subliminal_patch/providers/subtitulamostv.py +++ b/libs/subliminal_patch/providers/subtitulamostv.py @@ -28,18 +28,19 @@ class SubtitulamosTVSubtitle(Subtitle): def __init__(self, language, page_link, download_link, description, title, matches, release_info): super(SubtitulamosTVSubtitle, self).__init__(language, hearing_impaired=False, - page_link=page_link, release_info=release_info, encoding="windows-1252") + page_link=page_link, encoding="windows-1252") self.download_link = download_link self.description = description.lower() self.title = title - self.matches = matches + self.release_info = release_info + self.found_matches = matches @property def id(self): return self.download_link def get_matches(self, video): - matches = self.matches + matches = self.found_matches # release_group if video.release_group and video.release_group.lower() in self.description: @@ -92,7 +93,6 @@ class SubtitulamosTVProvider(Provider): def query(self, languages, video): # query the server result = None - matches = set() year = (" (%d)" % video.year) if video.year else "" q = "%s%s %dx%02d" % (video.series, year, video.season, video.episode) logger.debug('Searching subtitles "%s"', q) @@ -117,7 +117,7 @@ class SubtitulamosTVProvider(Provider): logger.debug('Found subtitles in "%s" language.', language) for subt_m in re.finditer(r"
(.*?)
.*?(?:.*?
.*?(.*?)

)?", lang_m.group(0), re.S): - logger.debug('Found release "%s".', subt_m.group(1)) + matches = set() if video.alternative_series is None: if video.series.lower() == s['name'].lower(): matches.add('series') @@ -131,7 +131,6 @@ class SubtitulamosTVProvider(Provider): matches.add('title') #if video.year is None or ("(%d)" % video.year) in s['name']: matches.add('year') - logger.debug('%d' % video.year) subtitles.append( SubtitulamosTVSubtitle( Language.fromietf(language), @@ -140,7 +139,7 @@ class SubtitulamosTVProvider(Provider): subt_m.group(1)+(subt_m.group(3) if not subt_m.group(3) is None else ""), e['name'], matches, - ('%s %dx%d' % (s['name'], e['season'], e['number']), subt_m.group(1), lang_m.group(1)) + '%s %dx%d,%s,%s' % (s['name'], e['season'], e['number'], subt_m.group(1), lang_m.group(1)), ) ) From dd41a30ab1b472a1ee8924ae2150c4a9bac72e98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Tue, 29 Oct 2019 06:56:20 -0400 Subject: [PATCH 04/14] Fix for #633 --- bazarr/list_subtitles.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bazarr/list_subtitles.py b/bazarr/list_subtitles.py index 23fff3384..d873a5cee 100644 --- a/bazarr/list_subtitles.py +++ b/bazarr/list_subtitles.py @@ -76,7 +76,8 @@ def store_subtitles(file): logging.debug("BAZARR external subtitles detected: " + "pb:forced") actual_subtitles.append( [str("pb:forced"), path_replace_reverse(subtitle_path)]) - + elif not language: + continue elif str(language) != 'und': logging.debug("BAZARR external subtitles detected: " + str(language)) actual_subtitles.append( @@ -172,6 +173,8 @@ def store_subtitles_movie(file): actual_subtitles.append( [str("pb:forced"), path_replace_reverse_movie(os.path.join(os.path.dirname(file), dest_folder, subtitle))]) + elif not language: + continue elif str(language) != 'und': logging.debug("BAZARR external subtitles detected: " + str(language)) actual_subtitles.append( From 50dfacece00f01987e87eff9094d7d51ef098419 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Tue, 29 Oct 2019 11:43:09 -0400 Subject: [PATCH 05/14] Added values to debug logging of DB requests. --- libs/sqlite3worker.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libs/sqlite3worker.py b/libs/sqlite3worker.py index ab7c1b8ce..fc589cc0b 100644 --- a/libs/sqlite3worker.py +++ b/libs/sqlite3worker.py @@ -93,7 +93,7 @@ class Sqlite3Worker(threading.Thread): for token, query, values, only_one in iter(self.sql_queue.get, None): LOGGER.debug("sql_queue: %s", self.sql_queue.qsize()) if token != self.exit_token: - LOGGER.debug("run: %s", query) + LOGGER.debug("run: %s, %s", query, values) self.run_query(token, query, values, only_one) execute_count += 1 # Let the executes build up a little before committing to disk @@ -188,9 +188,9 @@ class Sqlite3Worker(threading.Thread): If it's a select query it will return the results of the query. """ if self.exit_set: - LOGGER.debug("Exit set, not running: %s", query) + LOGGER.debug("Exit set, not running: %s, %s", query, values) return "Exit Called" - LOGGER.debug("execute: %s", query) + LOGGER.debug("execute: %s, %s", query, values) values = values or [] # A token to track this query with. token = str(uuid.uuid4()) From 5a2a5e5e8f393cd808405be4dc1872f8f977f124 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Tue, 29 Oct 2019 19:55:53 -0400 Subject: [PATCH 06/14] Small fix for #633. --- bazarr/list_subtitles.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bazarr/list_subtitles.py b/bazarr/list_subtitles.py index d873a5cee..bd3caf4e9 100644 --- a/bazarr/list_subtitles.py +++ b/bazarr/list_subtitles.py @@ -205,11 +205,12 @@ def store_subtitles_movie(file): database.execute("UPDATE table_movies SET subtitles=? WHERE path=?", (str(actual_subtitles), path_replace_reverse_movie(file))) - movie = database.execute("SELECT radarrId FROM table_movies WHERE path=?", (path_replace_reverse_movie(file),)) + movie = database.execute("SELECT radarrId FROM table_movies WHERE path=?", + (path_replace_reverse_movie(file),), only_one=True) if len(movie): logging.debug("BAZARR storing those languages to DB: " + str(actual_subtitles)) - list_missing_subtitles_movies(no=movie[0]['radarrId']) + list_missing_subtitles_movies(no=movie['radarrId']) else: logging.debug("BAZARR haven't been able to update existing subtitles to DB : " + str(actual_subtitles)) else: From 24fc1473dfbbce7d7fbf515241873e2cfbc78f5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Tue, 29 Oct 2019 21:20:52 -0400 Subject: [PATCH 07/14] Fix for #633. --- bazarr/get_episodes.py | 21 ++++++++++++--------- bazarr/get_movies.py | 30 +++++++++++++++++------------- bazarr/get_series.py | 21 +++++++++++++-------- 3 files changed, 42 insertions(+), 30 deletions(-) diff --git a/bazarr/get_episodes.py b/bazarr/get_episodes.py index 7366b9d8e..41e1bae2c 100644 --- a/bazarr/get_episodes.py +++ b/bazarr/get_episodes.py @@ -123,6 +123,12 @@ def sync_episodes(): 'audio_codec': audioCodec, 'episode_file_id': episode['episodeFile']['id']}) + # Remove old episodes from DB + removed_episodes = list(set(current_episodes_db_list) - set(current_episodes_sonarr)) + + for removed_episode in removed_episodes: + database.execute("DELETE FROM table_episodes WHERE sonarrEpisodeId=?", (removed_episode,)) + # Update existing episodes in DB episode_in_db_list = [] episodes_in_db = database.execute("SELECT sonarrSeriesId, sonarrEpisodeId, title, path, season, episode, " @@ -145,17 +151,14 @@ def sync_episodes(): # Insert new episodes in DB for added_episode in episodes_to_add: query = dict_converter.convert(added_episode) - database.execute( + result = database.execute( '''INSERT OR IGNORE INTO table_episodes(''' + query.keys_insert + ''') VALUES(''' + query.question_marks + ''')''', query.values) - altered_episodes.append([added_episode['sonarrEpisodeId'], - added_episode['path']]) - - # Remove old episodes from DB - removed_episodes = list(set(current_episodes_db_list) - set(current_episodes_sonarr)) - - for removed_episode in removed_episodes: - database.execute("DELETE FROM table_episodes WHERE sonarrEpisodeId=?", (removed_episode,)) + if result: + altered_episodes.append([added_episode['sonarrEpisodeId'], added_episode['path']]) + else: + logging.debug('BAZARR unable to insert this episode into the database:', + path_replace(added_episode['path'])) # Store subtitles for added or modified episodes for i, altered_episode in enumerate(altered_episodes, 1): diff --git a/bazarr/get_movies.py b/bazarr/get_movies.py index f67082334..11a35dbf0 100644 --- a/bazarr/get_movies.py +++ b/bazarr/get_movies.py @@ -204,8 +204,14 @@ def update_movies(): logging.error( 'BAZARR Radarr returned a movie without a file path: ' + movie["path"] + separator + movie['movieFile']['relativePath']) - - # Update or insert movies in DB + + # Remove old movies from DB + removed_movies = list(set(current_movies_db_list) - set(current_movies_radarr)) + + for removed_movie in removed_movies: + database.execute("DELETE FROM table_movies WHERE tmdbId=?", (removed_movie,)) + + # Update movies in DB movies_in_db_list = [] movies_in_db = database.execute("SELECT radarrId, title, path, tmdbId, overview, poster, fanart, " "audio_language, sceneName, monitored, sortTitle, year, " @@ -229,19 +235,17 @@ def update_movies(): # Insert new movies in DB for added_movie in movies_to_add: query = dict_converter.convert(added_movie) - database.execute( + result = database.execute( '''INSERT OR IGNORE INTO table_movies(''' + query.keys_insert + ''') VALUES(''' + query.question_marks + ''')''', query.values) - altered_movies.append([added_movie['tmdbId'], - added_movie['path'], - added_movie['radarrId'], - added_movie['monitored']]) - - # Remove old movies from DB - removed_movies = list(set(current_movies_db_list) - set(current_movies_radarr)) - - for removed_movie in removed_movies: - database.execute("DELETE FROM table_movies WHERE tmdbId=?", (removed_movie,)) + if result: + altered_movies.append([added_movie['tmdbId'], + added_movie['path'], + added_movie['radarrId'], + added_movie['monitored']]) + else: + logging.debug('BAZARR unable to insert this movie into the database:', + path_replace_movie(added_movie['path'])) # Store subtitles for added or modified movies for i, altered_movie in enumerate(altered_movies, 1): diff --git a/bazarr/get_series.py b/bazarr/get_series.py index ee7c98f9f..0900d6e57 100644 --- a/bazarr/get_series.py +++ b/bazarr/get_series.py @@ -12,6 +12,7 @@ from config import settings, url_sonarr from list_subtitles import list_missing_subtitles from database import database, dict_converter from utils import get_sonarr_version +from helper import path_replace def update_series(): @@ -120,6 +121,12 @@ def update_series(): 'sortTitle': show['sortTitle'], 'year': show['year'], 'alternateTitles': alternateTitles}) + + # Remove old series from DB + removed_series = list(set(current_shows_db_list) - set(current_shows_sonarr)) + + for series in removed_series: + database.execute("DELETE FROM table_shows WHERE tvdbId=?",(series,)) # Update existing series in DB series_in_db_list = [] @@ -139,16 +146,14 @@ def update_series(): # Insert new series in DB for added_series in series_to_add: query = dict_converter.convert(added_series) - database.execute( + result = database.execute( '''INSERT OR IGNORE INTO table_shows(''' + query.keys_insert + ''') VALUES(''' + query.question_marks + ''')''', query.values) - list_missing_subtitles(no=added_series['sonarrSeriesId']) - - # Remove old series from DB - removed_series = list(set(current_shows_db_list) - set(current_shows_sonarr)) - - for series in removed_series: - database.execute("DELETE FROM table_shows WHERE tvdbId=?",(series,)) + if result: + list_missing_subtitles(no=added_series['sonarrSeriesId']) + else: + logging.debug('BAZARR unable to insert this series into the database:', + path_replace(added_series['path'])) logging.debug('BAZARR All series synced from Sonarr into database.') From c933481d9aef4aa4dceead628ce7e3d496e0c7cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Wed, 30 Oct 2019 18:07:39 -0400 Subject: [PATCH 08/14] Some DB fixes. --- bazarr/get_subtitle.py | 5 +++-- bazarr/list_subtitles.py | 4 ++-- views/historymovies.tpl | 2 +- views/historyseries.tpl | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/bazarr/get_subtitle.py b/bazarr/get_subtitle.py index 221417c49..2be650122 100644 --- a/bazarr/get_subtitle.py +++ b/bazarr/get_subtitle.py @@ -996,7 +996,8 @@ def upgrade_subtitles(): "table_history.score, table_shows.hearing_impaired, " "table_episodes.scene_name, table_episodes.title," "table_episodes.sonarrSeriesId, table_episodes.sonarrEpisodeId," - "MAX(table_history.timestamp), table_shows.languages, table_shows.forced " + "MAX(table_history.timestamp) as timestamp, " + "table_shows.languages, table_shows.forced " "FROM table_history INNER JOIN table_shows on " "table_shows.sonarrSeriesId = table_history.sonarrSeriesId INNER JOIN " "table_episodes on table_episodes.sonarrEpisodeId = " @@ -1028,7 +1029,7 @@ def upgrade_subtitles(): upgradable_movies = database.execute("SELECT table_history_movie.video_path, table_history_movie.language, " "table_history_movie.score, table_movies.hearing_impaired, " "table_movies.sceneName, table_movies.title, table_movies.radarrId, " - "MAX(table_history_movie.timestamp), table_movies.languages, " + "MAX(table_history_movie.timestamp) as timestamp, table_movies.languages, " "table_movies.forced FROM table_history_movie INNER JOIN " "table_movies on table_movies.radarrId = table_history_movie.radarrId " "WHERE action IN (" + ','.join(map(str, query_actions)) + diff --git a/bazarr/list_subtitles.py b/bazarr/list_subtitles.py index bd3caf4e9..a1b5e983a 100644 --- a/bazarr/list_subtitles.py +++ b/bazarr/list_subtitles.py @@ -110,7 +110,7 @@ def store_subtitles(file): episode = database.execute("SELECT sonarrEpisodeId FROM table_episodes WHERE path=?", (path_replace_reverse(file),), only_one=True) - if len(episode): + if episode: logging.debug("BAZARR storing those languages to DB: " + str(actual_subtitles)) list_missing_subtitles(epno=episode['sonarrEpisodeId']) else: @@ -208,7 +208,7 @@ def store_subtitles_movie(file): movie = database.execute("SELECT radarrId FROM table_movies WHERE path=?", (path_replace_reverse_movie(file),), only_one=True) - if len(movie): + if movie: logging.debug("BAZARR storing those languages to DB: " + str(actual_subtitles)) list_missing_subtitles_movies(no=movie['radarrId']) else: diff --git a/views/historymovies.tpl b/views/historymovies.tpl index 27981f3c3..a10411988 100644 --- a/views/historymovies.tpl +++ b/views/historymovies.tpl @@ -89,7 +89,7 @@
- % upgradable_criteria = (row['timestamp'], row['video_path'], row['score']) + % upgradable_criteria = dict([('timestamp', row['timestamp']), ('video_path', row['video_path']), ('score', row['score'])]) % if upgradable_criteria in upgradable_movies: % if row['languages'] != "None": % desired_languages = ast.literal_eval(str(row['languages'])) diff --git a/views/historyseries.tpl b/views/historyseries.tpl index 4bbf05360..24a97f6d4 100644 --- a/views/historyseries.tpl +++ b/views/historyseries.tpl @@ -104,7 +104,7 @@
- % upgradable_criteria = (row['timestamp'], row['path'], row['score']) + % upgradable_criteria = dict([('timestamp', row['timestamp']), ('video_path', row['path']), ('score', row['score'])]) % if upgradable_criteria in upgradable_episodes: % if row['languages'] != "None": % desired_languages = ast.literal_eval(str(row['languages'])) From e80387e1501db8e1e25d3ae192fe8a67d5e371d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Wed, 30 Oct 2019 20:15:55 -0400 Subject: [PATCH 09/14] Added searches to analytics data. --- bazarr/get_subtitle.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bazarr/get_subtitle.py b/bazarr/get_subtitle.py index 2be650122..22f6bf361 100644 --- a/bazarr/get_subtitle.py +++ b/bazarr/get_subtitle.py @@ -177,6 +177,8 @@ def download_subtitle(path, language, hi, forced, providers, providers_auth, sce pre_download_hook=None, # fixme post_download_hook=None, # fixme language_hook=None) # fixme + for provider in providers: + track_event(category=provider, action='search', label=language[0]) else: downloaded_subtitles = None logging.info("BAZARR All providers are throttled") @@ -333,6 +335,8 @@ def manual_search(path, language, hi, forced, providers, providers_auth, sceneNa provider_configs=providers_auth, throttle_callback=provider_throttle, language_hook=None) # fixme + for provider in providers: + track_event(category=provider, action='search', label=language[0]) else: subtitles = [] logging.info("BAZARR All providers are throttled") From 2e645039f11fd0e6343709f1f094ffa5d9c5396d Mon Sep 17 00:00:00 2001 From: Gonzalo Merciel Date: Wed, 30 Oct 2019 22:57:50 -0500 Subject: [PATCH 10/14] Fixed encoding issues --- libs/subliminal_patch/providers/subtitulamostv.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/subliminal_patch/providers/subtitulamostv.py b/libs/subliminal_patch/providers/subtitulamostv.py index 789ecd51e..4ad74a67b 100644 --- a/libs/subliminal_patch/providers/subtitulamostv.py +++ b/libs/subliminal_patch/providers/subtitulamostv.py @@ -15,7 +15,6 @@ from subliminal.exceptions import AuthenticationError, ConfigurationError, Downl from subliminal_patch.subtitle import Subtitle, guess_matches from subliminal.subtitle import fix_line_ending, SUBTITLE_EXTENSIONS from subliminal_patch.providers import Provider -from subzero.language import Language logger = logging.getLogger(__name__) @@ -28,7 +27,7 @@ class SubtitulamosTVSubtitle(Subtitle): def __init__(self, language, page_link, download_link, description, title, matches, release_info): super(SubtitulamosTVSubtitle, self).__init__(language, hearing_impaired=False, - page_link=page_link, encoding="windows-1252") + page_link=page_link) self.download_link = download_link self.description = description.lower() self.title = title @@ -153,6 +152,7 @@ class SubtitulamosTVProvider(Provider): r = self.session.get(subtitle.download_link, timeout=10) r.raise_for_status() + r.encoding = "ISO-8859-1" subtitle_content = r.text if subtitle_content: From 086f54a65960c564639f6efc75a336bfc3a90e0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Thu, 31 Oct 2019 06:40:45 -0400 Subject: [PATCH 11/14] Another fix for missing movies. --- bazarr/get_subtitle.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/bazarr/get_subtitle.py b/bazarr/get_subtitle.py index 22f6bf361..36a51079a 100644 --- a/bazarr/get_subtitle.py +++ b/bazarr/get_subtitle.py @@ -562,9 +562,15 @@ def series_download_subtitles(no): episodes_details = database.execute("SELECT path, missing_subtitles, sonarrEpisodeId, scene_name " "FROM table_episodes WHERE sonarrSeriesId=? and missing_subtitles!='[]'" + episodes_details_clause, (no,)) + if not episodes_details: + logging.debug("BAZARR no episode for that sonarrSeriesId can be found in database:", str(no)) + return series_details = database.execute("SELECT hearing_impaired, title, forced FROM table_shows WHERE sonarrSeriesId=?", (no,), only_one=True) + if not series_details: + logging.debug("BAZARR no series with that sonarrSeriesId can be found in database:", str(no)) + return providers_list = get_providers() providers_auth = get_providers_auth() @@ -618,6 +624,10 @@ def episode_download_subtitles(no): "table_shows.forced FROM table_episodes LEFT JOIN table_shows on " "table_episodes.sonarrSeriesId = table_shows.sonarrSeriesId " "WHERE sonarrEpisodeId=?" + episodes_details_clause, (no,)) + + if not episodes_details: + logging.debug("BAZARR no episode with that sonarrEpisodeId can be found in database:", str(no)) + return providers_list = get_providers() providers_auth = get_providers_auth() @@ -662,6 +672,10 @@ def movies_download_subtitles(no): movie = database.execute("SELECT path, missing_subtitles, radarrId, sceneName, hearing_impaired, title, forced " "FROM table_movies WHERE radarrId=?" + movie_details_clause, (no,), only_one=True) + + if not movie: + logging.debug("BAZARR no movie with that radarrId can be found in database:", str(no)) + return providers_list = get_providers() providers_auth = get_providers_auth() From 2565a0395fd56fce550967507c1e9d4def25b926 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Thu, 31 Oct 2019 06:41:34 -0400 Subject: [PATCH 12/14] Changed row factory. --- libs/sqlite3worker.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/libs/sqlite3worker.py b/libs/sqlite3worker.py index fc589cc0b..61be9d423 100644 --- a/libs/sqlite3worker.py +++ b/libs/sqlite3worker.py @@ -66,11 +66,12 @@ class Sqlite3Worker(threading.Thread): file_name, check_same_thread=False, detect_types=sqlite3.PARSE_DECLTYPES) if as_dict: - self.sqlite3_conn.row_factory = dict_factory + self.sqlite3_conn.row_factory = sqlite3.Row self.sqlite3_cursor = self.sqlite3_conn.cursor() self.sql_queue = Queue.Queue(maxsize=max_queue_size) self.results = {} self.max_queue_size = max_queue_size + self.as_dict = as_dict self.exit_set = False # Token that is put into queue when close() is called. self.exit_token = str(uuid.uuid4()) @@ -126,7 +127,10 @@ class Sqlite3Worker(threading.Thread): if only_one: self.results[token] = self.sqlite3_cursor.fetchone() else: - self.results[token] = self.sqlite3_cursor.fetchall() + if self.as_dict: + self.results[token] = [dict(row) for row in self.sqlite3_cursor.fetchall()] + else: + self.results[token] = self.sqlite3_cursor.fetchall() except sqlite3.Error as err: # Put the error into the output queue since a response # is required. @@ -183,9 +187,11 @@ class Sqlite3Worker(threading.Thread): Args: query: The sql string using ? for placeholders of dynamic values. values: A tuple of values to be replaced into the ? of the query. + only_one: A boolean to use fetchone() instead of fetchall(). Returns: If it's a select query it will return the results of the query. + Normally it return a list of lists/dicts but if only_one it return a single list/dict. """ if self.exit_set: LOGGER.debug("Exit set, not running: %s, %s", query, values) @@ -201,10 +207,3 @@ class Sqlite3Worker(threading.Thread): return self.query_results(token) else: self.sql_queue.put((token, query, values, only_one), timeout=5) - - -def dict_factory(cursor, row): - d = {} - for idx, col in enumerate(cursor.description): - d[col[0]] = row[idx] - return d From 3afaa501d855d85d43242acb03609b7dcd6a02bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Thu, 31 Oct 2019 12:52:59 -0400 Subject: [PATCH 13/14] Revert "Changed row factory." This reverts commit 2565a039 --- libs/sqlite3worker.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/libs/sqlite3worker.py b/libs/sqlite3worker.py index 61be9d423..fc589cc0b 100644 --- a/libs/sqlite3worker.py +++ b/libs/sqlite3worker.py @@ -66,12 +66,11 @@ class Sqlite3Worker(threading.Thread): file_name, check_same_thread=False, detect_types=sqlite3.PARSE_DECLTYPES) if as_dict: - self.sqlite3_conn.row_factory = sqlite3.Row + self.sqlite3_conn.row_factory = dict_factory self.sqlite3_cursor = self.sqlite3_conn.cursor() self.sql_queue = Queue.Queue(maxsize=max_queue_size) self.results = {} self.max_queue_size = max_queue_size - self.as_dict = as_dict self.exit_set = False # Token that is put into queue when close() is called. self.exit_token = str(uuid.uuid4()) @@ -127,10 +126,7 @@ class Sqlite3Worker(threading.Thread): if only_one: self.results[token] = self.sqlite3_cursor.fetchone() else: - if self.as_dict: - self.results[token] = [dict(row) for row in self.sqlite3_cursor.fetchall()] - else: - self.results[token] = self.sqlite3_cursor.fetchall() + self.results[token] = self.sqlite3_cursor.fetchall() except sqlite3.Error as err: # Put the error into the output queue since a response # is required. @@ -187,11 +183,9 @@ class Sqlite3Worker(threading.Thread): Args: query: The sql string using ? for placeholders of dynamic values. values: A tuple of values to be replaced into the ? of the query. - only_one: A boolean to use fetchone() instead of fetchall(). Returns: If it's a select query it will return the results of the query. - Normally it return a list of lists/dicts but if only_one it return a single list/dict. """ if self.exit_set: LOGGER.debug("Exit set, not running: %s, %s", query, values) @@ -207,3 +201,10 @@ class Sqlite3Worker(threading.Thread): return self.query_results(token) else: self.sql_queue.put((token, query, values, only_one), timeout=5) + + +def dict_factory(cursor, row): + d = {} + for idx, col in enumerate(cursor.description): + d[col[0]] = row[idx] + return d From 95c3bcac7d33cc87f24cb58ad9dd3a07cf6c8379 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20V=C3=A9zina?= <5130500+morpheus65535@users.noreply.github.com> Date: Thu, 31 Oct 2019 12:56:12 -0400 Subject: [PATCH 14/14] Small fix to episodes to upgrade list SQL query. --- bazarr/get_subtitle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bazarr/get_subtitle.py b/bazarr/get_subtitle.py index 36a51079a..f8d98229f 100644 --- a/bazarr/get_subtitle.py +++ b/bazarr/get_subtitle.py @@ -1022,7 +1022,7 @@ def upgrade_subtitles(): "table_history.sonarrEpisodeId WHERE action IN " "(" + ','.join(map(str, query_actions)) + ") AND timestamp > ? AND " "score is not null" + series_monitored_only_query_string + - "GROUP BY table_history.video_path, table_history.language", + " GROUP BY table_history.video_path, table_history.language", (minimum_timestamp,)) upgradable_episodes_not_perfect = []