Adaptive searching (#130)

* Move settings from Database to Config file

* Delete unused config and forgot upload init

* Fix

* Update get_subtitles and fix provider auth

* Revert "Update get_subtitles and fix provider auth"

This reverts commit c904bf6b5a.

* Update get_sebtitles

* more fixes

* Update requirements

* Initial Commit

* Finished

bazarr.py

@@ -755,6 +755,11 @@ def save_settings():
         settings_general_only_monitored = 'False'
     else:
         settings_general_only_monitored = 'True'
+    settings_general_adaptive_searching = request.forms.get('settings_general_adaptive_searching')
+    if settings_general_adaptive_searching is None:
+        settings_general_adaptive_searching = 'False'
+    else:
+        settings_general_adaptive_searching = 'True'
     settings_general_minimum_score = request.forms.get('settings_general_minimum_score')
     settings_general_minimum_score_movies = request.forms.get('settings_general_minimum_score_movies')
     settings_general_use_postprocessing = request.forms.get('settings_general_use_postprocessing')
@@ -805,8 +810,8 @@ def save_settings():
     cfg.set('general', 'minimum_score_movie', text_type(settings_general_minimum_score_movies))
     cfg.set('general', 'use_embedded_subs', text_type(settings_general_embedded))
     cfg.set('general', 'only_monitored', text_type(settings_general_only_monitored))
-    # cfg.set('general', 'configured', text_type(configured))
-    # cfg.set('general', 'updated', text_type(updated))
+    cfg.set('general', 'adaptive_searching', text_type(settings_general_adaptive_searching))
+
 
     if after != before:
         configured()

create_db.sql

@@ -46,6 +46,7 @@ CREATE TABLE "table_episodes" (
 	`missing_subtitles`	TEXT,
 	`scene_name`    TEXT,
 	`monitored` TEXT
+	'failedAttempts' 'text'
 );
 CREATE TABLE "table_movies" (
     `tmdbId` TEXT NOT NULL UNIQUE,
@@ -62,6 +63,7 @@ CREATE TABLE "table_movies" (
     `audio_language` "text",
     `sceneName` TEXT,
     `monitored` TEXT, PRIMARY KEY(`tmdbId`)
+    'failedAttempts' 'text'
 );
 CREATE TABLE "table_history_movie" (
     `id` INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,

get_episodes.py

@@ -21,6 +21,7 @@ def update_all_movies():
     logging.info('All missing movie subtitles updated in database.')
 
 def sync_episodes():
+    logging.debug('Starting episode sync from Sonarr.')
     from get_settings import get_sonarr_settings
     url_sonarr = get_sonarr_settings()[6]
     apikey_sonarr = get_sonarr_settings()[4]

get_movies.py

@@ -9,6 +9,7 @@ from get_settings import get_general_settings, path_replace_movie
 from list_subtitles import store_subtitles_movie, list_missing_subtitles_movies
 
 def update_movies():
+    logging.debug('Starting movie sync from Radarr.')
     from get_settings import get_radarr_settings
     url_radarr = get_radarr_settings()[6]
     # url_radarr_short = get_radarr_settings()[7]
@@ -105,7 +106,10 @@ def update_movies():
     # Close database connection
     db.close()
 
+    logging.debug('All movies synced from Radarr into database.')
+
     list_missing_subtitles_movies()
+    logging.debug('All movie missing subtitles updated in database.')
 
 def get_profile_list():
     from get_settings import get_radarr_settings

get_settings.py

@@ -141,6 +141,11 @@ def get_general_settings():
             only_monitored = cfg.getboolean('general', 'only_monitored')
         else:
             only_monitored = False
+            
+        if cfg.has_option('general', 'adaptive_searching'):
+            adaptive_searching = cfg.getboolean('general', 'adaptive_searching')
+        else:
+            adaptive_searching = False
 
     else:
         ip = '0.0.0.0'
@@ -168,8 +173,9 @@ def get_general_settings():
         minimum_score_movie = '100'
         use_embedded_subs = False
         only_monitored = False
+        adaptive_searching = False
 
-    return [ip, port, base_url, path_mappings, log_level, branch, auto_update, single_language, minimum_score, use_scenename, use_postprocessing, postprocessing_cmd, use_sonarr, use_radarr, path_mappings_movie, serie_default_enabled, serie_default_language, serie_default_hi, movie_default_enabled,movie_default_language, movie_default_hi, page_size, minimum_score_movie, use_embedded_subs, only_monitored]
+    return [ip, port, base_url, path_mappings, log_level, branch, auto_update, single_language, minimum_score, use_scenename, use_postprocessing, postprocessing_cmd, use_sonarr, use_radarr, path_mappings_movie, serie_default_enabled, serie_default_language, serie_default_hi, movie_default_enabled,movie_default_language, movie_default_hi, page_size, minimum_score_movie, use_embedded_subs, only_monitored, adaptive_searching]
 
 
 def get_auth_settings():
@@ -413,4 +419,5 @@ movie_default_hi = result[20]
 page_size = result[21]
 minimum_score_movie = result[22]
 use_embedded_subs = result[23]
-only_monitored = result[24]
+only_monitored = result[24]
+adaptive_searching = result[25]

get_subtitle.py

@@ -5,6 +5,8 @@ import sqlite3
 import ast
 import logging
 import subprocess
+import time
+from datetime import datetime, timedelta
 from babelfish import Language
 from subliminal import region, scan_video, Video, download_best_subtitles, compute_score, save_subtitles
 from get_languages import language_from_alpha3, alpha2_from_alpha3, alpha3_from_alpha2
@@ -137,7 +139,7 @@ def series_download_subtitles(no):
     else:
         providers_list = None
         providers_auth = None
-            
+        
     for episode in episodes_details:
         for language in ast.literal_eval(episode[1]):
             if language is not None:
@@ -186,7 +188,7 @@ def movies_download_subtitles(no):
 def wanted_download_subtitles(path):
     conn_db = sqlite3.connect(os.path.join(config_dir, 'db/bazarr.db'), timeout=30)
     c_db = conn_db.cursor()
-    episodes_details = c_db.execute("SELECT table_episodes.path, table_episodes.missing_subtitles, table_episodes.sonarrEpisodeId, table_episodes.sonarrSeriesId, table_shows.hearing_impaired, table_episodes.scene_name FROM table_episodes INNER JOIN table_shows on table_shows.sonarrSeriesId = table_episodes.sonarrSeriesId WHERE table_episodes.path = ? AND missing_subtitles != '[]'", (path_replace_reverse(path),)).fetchall()
+    episodes_details = c_db.execute("SELECT table_episodes.path, table_episodes.missing_subtitles, table_episodes.sonarrEpisodeId, table_episodes.sonarrSeriesId, table_shows.hearing_impaired, table_episodes.scene_name, table_episodes.failedAttempts FROM table_episodes INNER JOIN table_shows on table_shows.sonarrSeriesId = table_episodes.sonarrSeriesId WHERE table_episodes.path = ? AND missing_subtitles != '[]'", (path_replace_reverse(path),)).fetchall()
     enabled_providers = c_db.execute("SELECT * FROM table_settings_providers WHERE enabled = 1").fetchall()
     c_db.close()
 
@@ -208,19 +210,41 @@ def wanted_download_subtitles(path):
         providers_auth = None
         
     for episode in episodes_details:
+        attempt = episode[6]
+        if type(attempt) == unicode:
+            attempt = ast.literal_eval(attempt)
         for language in ast.literal_eval(episode[1]):
-            message = download_subtitle(path_replace(episode[0]), str(alpha3_from_alpha2(language)), episode[4], providers_list, providers_auth, episode[5], 'series')
-            if message is not None:
-                store_subtitles(path_replace(episode[0]))
-                list_missing_subtitles(episode[3])
-                history_log(1, episode[3], episode[2], message)
-                send_notifications(episode[3], episode[2], message)
+            if attempt is None:
+                attempt = []
+                attempt.append([language, time.time()])
+            else:
+                att = zip(*attempt)[0]
+                if language not in att:
+                    attempt.append([language, time.time()])
+                    
+            conn_db = sqlite3.connect(os.path.join(config_dir, 'db/bazarr.db'), timeout=30)
+            c_db = conn_db.cursor()
+            c_db.execute('UPDATE table_episodes SET failedAttempts = ? WHERE sonarrEpisodeId = ?', (unicode(attempt), episode[2]))
+            conn_db.commit()
+            c_db.close()
+            
+            for i in range(len(attempt)):
+                if attempt[i][0] == language:
+                    if search_active(attempt[i][1]) is True:
+                        message = download_subtitle(path_replace(episode[0]), str(alpha3_from_alpha2(language)), episode[4], providers_list, providers_auth, episode[5], 'series')
+                        if message is not None:
+                            store_subtitles(path_replace(episode[0]))
+                            list_missing_subtitles(episode[3])
+                            history_log(1, episode[3], episode[2], message)
+                            send_notifications(episode[3], episode[2], message)
+                    else:
+                        logging.debug('Search is not active for episode ' + episode[0] + ' Language: ' + attempt[i][0])
 
 
 def wanted_download_subtitles_movie(path):
     conn_db = sqlite3.connect(os.path.join(config_dir, 'db/bazarr.db'), timeout=30)
     c_db = conn_db.cursor()
-    movies_details = c_db.execute("SELECT path, missing_subtitles, radarrId, radarrId, hearing_impaired, sceneName FROM table_movies WHERE path = ? AND missing_subtitles != '[]'", (path_replace_reverse_movie(path),)).fetchall()
+    movies_details = c_db.execute("SELECT path, missing_subtitles, radarrId, radarrId, hearing_impaired, sceneName, failedAttempts FROM table_movies WHERE path = ? AND missing_subtitles != '[]'", (path_replace_reverse_movie(path),)).fetchall()
     enabled_providers = c_db.execute("SELECT * FROM table_settings_providers WHERE enabled = 1").fetchall()
     c_db.close()
 
@@ -242,13 +266,35 @@ def wanted_download_subtitles_movie(path):
         providers_auth = None
 
     for movie in movies_details:
+        attempt = movie[6]
+        if type(attempt) == unicode:
+            attempt = ast.literal_eval(attempt)
         for language in ast.literal_eval(movie[1]):
-            message = download_subtitle(path_replace_movie(movie[0]), str(alpha3_from_alpha2(language)), movie[4], providers_list, providers_auth, movie[5], 'movies')
-            if message is not None:
-                store_subtitles_movie(path_replace_movie(movie[0]))
-                list_missing_subtitles_movies(movie[3])
-                history_log_movie(1, movie[3], message)
-                send_notifications_movie(movie[3], message)
+            if attempt is None:
+                attempt = []
+                attempt.append([language, time.time()])
+            else:
+                att = zip(*attempt)[0]
+                if language not in att:
+                    attempt.append([language, time.time()])
+                    
+            conn_db = sqlite3.connect(os.path.join(config_dir, 'db/bazarr.db'), timeout=30)
+            c_db = conn_db.cursor()
+            c_db.execute('UPDATE table_movies SET failedAttempts = ? WHERE radarrId = ?', (unicode(attempt), movie[2]))
+            conn_db.commit()
+            c_db.close()
+            
+            for i in range(len(attempt)):
+                if attempt[i][0] == language:
+                    if search_active(attempt[i][1]) is True:
+                        message = download_subtitle(path_replace_movie(movie[0]), str(alpha3_from_alpha2(language)), movie[4], providers_list, providers_auth, movie[5], 'movies')
+                        if message is not None:
+                            store_subtitles_movie(path_replace_movie(movie[0]))
+                            list_missing_subtitles_movies(movie[3])
+                            history_log_movie(1, movie[3], message)
+                            send_notifications_movie(movie[3], message)
+                    else:
+                        logging.info('Search is not active for movie ' + movie[0] + ' Language: ' + attempt[i][0])
 
 
 def wanted_search_missing_subtitles():
@@ -281,3 +327,22 @@ def wanted_search_missing_subtitles():
             wanted_download_subtitles_movie(movie[0])
 
     logging.info('Finished searching for missing subtitles. Check histories for more information.')
+  
+
+def search_active(timestamp):
+    if get_general_settings()[25] is True:
+        search_deadline = timedelta(weeks=3)
+        search_delta = timedelta(weeks=1)
+        aa = datetime.fromtimestamp(float(timestamp))
+        attempt_datetime = datetime.strptime(str(aa).split(".")[0], '%Y-%m-%d %H:%M:%S')
+        attempt_search_deadline = attempt_datetime + search_deadline
+        today = datetime.today()
+        attempt_age_in_days = (today.date() - attempt_search_deadline.date()).days
+        if today.date() <= attempt_search_deadline.date():
+            return True
+        elif attempt_age_in_days % search_delta.days == 0:
+            return True
+        else:
+            return False
+    else:
+        return True

init.py

@@ -124,7 +124,6 @@ try:
     logging.info('Config file succesfully migrated from database')
 
 except sqlite3.OperationalError:
-# if not os.path.exists(config_file):
     if os.path.exists(config_file) is False:
         cfg = ConfigParser()
 
@@ -158,6 +157,7 @@ except sqlite3.OperationalError:
         cfg.set(section, 'minimum_score_movie', "100")
         cfg.set(section, 'use_embedded_subs', "False")
         cfg.set(section, 'only_monitored', "False")
+        cfg.set(section, 'adaptive_searching', "False")
 
         section = 'auth'
 

update_db.py

@@ -69,6 +69,12 @@ if os.path.exists(os.path.join(config_dir, 'db/bazarr.db')) == True:
     except:
         pass
 
+    try:
+        c.execute('alter table table_movies add column "failedAttempts" "text"')
+        c.execute('alter table table_episodes add column "failedAttempts" "text"')
+    except:
+        pass
+
 
     # Commit change to db
     db.commit()