StuffAnThings/qbit_manage
1
1
Fork 0
mirror of https://github.com/StuffAnThings/qbit_manage.git synced 2025-11-08 15:30:38 +08:00
Code Issues Projects Releases Packages Wiki Activity

refactor(scheduler): migrate persistence to qbm_settings.yml

Browse source 
- Update scheduler to use qbm_settings.yml instead of schedule.yml
- Add automatic migration from legacy schedule.yml to new format
- Restructure settings with 'schedule' root key for better organization
- Update file paths and references across modules for consistency
- Preserve backward compatibility with migration logic

Legacy schedule.yml files are automatically migrated and removed.
This commit is contained in:
bobokun 2025-09-04 08:33:58 -04:00
parent f500888182
commit 5d70428e90
No known key found for this signature in database
GPG key ID: B73932169607D927
5 changed files with 168 additions and 79 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
VERSION
View file

@ -1 +1 @@
4.6.1-develop5 4.6.1-develop6

181
modules/scheduler.py
View file

@ -31,13 +31,15 @@ logger = util.logger
class Scheduler: class Scheduler:
""" """
Simplified scheduler with built-in persistence support. Simplified scheduler with built-in persistence support.
Handles both cron expressions and interval scheduling with automatic persistence. Handles both cron expressions and interval scheduling with automatic persistence to qbm_settings.yml.
""" """
def __init__(self, config_dir: str = "config", suppress_logging: bool = False, read_only: bool = False): def __init__(self, config_dir: str = "config", suppress_logging: bool = False, read_only: bool = False):
"""Initialize the Scheduler with persistence support.""" """Initialize the Scheduler with persistence support."""
self.config_dir = Path(config_dir) self.config_dir = Path(config_dir)
self.schedule_file = self.config_dir / "schedule.yml" self.settings_file = self.config_dir / "qbm_settings.yml"
# Legacy file path for migration
self.legacy_schedule_file = self.config_dir / "schedule.yml"
self.config_dir.mkdir(exist_ok=True, parents=True) self.config_dir.mkdir(exist_ok=True, parents=True)
# Thread-safe components # Thread-safe components
@ -50,7 +52,7 @@ class Scheduler:
self._callback = None self._callback = None
self._read_only = read_only self._read_only = read_only
# Persistence disabled flag (stored inside schedule.yml as 'disabled: true') # Persistence disabled flag (stored inside qbm_settings.yml under schedule.disabled)
self._persistence_disabled = False self._persistence_disabled = False
# Load schedule on initialization (will set _persistence_disabled if file says disabled) # Load schedule on initialization (will set _persistence_disabled if file says disabled)
@ -63,31 +65,51 @@ class Scheduler:
Load schedule from persistent file or environment variable. Load schedule from persistent file or environment variable.
Priority (when not disabled): Priority (when not disabled):
1. schedule.yml file (persistent) 1. qbm_settings.yml file (persistent) - new structure with 'schedule' root key
2. QBT_SCHEDULE environment variable (fallback) 2. schedule.yml file (legacy, will be migrated to new format)
3. QBT_SCHEDULE environment variable (fallback)
If 'disabled: true' in schedule.yml, skip loading its schedule and fall back to env/none. If 'disabled: true' in settings file, skip loading its schedule and fall back to env/none.
Returns: Returns:
bool: True if schedule loaded successfully bool: True if schedule loaded successfully
""" """
schedule_path = str(self.schedule_file) settings_path = str(self.settings_file)
# Reset in-memory state; _persistence_disabled will be set if file indicates # Reset in-memory state; _persistence_disabled will be set if file indicates
self._persistence_disabled = False self._persistence_disabled = False
if self.schedule_file.exists(): # Check for new settings file first
if self.settings_file.exists():
return self._load_from_settings_file(settings_path, suppress_logging)
# Check for legacy schedule file and migrate if found
if self.legacy_schedule_file.exists():
if self._migrate_legacy_schedule_file(suppress_logging):
return self._load_from_settings_file(settings_path, suppress_logging)
if not suppress_logging:
logger.debug(f"No settings file found at startup (expected path: {settings_path})")
return self._load_from_environment(suppress_logging)
def _load_from_settings_file(self, settings_path: str, suppress_logging: bool = False) -> bool:
"""Load schedule from qbm_settings.yml file."""
try: try:
yaml_loader = YAML(str(self.schedule_file)) yaml_loader = YAML(settings_path)
data = yaml_loader.data data = yaml_loader.data
if data and isinstance(data, dict): if data and isinstance(data, dict):
# Handle new structure with 'schedule' root key
schedule_data = data.get("schedule", {})
# Read disabled flag first # Read disabled flag first
if bool(data.get("disabled")): if bool(schedule_data.get("disabled")):
self._persistence_disabled = True self._persistence_disabled = True
if not suppress_logging: if not suppress_logging:
logger.debug(f"Persistent schedule disabled (disabled: true in {schedule_path})") logger.debug(f"Persistent schedule disabled (disabled: true in {settings_path})")
schedule_type = data.get("type")
schedule_value = data.get("value") schedule_type = schedule_data.get("type")
schedule_value = schedule_data.get("value")
if not self._persistence_disabled and schedule_type and schedule_value is not None: if not self._persistence_disabled and schedule_type and schedule_value is not None:
if self._validate_schedule(schedule_type, schedule_value): if self._validate_schedule(schedule_type, schedule_value):
self.current_schedule = (schedule_type, schedule_value) self.current_schedule = (schedule_type, schedule_value)
@ -96,20 +118,64 @@ class Scheduler:
next_run_info = calc_next_run(self.next_run) next_run_info = calc_next_run(self.next_run)
logger.info(f"{next_run_info['next_run_str']}") logger.info(f"{next_run_info['next_run_str']}")
if not suppress_logging: if not suppress_logging:
logger.debug( logger.debug(f"Schedule loaded from file: {schedule_type}={schedule_value} (path={settings_path})")
f"Schedule loaded from file: {schedule_type}={schedule_value} (path={schedule_path})"
)
return True return True
else: else:
logger.warning(f"Invalid schedule structure in file {schedule_path}: {data}") logger.warning(f"Invalid schedule structure in file {settings_path}: {schedule_data}")
else: elif self._persistence_disabled:
logger.warning(f"schedule.yml did not contain a dict at {schedule_path}")
except Exception as e:
logger.error(f"Error loading schedule.yml at {schedule_path}: {e}")
else:
if not suppress_logging: if not suppress_logging:
logger.debug(f"No schedule.yml found at startup (expected path: {schedule_path})") logger.debug(f"Schedule persistence disabled in {settings_path}")
return False
else:
logger.warning(f"qbm_settings.yml missing schedule data at {settings_path}")
else:
logger.warning(f"qbm_settings.yml did not contain a dict at {settings_path}")
except Exception as e:
logger.error(f"Error loading qbm_settings.yml at {settings_path}: {e}")
return False
def _migrate_legacy_schedule_file(self, suppress_logging: bool = False) -> bool:
"""Migrate legacy schedule.yml to new qbm_settings.yml format."""
try:
# Read legacy file
yaml_loader = YAML(str(self.legacy_schedule_file))
legacy_data = yaml_loader.data
if legacy_data and isinstance(legacy_data, dict):
# Create new structure with schedule as root key
new_data = {
"schedule": {
"type": legacy_data.get("type"),
"value": legacy_data.get("value"),
"disabled": legacy_data.get("disabled", False),
"updated_at": legacy_data.get("updated_at", datetime.now().isoformat()),
# Remove version field from new structure
}
}
# Write new settings file
tmp_path = self.settings_file.with_suffix(".yml.tmp")
yaml_writer = YAML(input_data="")
yaml_writer.data = new_data
yaml_writer.path = str(tmp_path)
yaml_writer.save()
os.replace(tmp_path, self.settings_file)
# Remove legacy file
self.legacy_schedule_file.unlink()
if not suppress_logging:
logger.info("Migrated legacy schedule.yml to qbm_settings.yml")
return True
else:
logger.warning("Invalid legacy schedule.yml structure, skipping migration")
return False
except Exception as e:
logger.error(f"Error migrating legacy schedule.yml: {e}")
return False
def _load_from_environment(self, suppress_logging: bool = False) -> bool:
"""Load schedule from environment variable as fallback."""
# If disabled, do not attempt env override unless we want an environment fallback # If disabled, do not attempt env override unless we want an environment fallback
if self._persistence_disabled: if self._persistence_disabled:
# Attempt env fallback only if present # Attempt env fallback only if present
@ -177,8 +243,9 @@ class Scheduler:
def save_schedule(self, schedule_type: str, schedule_value: Union[str, int]) -> bool: def save_schedule(self, schedule_type: str, schedule_value: Union[str, int]) -> bool:
""" """
Save schedule configuration to persistent file (includes disabled flag). Save schedule configuration to qbm_settings.yml (includes disabled flag).
Always re-enables persistence (disabled flag cleared) when an explicit save is requested. Always re-enables persistence (disabled flag cleared) when an explicit save is requested.
Uses new structure with 'schedule' root key.
""" """
if not self._validate_schedule(schedule_type, schedule_value): if not self._validate_schedule(schedule_type, schedule_value):
logger.error(f"Invalid schedule: {schedule_type}={schedule_value}") logger.error(f"Invalid schedule: {schedule_type}={schedule_value}")
@ -206,18 +273,20 @@ class Scheduler:
def toggle_persistence(self) -> bool: def toggle_persistence(self) -> bool:
""" """
Toggle persistent schedule enable/disable (non-destructive, stored in schedule.yml as disabled flag). Toggle persistent schedule enable/disable (non-destructive, stored in qbm_settings.yml under schedule.disabled).
Reduced logging (no stack trace). Reduced logging (no stack trace).
""" """
try: try:
# Load existing file data (if any) to preserve schedule type/value # Load existing file data (if any) to preserve schedule type/value
existing_type = None existing_type = None
existing_value = None existing_value = None
if self.schedule_file.exists(): if self.settings_file.exists():
file_data = self._read_schedule_file() file_data = self._read_schedule_file()
if file_data: if file_data:
existing_type = file_data.get("type") # Handle new structure with 'schedule' root key
existing_value = file_data.get("value") schedule_data = file_data.get("schedule", {})
existing_type = schedule_data.get("type")
existing_value = schedule_data.get("value")
if not self._persistence_disabled: if not self._persistence_disabled:
# Disable persistence (set disabled true, keep schedule metadata) # Disable persistence (set disabled true, keep schedule metadata)
@ -250,42 +319,47 @@ class Scheduler:
return False return False
def _read_schedule_file(self) -> Optional[dict[str, Any]]: def _read_schedule_file(self) -> Optional[dict[str, Any]]:
"""Read schedule data from file without modifying scheduler state.""" """Read qbm_settings.yml data from file without modifying scheduler state."""
if not self.schedule_file.exists(): if not self.settings_file.exists():
return None return None
try: try:
yaml_loader = YAML(str(self.schedule_file)) yaml_loader = YAML(str(self.settings_file))
data = yaml_loader.data data = yaml_loader.data
if data and isinstance(data, dict): if data and isinstance(data, dict):
return data return data
except Exception as e: except Exception as e:
logger.error(f"Error reading schedule file: {e}") logger.error(f"Error reading settings file: {e}")
return None return None
def get_schedule_info(self) -> dict[str, Any]: def get_schedule_info(self) -> dict[str, Any]:
"""Get detailed schedule information including source, persistence, and disabled state.""" """Get detailed schedule information including source, persistence, and disabled state from qbm_settings.yml."""
with self.lock: with self.lock:
disabled = self._persistence_disabled disabled = self._persistence_disabled
file_exists = self.schedule_file.exists() file_exists = self.settings_file.exists()
file_data = None file_data = None
if file_exists: if file_exists:
try: try:
file_data = self._read_schedule_file() file_data = self._read_schedule_file()
if file_data and bool(file_data.get("disabled")) != disabled: if file_data:
# Handle new structure with 'schedule' root key
schedule_data = file_data.get("schedule", {})
if bool(schedule_data.get("disabled")) != disabled:
# Keep in-memory flag consistent with file if manual edits occurred # Keep in-memory flag consistent with file if manual edits occurred
disabled = bool(file_data.get("disabled")) disabled = bool(schedule_data.get("disabled"))
self._persistence_disabled = disabled self._persistence_disabled = disabled
except Exception as e: except Exception as e:
logger.error(f"Error reading schedule file: {e}") logger.error(f"Error reading settings file: {e}")
if not disabled and file_data: if not disabled and file_data:
schedule_type = file_data.get("type") # Handle new structure with 'schedule' root key
schedule_value = file_data.get("value") schedule_data = file_data.get("schedule", {})
schedule_type = schedule_data.get("type")
schedule_value = schedule_data.get("value")
return { return {
"schedule": str(schedule_value), "schedule": str(schedule_value),
"type": schedule_type, "type": schedule_type,
"source": self.schedule_file.name, "source": self.settings_file.name,
"persistent": True, "persistent": True,
"file_exists": True, "file_exists": True,
"disabled": False, "disabled": False,
@ -495,22 +569,37 @@ class Scheduler:
def _persist_schedule_file(self, schedule_type: Optional[str], schedule_value: Optional[Union[str, int]]) -> None: def _persist_schedule_file(self, schedule_type: Optional[str], schedule_value: Optional[Union[str, int]]) -> None:
""" """
Internal helper to persist schedule.yml including disabled flag. Internal helper to persist qbm_settings.yml including disabled flag.
Uses new structure with 'schedule' root key. Preserves other settings in the file.
If schedule_type/value are None (e.g., user disabled before ever saving), we still write disabled state. If schedule_type/value are None (e.g., user disabled before ever saving), we still write disabled state.
""" """
data = { # Load existing settings file to preserve other settings
existing_data = {}
if self.settings_file.exists():
try:
yaml_loader = YAML(str(self.settings_file))
existing_data = yaml_loader.data or {}
except Exception:
# If we can't read existing file, start fresh
existing_data = {}
# Update schedule section
schedule_data = {
"type": schedule_type, "type": schedule_type,
"value": schedule_value, "value": schedule_value,
"disabled": self._persistence_disabled, "disabled": self._persistence_disabled,
"updated_at": datetime.now().isoformat(), "updated_at": datetime.now().isoformat(),
"version": 1,
} }
tmp_path = self.schedule_file.with_suffix(".yml.tmp")
# Merge with existing data
existing_data["schedule"] = schedule_data
tmp_path = self.settings_file.with_suffix(".yml.tmp")
yaml_writer = YAML(input_data="") yaml_writer = YAML(input_data="")
yaml_writer.data = data yaml_writer.data = existing_data
yaml_writer.path = str(tmp_path) yaml_writer.path = str(tmp_path)
yaml_writer.save() yaml_writer.save()
os.replace(tmp_path, self.schedule_file) os.replace(tmp_path, self.settings_file)
def _scheduler_loop(self): def _scheduler_loop(self):
"""Main scheduler loop running in background thread.""" """Main scheduler loop running in background thread."""

8
modules/util.py
View file

@ -253,7 +253,7 @@ def get_default_config_dir(config_hint: str = None, config_dir: str = None) -> s
2) If config_hint is an absolute path or contains a directory component, use its parent directory 2) If config_hint is an absolute path or contains a directory component, use its parent directory
3) Otherwise, if config_hint is a name/pattern (e.g. 'config.yml'), search common bases for: 3) Otherwise, if config_hint is a name/pattern (e.g. 'config.yml'), search common bases for:
- A direct match to that filename/pattern - A direct match to that filename/pattern
- OR a persisted scheduler file 'schedule.yml' (so we don't lose an existing schedule when config.yml is absent) - OR a persisted scheduler file 'qbm_settings.yml' or legacy 'schedule.yml' (so we don't lose an existing schedule)
Common bases (in order): Common bases (in order):
- /config (container volume) - /config (container volume)
- repository ./config - repository ./config
@ -288,8 +288,8 @@ def get_default_config_dir(config_hint: str = None, config_dir: str = None) -> s
for base in candidates: for base in candidates:
try: try:
# Match the primary pattern OR detect existing schedule.yml (persistence) # Match the primary pattern OR detect existing settings files (persistence)
if list(base.glob(primary)) or (base / "schedule.yml").exists(): if list(base.glob(primary)) or (base / "qbm_settings.yml").exists() or (base / "schedule.yml").exists():
return str(base.resolve()) return str(base.resolve())
except Exception: except Exception:
# ignore and continue to next base # ignore and continue to next base
@ -1625,7 +1625,7 @@ def get_matching_config_files(config_pattern: str, default_dir: str, use_config_
for config_file in glob_configs: for config_file in glob_configs:
filename = os.path.basename(config_file) filename = os.path.basename(config_file)
# Exclude reserved files # Exclude reserved files
if filename != "schedule.yml": if filename not in ("schedule.yml", "qbm_settings.yml"):
config_files.append(filename) config_files.append(filename)
if config_files: if config_files:

14
modules/web_api.py
View file

@ -1391,20 +1391,20 @@ class WebAPI:
parsed_value = schedule_value # cron parsed_value = schedule_value # cron
scheduler = Scheduler(self.default_dir, suppress_logging=True, read_only=True) scheduler = Scheduler(self.default_dir, suppress_logging=True, read_only=True)
existed_before = scheduler.schedule_file.exists() existed_before = scheduler.settings_file.exists()
prev_contents = None prev_contents = None
if existed_before: if existed_before:
try: try:
with open(scheduler.schedule_file, encoding="utf-8", errors="ignore") as f: with open(scheduler.settings_file, encoding="utf-8", errors="ignore") as f:
prev_contents = f.read().strip() prev_contents = f.read().strip()
except Exception: except Exception:
prev_contents = "<read_error>" prev_contents = "<read_error>"
success = scheduler.save_schedule(schedule_type, str(parsed_value)) success = scheduler.save_schedule(schedule_type, str(parsed_value))
new_size = None new_size = None
if scheduler.schedule_file.exists(): if scheduler.settings_file.exists():
try: try:
new_size = scheduler.schedule_file.stat().st_size new_size = scheduler.settings_file.stat().st_size
except Exception: except Exception:
pass pass
@ -1413,8 +1413,8 @@ class WebAPI:
raise HTTPException(status_code=500, detail="Failed to save schedule") raise HTTPException(status_code=500, detail="Failed to save schedule")
logger.debug( logger.debug(
f"UPDATE /schedule cid={correlation_id} persisted path={scheduler.schedule_file} " f"UPDATE /schedule cid={correlation_id} persisted path={scheduler.settings_file} "
f"existed_before={existed_before} new_exists={scheduler.schedule_file.exists()} " f"existed_before={existed_before} new_exists={scheduler.settings_file.exists()} "
f"new_size={new_size} prev_hash={hash(prev_contents) if prev_contents else None}" f"new_size={new_size} prev_hash={hash(prev_contents) if prev_contents else None}"
) )
@ -1452,7 +1452,7 @@ class WebAPI:
try: try:
correlation_id = uuid.uuid4().hex[:12] correlation_id = uuid.uuid4().hex[:12]
scheduler = Scheduler(self.default_dir, suppress_logging=True, read_only=True) scheduler = Scheduler(self.default_dir, suppress_logging=True, read_only=True)
file_exists_before = scheduler.schedule_file.exists() file_exists_before = scheduler.settings_file.exists()
# Execute toggle (scheduler emits single summary line internally) # Execute toggle (scheduler emits single summary line internally)
success = scheduler.toggle_persistence() success = scheduler.toggle_persistence()

4
qbit_manage.py
View file

@ -860,14 +860,14 @@ def main():
# Start scheduler for subsequent runs # Start scheduler for subsequent runs
scheduler.start(callback=start_loop) scheduler.start(callback=start_loop)
# Check if scheduler already has a schedule loaded from schedule.yml # Check if scheduler already has a schedule loaded from settings file
current_schedule = scheduler.get_current_schedule() current_schedule = scheduler.get_current_schedule()
if current_schedule: if current_schedule:
# Scheduler already loaded a schedule - determine the actual source # Scheduler already loaded a schedule - determine the actual source
schedule_info = scheduler.get_schedule_info() schedule_info = scheduler.get_schedule_info()
schedule_type, schedule_value = current_schedule schedule_type, schedule_value = current_schedule
source = schedule_info.get("source", "unknown") source = schedule_info.get("source", "unknown")
source_text = "persistent file" if source == "schedule.yml" else "environment" source_text = "persistent file" if source in ("qbm_settings.yml", "schedule.yml") else "environment"
# Use helper; already_configured=True because Scheduler loaded it # Use helper; already_configured=True because Scheduler loaded it
run_scheduled_mode(schedule_type, schedule_value, source_text, already_configured=True) run_scheduled_mode(schedule_type, schedule_value, source_text, already_configured=True)