From 7f7ece2be0b4847a0d424e24a92470f03ec7d8ba Mon Sep 17 00:00:00 2001 From: Oleksii Kriuchykhin Date: Fri, 21 Sep 2018 17:46:18 +0200 Subject: [PATCH] Improve delayed jobs for TempFile deletion and Asset text extraction [SCI-2738] --- app/controllers/teams_controller.rb | 2 +- app/models/asset.rb | 36 ++++++++++--------- app/models/temp_file.rb | 14 +++++--- .../import_repository/parse_repository.rb | 2 +- 4 files changed, 31 insertions(+), 23 deletions(-) diff --git a/app/controllers/teams_controller.rb b/app/controllers/teams_controller.rb index 925ca8f10..3fed1278c 100644 --- a/app/controllers/teams_controller.rb +++ b/app/controllers/teams_controller.rb @@ -39,7 +39,7 @@ class TeamsController < ApplicationController ) if @temp_file.save - @temp_file.destroy_obsolete + TempFile.destroy_obsolete(@temp_file.id) respond_to do |format| format.json do render json: { diff --git a/app/models/asset.rb b/app/models/asset.rb index cbe9fc636..98f17659f 100644 --- a/app/models/asset.rb +++ b/app/models/asset.rb @@ -221,48 +221,52 @@ class Asset < ApplicationRecord Rails.logger.info "Asset #{id}: Creating extract text job" # The extract_asset_text also includes # estimated size calculation - delay(queue: :assets, run_at: 20.minutes.from_now) - .extract_asset_text(team) + Asset.delay(queue: :assets, run_at: 20.minutes.from_now) + .extract_asset_text(id) else # Update asset's estimated size immediately update_estimated_size(team) end end - def extract_asset_text(team = nil) - return if file.blank? + def self.extract_asset_text(asset_id) + asset = find_by_id(asset_id) + return unless asset.present? && asset.file.present? begin - file_path = file.path + file_path = asset.file.path - if file.is_stored_on_s3? - fa = file.fetch + if asset.file.is_stored_on_s3? + fa = asset.file.fetch file_path = fa.path end - if (!Yomu.class_eval('@@server_pid')) - Yomu.server(:text,nil) + unless Yomu.class_eval('@@server_pid') + Yomu.server(:text, nil) sleep(5) end y = Yomu.new file_path text_data = y.text - if asset_text_datum.present? + if asset.asset_text_datum.present? # Update existing text datum if it exists - asset_text_datum.update(data: text_data) + asset.asset_text_datum.update(data: text_data) else # Create new text datum - AssetTextDatum.create(data: text_data, asset: self) + AssetTextDatum.create(data: text_data, asset: asset) end - Rails.logger.info "Asset #{id}: Asset file successfully extracted" + Rails.logger.info "Asset #{asset.id}: Asset file successfully extracted" # Finally, update asset's estimated size to include # the data vector - update_estimated_size(team) - rescue Exception => e - Rails.logger.fatal "Asset #{id}: Error extracting contents from asset file #{file.path}: " + e.message + asset.update_estimated_size(asset.team) + rescue StandardError => e + Rails.logger.fatal( + "Asset #{asset.id}: Error extracting contents from asset "\ + "file #{asset.file.path}: #{e.message}" + ) ensure File.delete file_path if fa end diff --git a/app/models/temp_file.rb b/app/models/temp_file.rb index 0cc3784ee..a1d6cb2f7 100644 --- a/app/models/temp_file.rb +++ b/app/models/temp_file.rb @@ -4,10 +4,14 @@ class TempFile < ApplicationRecord has_attached_file :file do_not_validate_attachment_file_type :file - def destroy_obsolete - destroy! if self - end + class << self + def destroy_obsolete(temp_file_id) + temp_file = find_by_id(temp_file_id) + return unless temp_file.present? + temp_file.destroy! + end - handle_asynchronously :destroy_obsolete, - run_at: proc { 7.days.from_now } + handle_asynchronously :destroy_obsolete, + run_at: proc { 7.days.from_now } + end end diff --git a/app/services/import_repository/parse_repository.rb b/app/services/import_repository/parse_repository.rb index 24604da63..23c6e1fc1 100644 --- a/app/services/import_repository/parse_repository.rb +++ b/app/services/import_repository/parse_repository.rb @@ -36,7 +36,7 @@ module ImportRepository ) if temp_file.save - temp_file.destroy_obsolete + TempFile.destroy_obsolete(temp_file.id) return temp_file end end