diff --git a/Gemfile b/Gemfile index 72e09a076..a1d7073b7 100644 --- a/Gemfile +++ b/Gemfile @@ -81,6 +81,7 @@ gem 'delayed_job_active_record' gem 'devise-async', git: 'https://github.com/mhfs/devise-async.git', branch: 'devise-4.x' +gem 'rufus-scheduler', '~> 3.5' gem 'discard', '~> 1.0' diff --git a/Gemfile.lock b/Gemfile.lock index 12680e154..3b8db7b14 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -219,6 +219,8 @@ GEM doorkeeper (5.0.0) railties (>= 4.2) erubi (1.7.1) + et-orbi (1.1.7) + tzinfo execjs (2.7.0) factory_bot (4.8.2) activesupport (>= 3.0.0) @@ -232,6 +234,9 @@ GEM ffi (1.9.18) figaro (1.1.1) thor (~> 0.14) + fugit (1.1.7) + et-orbi (~> 1.1, >= 1.1.7) + raabro (~> 1.1) gherkin (5.0.0) globalid (0.4.1) activesupport (>= 4.2.0) @@ -357,6 +362,7 @@ GEM pry (>= 0.10.4) public_suffix (3.0.2) puma (3.11.2) + raabro (1.1.6) rack (2.0.6) rack-attack (5.4.1) rack (>= 1.0, < 3) @@ -443,6 +449,8 @@ GEM ruby-progressbar (1.10.0) ruby_dep (1.5.0) rubyzip (1.2.1) + rufus-scheduler (3.5.2) + fugit (~> 1.1, >= 1.1.5) sanitize (4.6.6) crass (~> 1.0.2) nokogiri (>= 1.4.4) @@ -610,6 +618,7 @@ DEPENDENCIES rubocop (>= 0.59.0) ruby-graphviz (~> 1.2) rubyzip + rufus-scheduler (~> 3.5) sanitize (~> 4.4) sass-rails (~> 5.0.6) scenic (~> 1.4) @@ -634,7 +643,7 @@ DEPENDENCIES yomu RUBY VERSION - ruby 2.4.4p296 + ruby 2.4.5p335 BUNDLED WITH - 1.16.6 + 1.17.2 diff --git a/app/assets/templates/.keep b/app/assets/templates/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/app/models/asset.rb b/app/models/asset.rb index abfb9caa5..0893645cf 100644 --- a/app/models/asset.rb +++ b/app/models/asset.rb @@ -73,7 +73,7 @@ class Asset < ApplicationRecord before_destroy :paperclip_delete, prepend: true after_save { result&.touch; step&.touch } - attr_accessor :file_content, :file_info, :preview_cached + attr_accessor :file_content, :file_info, :preview_cached, :in_template def file_empty(name, size) file_ext = name.split(".").last @@ -223,16 +223,17 @@ class Asset < ApplicationRecord # The extract_asset_text also includes # estimated size calculation Asset.delay(queue: :assets, run_at: 20.minutes.from_now) - .extract_asset_text(id) + .extract_asset_text(id, in_template) else # Update asset's estimated size immediately update_estimated_size(team) end end - def self.extract_asset_text(asset_id) + def self.extract_asset_text(asset_id, in_template = false) asset = find_by_id(asset_id) return unless asset.present? && asset.file.present? + asset.in_template = in_template begin file_path = asset.file.path @@ -293,12 +294,10 @@ class Asset < ApplicationRecord # If team is provided, its space_taken # is updated as well def update_estimated_size(team = nil) - if file_file_size.blank? - return - end + return if file_file_size.blank? || in_template es = file_file_size - if asset_text_datum.present? and asset_text_datum.persisted? then + if asset_text_datum.present? && asset_text_datum.persisted? asset_text_datum.reload es += get_octet_length_record(asset_text_datum, :data) es += get_octet_length_record(asset_text_datum, :data_vector) diff --git a/app/models/experiment.rb b/app/models/experiment.rb index 69d2926af..51dacecbd 100644 --- a/app/models/experiment.rb +++ b/app/models/experiment.rb @@ -40,6 +40,8 @@ class Experiment < ApplicationRecord validates :project, presence: true validates :created_by, presence: true validates :last_modified_by, presence: true + validates :uuid, uniqueness: { scope: :project }, + unless: proc { |e| e.uuid.blank? } with_options if: :archived do |experiment| experiment.validates :archived_by, presence: true experiment.validates :archived_on, presence: true diff --git a/app/models/project.rb b/app/models/project.rb index 73ffb8f4a..b9d45603e 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -49,6 +49,8 @@ class Project < ApplicationRecord end end) + scope :templates, -> { where(template: true) } + def self.visible_from_user_by_name(user, team, name) if user.is_admin_of_team? team return where('projects.archived IS FALSE AND projects.name ILIKE ?', diff --git a/app/models/team.rb b/app/models/team.rb index 3ade33d47..d6540cd11 100644 --- a/app/models/team.rb +++ b/app/models/team.rb @@ -6,6 +6,8 @@ class Team < ApplicationRecord # output in space_taken related functions include ActionView::Helpers::NumberHelper + after_create :generate_template_project + auto_strip_attributes :name, :description, nullify: false validates :name, length: { minimum: Constants::NAME_MIN_LENGTH, @@ -304,4 +306,20 @@ class Team < ApplicationRecord def protocol_keywords_list ProtocolKeyword.where(team: self).pluck(:name) end + + private + + def generate_template_project + user = created_by + return unless user + Project.transaction do + tmpl_project = projects.create!( + name: Constants::TEMPLATES_PROJECT_NAME, + visibility: :visible, + template: true + ) + tmpl_project.user_projects.create!(user: user, role: 0) + TemplatesService.new.update_project(tmpl_project) + end + end end diff --git a/app/services/model_exporters/experiment_exporter.rb b/app/services/model_exporters/experiment_exporter.rb index d6f236a56..f37b1ccc2 100644 --- a/app/services/model_exporters/experiment_exporter.rb +++ b/app/services/model_exporters/experiment_exporter.rb @@ -9,9 +9,10 @@ module ModelExporters @assets_to_copy = [] end - def export_to_dir + def export_template_to_dir @asset_counter = 0 @experiment.transaction(isolation: :serializable) do + @experiment.uuid ||= SecureRandom.uuid @dir_to_export = FileUtils.mkdir_p( File.join("tmp/experiment_#{@experiment.id}" \ "_export_#{Time.now.to_i}") @@ -19,11 +20,12 @@ module ModelExporters # Writing JSON file with experiment structure File.write( - File.join(@dir_to_export, 'experiment_export.json'), + File.join(@dir_to_export, 'experiment.json'), experiment[0].to_json ) # Copying assets - copy_files(@assets_to_copy, :file, File.join(@dir_to_export, 'assets')) do + assets_dir = File.join(@dir_to_export, 'assets') + copy_files(@assets_to_copy, :file, assets_dir) do @asset_counter += 1 end puts "Exported assets: #{@asset_counter}" diff --git a/app/services/team_importer.rb b/app/services/team_importer.rb index 5bd37669d..147d3b3e0 100644 --- a/app/services/team_importer.rb +++ b/app/services/team_importer.rb @@ -143,7 +143,7 @@ class TeamImporter end end - def import_template_experiment_from_dir(import_dir, project_id, + def import_experiment_template_from_dir(import_dir, project_id, user_id) # Remove callbacks that can cause problems when importing MyModule.skip_callback(:create, :before, :create_blank_protocol) @@ -154,56 +154,43 @@ class TeamImporter # Parse the experiment file and save it to DB project = Project.find_by_id(project_id) - experiment_json = JSON.parse(File.read("#{@import_dir}/experiment_export.json")) - experiment = create_experiment(experiment_json, project, user_id) + experiment_json = JSON.parse(File.read("#{@import_dir}/experiment.json")) - # Create connections for modules - experiment_json['my_modules'].each do |my_module_json| - create_task_connections(my_module_json['outputs']) + # Handle situation when experimaent with same name already exists + exp_name = experiment_json.dig('experiment', 'name') + if project.experiments.where(name: exp_name).present? + experiment_names = project.experiments.map(&:name) + i = 1 + new_name = "#{exp_name} (#{i})" + i += 1 while experiment_names.include?(new_name) + experiment_json['experiment']['name'] = new_name end + ActiveRecord::Base.transaction do + ActiveRecord::Base.no_touching do + experiment = create_experiment(experiment_json, project, user_id) - # Create UUID for the template experiment - # TODO: this requires DB migration first - # experiment.template_uid = Digest::SHA256.new("#{expirement.name + experiment.created_at}") - # experiment.save! + # Create connections for modules + experiment_json['my_modules'].each do |my_module_json| + create_task_connections(my_module_json['outputs']) + end + update_smart_annotations_in_project(project) + + puts "Imported experiment: #{experiment.id}" + return experiment + end + end + ensure # Reset callbacks MyModule.set_callback(:create, :before, :create_blank_protocol) Protocol.set_callback(:save, :after, :update_linked_children) - - puts "Imported experiment: #{experiment.id}" - experiment end private def update_smart_annotations(team) team.projects.each do |pr| - pr.project_comments.each do |comment| - comment.save! if update_annotation(comment.message) - end - pr.experiments.each do |exp| - exp.save! if update_annotation(exp.description) - exp.my_modules.each do |task| - task.task_comments.each do |comment| - comment.save! if update_annotation(comment.message) - end - task.save! if update_annotation(task.description) - task.protocol.steps.each do |step| - step.step_comments.each do |comment| - comment.save! if update_annotation(comment.message) - end - step.save! if update_annotation(step.description) - end - task.results.each do |res| - res.result_comments.each do |comment| - comment.save! if update_annotation(comment.message) - end - next unless res.result_text - res.save! if update_annotation(res.result_text.text) - end - end - end + update_smart_annotations_in_project(pr) end team.protocols.where(my_module: nil).each do |protocol| protocol.steps.each do |step| @@ -222,6 +209,34 @@ class TeamImporter end end + def update_smart_annotations_in_project(project) + project.project_comments.each do |comment| + comment.save! if update_annotation(comment.message) + end + project.experiments.each do |exp| + exp.save! if update_annotation(exp.description) + exp.my_modules.each do |task| + task.task_comments.each do |comment| + comment.save! if update_annotation(comment.message) + end + task.save! if update_annotation(task.description) + task.protocol.steps.each do |step| + step.step_comments.each do |comment| + comment.save! if update_annotation(comment.message) + end + step.save! if update_annotation(step.description) + end + task.results.each do |res| + res.result_comments.each do |comment| + comment.save! if update_annotation(comment.message) + end + next unless res.result_text + res.save! if update_annotation(res.result_text.text) + end + end + end + end + # Returns true if text was updated def update_annotation(text) return false if text.nil? @@ -761,7 +776,9 @@ class TeamImporter user_id || find_user(asset.last_modified_by_id) asset.team = team asset.file = file + asset.in_template = true if @is_template asset.save! + asset.post_process_file(team) @asset_mappings[orig_asset_id] = asset.id @asset_counter += 1 end diff --git a/app/services/templates_service.rb b/app/services/templates_service.rb new file mode 100644 index 000000000..d3d9dbbd3 --- /dev/null +++ b/app/services/templates_service.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +class TemplatesService + def initialize + templates_dir_pattern = "#{Rails.root}/app/assets/templates/experiment_*/" + @experiment_templates = {} + Dir.glob(templates_dir_pattern).each do |tmplt_dir| + id = /[0-9]+/.match(tmplt_dir.split('/').last)[0] + uuid = /\"uuid\":\"([a-fA-F0-9\-]{36})\"/ + .match(File.read(tmplt_dir + 'experiment.json'))[1] + @experiment_templates[uuid] = id.to_i + end + end + + def update_project(project) + return unless project.template == true + owner = project.user_projects + .where(role: 'owner') + .order(:created_at) + .first + .user + return unless owner.present? + updated = false + exp_tmplt_dir_prefix = "#{Rails.root}/app/assets/templates/experiment_" + existing = project.experiments.where.not(uuid: nil).pluck(:uuid) + @experiment_templates.except(*existing).each_value do |id| + importer_service = TeamImporter.new + importer_service.import_experiment_template_from_dir( + exp_tmplt_dir_prefix + id.to_s, project.id, owner.id + ) + updated = true + end + updated + end + + def update_all_projects + processed_counter = 0 + updated_counter = 0 + Project.where(template: true).find_each do |project| + processed_counter += 1 + updated_counter += 1 if update_project(project) + end + [updated_counter, processed_counter] + end +end diff --git a/config/initializers/constants.rb b/config/initializers/constants.rb index 2de7387cb..fc656f499 100644 --- a/config/initializers/constants.rb +++ b/config/initializers/constants.rb @@ -908,6 +908,8 @@ class Constants # Team name for default admin user DEFAULT_PRIVATE_TEAM_NAME = 'My projects'.freeze + TEMPLATES_PROJECT_NAME = 'Templates'.freeze + # ) \ / ( # /|\ )\_/( /|\ # * / | \ (/\|/\) / | \ * diff --git a/config/initializers/scheduler.rb b/config/initializers/scheduler.rb new file mode 100644 index 000000000..9ed3f35aa --- /dev/null +++ b/config/initializers/scheduler.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +require 'rufus-scheduler' + +scheduler = Rufus::Scheduler.singleton + +if ENV['ENABLE_TEMPLATES_SYNC'] && ARGV[0] == 'jobs:work' + # Templates sync periodic task + scheduler.every '1h' do + Rails.logger.info('Templates, syncing all template projects') + updated, total = TemplatesService.new.update_all_projects + Rails.logger.info( + "Templates, total number of updated projects: #{updated} out of #{total}}" + ) + Rails.logger.flush + end +end diff --git a/db/migrate/20190116101127_add_project_templates.rb b/db/migrate/20190116101127_add_project_templates.rb new file mode 100644 index 000000000..ec9e3526b --- /dev/null +++ b/db/migrate/20190116101127_add_project_templates.rb @@ -0,0 +1,6 @@ +class AddProjectTemplates < ActiveRecord::Migration[5.1] + def change + add_column :projects, :template, :boolean + add_column :experiments, :uuid, :uuid + end +end diff --git a/db/schema.rb b/db/schema.rb index ab055cf53..b2cf25a8f 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20181212162649) do +ActiveRecord::Schema.define(version: 20190116101127) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -163,6 +163,7 @@ ActiveRecord::Schema.define(version: 20181212162649) do t.string "workflowimg_content_type" t.integer "workflowimg_file_size" t.datetime "workflowimg_updated_at" + t.uuid "uuid" t.index ["archived_by_id"], name: "index_experiments_on_archived_by_id" t.index ["created_by_id"], name: "index_experiments_on_created_by_id" t.index ["last_modified_by_id"], name: "index_experiments_on_last_modified_by_id" @@ -296,6 +297,7 @@ ActiveRecord::Schema.define(version: 20181212162649) do t.integer "restored_by_id" t.datetime "restored_on" t.string "experiments_order" + t.boolean "template" t.index "trim_html_tags((name)::text) gin_trgm_ops", name: "index_projects_on_name", using: :gin t.index ["archived_by_id"], name: "index_projects_on_archived_by_id" t.index ["created_by_id"], name: "index_projects_on_created_by_id" diff --git a/lib/tasks/data.rake b/lib/tasks/data.rake index d206f6068..2f8f7f6e8 100644 --- a/lib/tasks/data.rake +++ b/lib/tasks/data.rake @@ -109,21 +109,30 @@ namespace :data do end desc 'Export experiment to directory' - task :experiment_export, [:experiment_id] => [:environment] do |_, args| + task :experiment_template_export, [:experiment_id] => [:environment] do |_, args| Rails.logger.info( - "Exporting experiment with ID:#{args[:experiment_id]} to directory in tmp" + "Exporting experiment template with ID:#{args[:experiment_id]} to directory in tmp" ) ee = ModelExporters::ExperimentExporter.new(args[:experiment_id]) - ee&.export_to_dir + ee&.export_template_to_dir end desc 'Import experiment from directory to given project' - task :experiment_import, %i(dir_path project_id user_id) => [:environment] do |_, args| + task :experiment_template_import, %i(dir_path project_id user_id) => [:environment] do |_, args| Rails.logger.info( "Importing experiment from directory #{args[:dir_path]}" ) - TeamImporter.new.import_template_experiment_from_dir(args[:dir_path], + TeamImporter.new.import_experiment_template_from_dir(args[:dir_path], args[:project_id], args[:user_id]) end + + desc 'Update all templates projects' + task update_all_templates: :environment do + Rails.logger.info('Templates, syncing all templates projects') + updated, total = TemplatesService.new.update_all_projects + Rails.logger.info( + "Templates, total number of updated projects: #{updated} out of #{total}}" + ) + end end diff --git a/spec/models/experiment_spec.rb b/spec/models/experiment_spec.rb index d30877fa4..6b787ece0 100644 --- a/spec/models/experiment_spec.rb +++ b/spec/models/experiment_spec.rb @@ -23,6 +23,7 @@ describe Experiment, type: :model do it { should have_db_column :workflowimg_content_type } it { should have_db_column :workflowimg_file_size } it { should have_db_column :workflowimg_updated_at } + it { should have_db_column :uuid } end describe 'Relations' do @@ -64,5 +65,21 @@ describe Experiment, type: :model do last_modified_by: project.created_by expect(new_exp).to_not be_valid end + + it 'should have uniq uuid scoped on project' do + uuid = SecureRandom.uuid + puts uuid + create :experiment, name: 'experiment', + project: project, + created_by: project.created_by, + last_modified_by: project.created_by, + uuid: uuid + new_exp = build_stubbed :experiment, name: 'new experiment', + project: project, + created_by: project.created_by, + last_modified_by: project.created_by, + uuid: uuid + expect(new_exp).to_not be_valid + end end end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 53270d9b5..c3f043d99 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -21,6 +21,7 @@ describe Project, type: :model do it { should have_db_column :restored_by_id } it { should have_db_column :restored_on } it { should have_db_column :experiments_order } + it { should have_db_column :template } end describe 'Relations' do