Add templates business logic [SCI-2939]

Gemfile

@@ -81,6 +81,7 @@ gem 'delayed_job_active_record'
 gem 'devise-async',
   git: 'https://github.com/mhfs/devise-async.git',
   branch: 'devise-4.x'
+gem 'rufus-scheduler', '~> 3.5'
 
 gem 'discard', '~> 1.0'
 

Gemfile.lock

@@ -219,6 +219,8 @@ GEM
     doorkeeper (5.0.0)
       railties (>= 4.2)
     erubi (1.7.1)
+    et-orbi (1.1.7)
+      tzinfo
     execjs (2.7.0)
     factory_bot (4.8.2)
       activesupport (>= 3.0.0)
@@ -232,6 +234,9 @@ GEM
     ffi (1.9.18)
     figaro (1.1.1)
       thor (~> 0.14)
+    fugit (1.1.7)
+      et-orbi (~> 1.1, >= 1.1.7)
+      raabro (~> 1.1)
     gherkin (5.0.0)
     globalid (0.4.1)
       activesupport (>= 4.2.0)
@@ -357,6 +362,7 @@ GEM
       pry (>= 0.10.4)
     public_suffix (3.0.2)
     puma (3.11.2)
+    raabro (1.1.6)
     rack (2.0.6)
     rack-attack (5.4.1)
       rack (>= 1.0, < 3)
@@ -443,6 +449,8 @@ GEM
     ruby-progressbar (1.10.0)
     ruby_dep (1.5.0)
     rubyzip (1.2.1)
+    rufus-scheduler (3.5.2)
+      fugit (~> 1.1, >= 1.1.5)
     sanitize (4.6.6)
       crass (~> 1.0.2)
       nokogiri (>= 1.4.4)
@@ -610,6 +618,7 @@ DEPENDENCIES
   rubocop (>= 0.59.0)
   ruby-graphviz (~> 1.2)
   rubyzip
+  rufus-scheduler (~> 3.5)
   sanitize (~> 4.4)
   sass-rails (~> 5.0.6)
   scenic (~> 1.4)
@@ -634,7 +643,7 @@ DEPENDENCIES
   yomu
 
 RUBY VERSION
-   ruby 2.4.4p296
+   ruby 2.4.5p335
 
 BUNDLED WITH
-   1.16.6
+   1.17.2

app/models/asset.rb

@@ -73,7 +73,7 @@ class Asset < ApplicationRecord
   before_destroy :paperclip_delete, prepend: true
   after_save { result&.touch; step&.touch }
 
-  attr_accessor :file_content, :file_info, :preview_cached
+  attr_accessor :file_content, :file_info, :preview_cached, :in_template
 
   def file_empty(name, size)
     file_ext = name.split(".").last
@@ -223,16 +223,17 @@ class Asset < ApplicationRecord
       # The extract_asset_text also includes
       # estimated size calculation
       Asset.delay(queue: :assets, run_at: 20.minutes.from_now)
-           .extract_asset_text(id)
+           .extract_asset_text(id, in_template)
     else
       # Update asset's estimated size immediately
       update_estimated_size(team)
     end
   end
 
-  def self.extract_asset_text(asset_id)
+  def self.extract_asset_text(asset_id, in_template = false)
     asset = find_by_id(asset_id)
     return unless asset.present? && asset.file.present?
+    asset.in_template = in_template
 
     begin
       file_path = asset.file.path
@@ -293,12 +294,10 @@ class Asset < ApplicationRecord
   # If team is provided, its space_taken
   # is updated as well
   def update_estimated_size(team = nil)
-    if file_file_size.blank?
-      return
-    end
+    return if file_file_size.blank? || in_template
 
     es = file_file_size
-    if asset_text_datum.present? and asset_text_datum.persisted? then
+    if asset_text_datum.present? && asset_text_datum.persisted?
       asset_text_datum.reload
       es += get_octet_length_record(asset_text_datum, :data)
       es += get_octet_length_record(asset_text_datum, :data_vector)

app/models/experiment.rb

@@ -40,6 +40,8 @@ class Experiment < ApplicationRecord
   validates :project, presence: true
   validates :created_by, presence: true
   validates :last_modified_by, presence: true
+  validates :uuid, uniqueness: { scope: :project },
+                   unless: proc { |e| e.uuid.blank? }
   with_options if: :archived do |experiment|
     experiment.validates :archived_by, presence: true
     experiment.validates :archived_on, presence: true

app/models/project.rb

@@ -49,6 +49,8 @@ class Project < ApplicationRecord
                         end
                       end)
 
+  scope :templates, -> { where(template: true) }
+
   def self.visible_from_user_by_name(user, team, name)
     if user.is_admin_of_team? team
       return where('projects.archived IS FALSE AND projects.name ILIKE ?',

app/models/team.rb

@@ -6,6 +6,8 @@ class Team < ApplicationRecord
   # output in space_taken related functions
   include ActionView::Helpers::NumberHelper
 
+  after_create :generate_template_project
+
   auto_strip_attributes :name, :description, nullify: false
   validates :name,
             length: { minimum: Constants::NAME_MIN_LENGTH,
@@ -304,4 +306,20 @@ class Team < ApplicationRecord
   def protocol_keywords_list
     ProtocolKeyword.where(team: self).pluck(:name)
   end
+
+  private
+
+  def generate_template_project
+    user = created_by
+    return unless user
+    Project.transaction do
+      tmpl_project = projects.create!(
+        name: Constants::TEMPLATES_PROJECT_NAME,
+        visibility: :visible,
+        template: true
+      )
+      tmpl_project.user_projects.create!(user: user, role: 0)
+      TemplatesService.new.update_project(tmpl_project)
+    end
+  end
 end

app/services/model_exporters/experiment_exporter.rb

@@ -9,9 +9,10 @@ module ModelExporters
       @assets_to_copy = []
     end
 
-    def export_to_dir
+    def export_template_to_dir
       @asset_counter = 0
       @experiment.transaction(isolation: :serializable) do
+        @experiment.uuid ||= SecureRandom.uuid
         @dir_to_export = FileUtils.mkdir_p(
           File.join("tmp/experiment_#{@experiment.id}" \
                     "_export_#{Time.now.to_i}")
@@ -19,11 +20,12 @@ module ModelExporters
 
         # Writing JSON file with experiment structure
         File.write(
-          File.join(@dir_to_export, 'experiment_export.json'),
+          File.join(@dir_to_export, 'experiment.json'),
           experiment[0].to_json
         )
         # Copying assets
-        copy_files(@assets_to_copy, :file, File.join(@dir_to_export, 'assets')) do
+        assets_dir = File.join(@dir_to_export, 'assets')
+        copy_files(@assets_to_copy, :file, assets_dir) do
           @asset_counter += 1
         end
         puts "Exported assets: #{@asset_counter}"

app/services/team_importer.rb

@@ -143,7 +143,7 @@ class TeamImporter
     end
   end
 
-  def import_template_experiment_from_dir(import_dir, project_id,
+  def import_experiment_template_from_dir(import_dir, project_id,
                                           user_id)
     # Remove callbacks that can cause problems when importing
     MyModule.skip_callback(:create, :before, :create_blank_protocol)
@@ -154,56 +154,43 @@ class TeamImporter
 
     # Parse the experiment file and save it to DB
     project = Project.find_by_id(project_id)
-    experiment_json = JSON.parse(File.read("#{@import_dir}/experiment_export.json"))
-    experiment = create_experiment(experiment_json, project, user_id)
+    experiment_json = JSON.parse(File.read("#{@import_dir}/experiment.json"))
 
-    # Create connections for modules
-    experiment_json['my_modules'].each do |my_module_json|
-      create_task_connections(my_module_json['outputs'])
+    # Handle situation when experimaent with same name already exists
+    exp_name = experiment_json.dig('experiment', 'name')
+    if project.experiments.where(name: exp_name).present?
+      experiment_names = project.experiments.map(&:name)
+      i = 1
+      new_name = "#{exp_name} (#{i})"
+      i += 1 while experiment_names.include?(new_name)
+      experiment_json['experiment']['name'] = new_name
     end
+    ActiveRecord::Base.transaction do
+      ActiveRecord::Base.no_touching do
+        experiment = create_experiment(experiment_json, project, user_id)
 
-    # Create UUID for the template experiment
-    # TODO: this requires DB migration first
-    # experiment.template_uid = Digest::SHA256.new("#{expirement.name + experiment.created_at}")
-    # experiment.save!
+        # Create connections for modules
+        experiment_json['my_modules'].each do |my_module_json|
+          create_task_connections(my_module_json['outputs'])
+        end
 
+        update_smart_annotations_in_project(project)
+
+        puts "Imported experiment: #{experiment.id}"
+        return experiment
+      end
+    end
+  ensure
     # Reset callbacks
     MyModule.set_callback(:create, :before, :create_blank_protocol)
     Protocol.set_callback(:save, :after, :update_linked_children)
-
-    puts "Imported experiment: #{experiment.id}"
-    experiment
   end
 
   private
 
   def update_smart_annotations(team)
     team.projects.each do |pr|
-      pr.project_comments.each do |comment|
-        comment.save! if update_annotation(comment.message)
-      end
-      pr.experiments.each do |exp|
-        exp.save! if update_annotation(exp.description)
-        exp.my_modules.each do |task|
-          task.task_comments.each do |comment|
-            comment.save! if update_annotation(comment.message)
-          end
-          task.save! if update_annotation(task.description)
-          task.protocol.steps.each do |step|
-            step.step_comments.each do |comment|
-              comment.save! if update_annotation(comment.message)
-            end
-            step.save! if update_annotation(step.description)
-          end
-          task.results.each do |res|
-            res.result_comments.each do |comment|
-              comment.save! if update_annotation(comment.message)
-            end
-            next unless res.result_text
-            res.save! if update_annotation(res.result_text.text)
-          end
-        end
-      end
+      update_smart_annotations_in_project(pr)
     end
     team.protocols.where(my_module: nil).each do |protocol|
       protocol.steps.each do |step|
@@ -222,6 +209,34 @@ class TeamImporter
     end
   end
 
+  def update_smart_annotations_in_project(project)
+    project.project_comments.each do |comment|
+      comment.save! if update_annotation(comment.message)
+    end
+    project.experiments.each do |exp|
+      exp.save! if update_annotation(exp.description)
+      exp.my_modules.each do |task|
+        task.task_comments.each do |comment|
+          comment.save! if update_annotation(comment.message)
+        end
+        task.save! if update_annotation(task.description)
+        task.protocol.steps.each do |step|
+          step.step_comments.each do |comment|
+            comment.save! if update_annotation(comment.message)
+          end
+          step.save! if update_annotation(step.description)
+        end
+        task.results.each do |res|
+          res.result_comments.each do |comment|
+            comment.save! if update_annotation(comment.message)
+          end
+          next unless res.result_text
+          res.save! if update_annotation(res.result_text.text)
+        end
+      end
+    end
+  end
+
   # Returns true if text was updated
   def update_annotation(text)
     return false if text.nil?
@@ -761,7 +776,9 @@ class TeamImporter
         user_id || find_user(asset.last_modified_by_id)
       asset.team = team
       asset.file = file
+      asset.in_template = true if @is_template
       asset.save!
+      asset.post_process_file(team)
       @asset_mappings[orig_asset_id] = asset.id
       @asset_counter += 1
     end

app/services/templates_service.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+class TemplatesService
+  def initialize
+    templates_dir_pattern = "#{Rails.root}/app/assets/templates/experiment_*/"
+    @experiment_templates = {}
+    Dir.glob(templates_dir_pattern).each do |tmplt_dir|
+      id = /[0-9]+/.match(tmplt_dir.split('/').last)[0]
+      uuid = /\"uuid\":\"([a-fA-F0-9\-]{36})\"/
+             .match(File.read(tmplt_dir + 'experiment.json'))[1]
+      @experiment_templates[uuid] = id.to_i
+    end
+  end
+
+  def update_project(project)
+    return unless project.template == true
+    owner = project.user_projects
+                   .where(role: 'owner')
+                   .order(:created_at)
+                   .first
+                   .user
+    return unless owner.present?
+    updated = false
+    exp_tmplt_dir_prefix = "#{Rails.root}/app/assets/templates/experiment_"
+    existing = project.experiments.where.not(uuid: nil).pluck(:uuid)
+    @experiment_templates.except(*existing).each_value do |id|
+      importer_service = TeamImporter.new
+      importer_service.import_experiment_template_from_dir(
+        exp_tmplt_dir_prefix + id.to_s, project.id, owner.id
+      )
+      updated = true
+    end
+    updated
+  end
+
+  def update_all_projects
+    processed_counter = 0
+    updated_counter = 0
+    Project.where(template: true).find_each do |project|
+      processed_counter += 1
+      updated_counter += 1 if update_project(project)
+    end
+    [updated_counter, processed_counter]
+  end
+end

config/initializers/constants.rb

@@ -908,6 +908,8 @@ class Constants
   # Team name for default admin user
   DEFAULT_PRIVATE_TEAM_NAME = 'My projects'.freeze
 
+  TEMPLATES_PROJECT_NAME = 'Templates'.freeze
+
   #                             )       \   /      (
   #                            /|\      )\_/(     /|\
   #   *                       / | \    (/\|/\)   / | \                      *

config/initializers/scheduler.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require 'rufus-scheduler'
+
+scheduler = Rufus::Scheduler.singleton
+
+if ENV['ENABLE_TEMPLATES_SYNC'] && ARGV[0] == 'jobs:work'
+  # Templates sync periodic task
+  scheduler.every '1h' do
+    Rails.logger.info('Templates, syncing all template projects')
+    updated, total = TemplatesService.new.update_all_projects
+    Rails.logger.info(
+      "Templates, total number of updated projects: #{updated} out of #{total}}"
+    )
+    Rails.logger.flush
+  end
+end

db/migrate/20190116101127_add_project_templates.rb

@@ -0,0 +1,6 @@
+class AddProjectTemplates < ActiveRecord::Migration[5.1]
+  def change
+    add_column :projects, :template, :boolean
+    add_column :experiments, :uuid, :uuid
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20181212162649) do
+ActiveRecord::Schema.define(version: 20190116101127) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -163,6 +163,7 @@ ActiveRecord::Schema.define(version: 20181212162649) do
     t.string "workflowimg_content_type"
     t.integer "workflowimg_file_size"
     t.datetime "workflowimg_updated_at"
+    t.uuid "uuid"
     t.index ["archived_by_id"], name: "index_experiments_on_archived_by_id"
     t.index ["created_by_id"], name: "index_experiments_on_created_by_id"
     t.index ["last_modified_by_id"], name: "index_experiments_on_last_modified_by_id"
@@ -296,6 +297,7 @@ ActiveRecord::Schema.define(version: 20181212162649) do
     t.integer "restored_by_id"
     t.datetime "restored_on"
     t.string "experiments_order"
+    t.boolean "template"
     t.index "trim_html_tags((name)::text) gin_trgm_ops", name: "index_projects_on_name", using: :gin
     t.index ["archived_by_id"], name: "index_projects_on_archived_by_id"
     t.index ["created_by_id"], name: "index_projects_on_created_by_id"

lib/tasks/data.rake

@@ -109,21 +109,30 @@ namespace :data do
   end
 
   desc 'Export experiment to directory'
-  task :experiment_export, [:experiment_id] => [:environment] do |_, args|
+  task :experiment_template_export, [:experiment_id] => [:environment] do |_, args|
     Rails.logger.info(
-      "Exporting experiment with ID:#{args[:experiment_id]} to directory in tmp"
+      "Exporting experiment template with ID:#{args[:experiment_id]} to directory in tmp"
     )
     ee = ModelExporters::ExperimentExporter.new(args[:experiment_id])
-    ee&.export_to_dir
+    ee&.export_template_to_dir
   end
 
   desc 'Import experiment from directory to given project'
-  task :experiment_import, %i(dir_path project_id user_id) => [:environment] do |_, args|
+  task :experiment_template_import, %i(dir_path project_id user_id) => [:environment] do |_, args|
     Rails.logger.info(
       "Importing experiment from directory #{args[:dir_path]}"
     )
-    TeamImporter.new.import_template_experiment_from_dir(args[:dir_path],
+    TeamImporter.new.import_experiment_template_from_dir(args[:dir_path],
                                                          args[:project_id],
                                                          args[:user_id])
   end
+
+  desc 'Update all templates projects'
+  task update_all_templates: :environment do
+    Rails.logger.info('Templates, syncing all templates projects')
+    updated, total = TemplatesService.new.update_all_projects
+    Rails.logger.info(
+      "Templates, total number of updated projects: #{updated} out of #{total}}"
+    )
+  end
 end

spec/models/experiment_spec.rb

@@ -23,6 +23,7 @@ describe Experiment, type: :model do
     it { should have_db_column :workflowimg_content_type }
     it { should have_db_column :workflowimg_file_size }
     it { should have_db_column :workflowimg_updated_at }
+    it { should have_db_column :uuid }
   end
 
   describe 'Relations' do
@@ -64,5 +65,21 @@ describe Experiment, type: :model do
                                            last_modified_by: project.created_by
       expect(new_exp).to_not be_valid
     end
+
+    it 'should have uniq uuid scoped on project' do
+      uuid = SecureRandom.uuid
+      puts uuid
+      create :experiment, name: 'experiment',
+                          project: project,
+                          created_by: project.created_by,
+                          last_modified_by: project.created_by,
+                          uuid: uuid
+      new_exp = build_stubbed :experiment, name: 'new experiment',
+                                           project: project,
+                                           created_by: project.created_by,
+                                           last_modified_by: project.created_by,
+                                           uuid: uuid
+      expect(new_exp).to_not be_valid
+    end
   end
 end

spec/models/project_spec.rb

@@ -21,6 +21,7 @@ describe Project, type: :model do
     it { should have_db_column :restored_by_id }
     it { should have_db_column :restored_on }
     it { should have_db_column :experiments_order }
+    it { should have_db_column :template }
   end
 
   describe 'Relations' do