Merge pull request #746 from okriuchykhin/ok_SCI_1487

Improve import of repository records plus tsv file import fixes [SCI-1487]

Gemfile

@@ -59,6 +59,7 @@ gem 'sneaky-save', git: 'https://github.com/einzige/sneaky-save'
 gem 'rails_autolink', '~> 1.1', '>= 1.1.6'
 gem 'delayed_paperclip'
 gem 'rubyzip'
+gem 'activerecord-import'
 
 gem 'paperclip', '~> 4.3' # File attachment, image attachment library
 gem 'aws-sdk', '~> 2.2.8'

Gemfile.lock

@@ -37,6 +37,8 @@ GEM
       activemodel (= 4.2.5)
       activesupport (= 4.2.5)
       arel (~> 6.0)
+    activerecord-import (0.19.0)
+      activerecord (>= 3.2)
     activesupport (4.2.5)
       i18n (~> 0.7)
       json (~> 1.7, >= 1.7.7)
@@ -338,6 +340,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  activerecord-import
   ajax-datatables-rails (~> 0.3.1)
   aspector
   auto_strip_attributes (~> 2.1)

app/assets/javascripts/repositories/import/records_importer.js

@@ -31,6 +31,7 @@
       disabledOptions = $("option[disabled='disabled']");
       disabledOptions.removeAttr('disabled');
       loadingRecords = true;
+      $('#parse-records-modal').modal('hide');
       animateSpinner();
     }).on('ajax:success', function(ev, data, status) {
       // Simply reload page to show flash and updated repository records list

app/assets/javascripts/repositories/index.js

@@ -18,7 +18,7 @@
     $('#form-records-file').on('ajax:success', function(ev, data) {
       $('#modal-import-records').modal('hide');
       $(data.html).appendTo('body').promise().done(function() {
-        $('#parse-records_modal')
+        $('#parse-records-modal')
           .modal('show')
           .on('hidden.bs.modal', function() {
             animateSpinner();

app/models/repository.rb

@@ -110,7 +110,7 @@ class Repository < ActiveRecord::Base
   # Imports records
   def import_records(sheet, mappings, user)
     errors = false
-    custom_fields = []
+    columns = []
     name_index = -1
     total_nr = 0
     nr_of_added = 0
@@ -118,54 +118,71 @@ class Repository < ActiveRecord::Base
     mappings.each.with_index do |(_k, value), index|
       if value == '-1'
         # Fill blank space, so our indices stay the same
-        custom_fields << nil
+        columns << nil
         name_index = index
       else
-        cf = repository_columns.find_by_id(value)
+        columns << repository_columns.find_by_id(value)
-        custom_fields << cf
       end
     end
 
-    # Now we can iterate through record data and save stuff into db
+    # Check for duplicate columns
-    (2..sheet.last_row).each do |i|
+    col_compact = columns.compact
-      total_nr += 1
+    unless col_compact.map(&:id).uniq.length == col_compact.length
-      cell_error = false
+      return { status: :error, nr_of_added: nr_of_added, total_nr: total_nr }
-      record_row = RepositoryRow.new(name: sheet.row(i)[name_index],
+    end
-                                 repository: self,
-                                 created_by: user,
-                                 last_modified_by: user)
 
-      unless record_row.valid?
+    # Now we can iterate through record data and save stuff into db
-        errors = true
+    transaction do
-        next
+      (2..sheet.last_row).each do |i|
-      end
+        total_nr += 1
-      sheet.row(i).each.with_index do |value, index|
+        record_row = RepositoryRow.new(name: sheet.row(i)[name_index],
-        if custom_fields[index] && value
+                                   repository: self,
-          rep_column = RepositoryTextValue.new(
+                                   created_by: user,
-            data: value,
+                                   last_modified_by: user)
-            created_by: user,
+        record_row.transaction(requires_new: true) do
-            last_modified_by: user,
+          unless record_row.save
-            repository_cell_attributes: {
+            errors = true
-              repository_row: record_row,
+            raise ActiveRecord::Rollback
-              repository_column: custom_fields[index]
+          end
-            }
+
-          )
+          row_cell_values = []
-          cell_error = true unless rep_column.save
+
+          sheet.row(i).each.with_index do |value, index|
+            if columns[index] && value
+              cell_value = RepositoryTextValue.new(
+                data: value,
+                created_by: user,
+                last_modified_by: user,
+                repository_cell_attributes: {
+                  repository_row: record_row,
+                  repository_column: columns[index]
+                }
+              )
+              cell = RepositoryCell.new(repository_row: record_row,
+                                        repository_column: columns[index],
+                                        value: cell_value)
+              cell.skip_on_import = true
+              cell_value.repository_cell = cell
+              unless cell.valid? && cell_value.valid?
+                errors = true
+                raise ActiveRecord::Rollback
+              end
+              row_cell_values << cell_value
+            end
+          end
+          if RepositoryTextValue.import(row_cell_values,
+                                        recursive: true,
+                                        validate: false).failed_instances.any?
+            errors = true
+            raise ActiveRecord::Rollback
+          end
+          nr_of_added += 1
         end
       end
-      if cell_error
-        errors = true
-        record_row.destroy
-      else
-        nr_of_added += 1
-        record_row.save
-      end
     end
 
     if errors
-      return { status: :error,
+      return { status: :error, nr_of_added: nr_of_added, total_nr: total_nr }
-               nr_of_added: nr_of_added,
-               total_nr: total_nr }
     end
     { status: :ok, nr_of_added: nr_of_added, total_nr: total_nr }
   end
@@ -176,13 +193,11 @@ class Repository < ActiveRecord::Base
     case File.extname(filename)
     when '.csv'
       Roo::CSV.new(file_path, extension: :csv)
-    when '.tdv'
+    when '.tsv'
-      Roo::CSV.new(file_path, nil, :ignore, csv_options: { col_sep: '\t' })
+      Roo::CSV.new(file_path, csv_options: { col_sep: "\t" })
     when '.txt'
       # This assumption is based purely on biologist's habits
-      Roo::CSV.new(file_path, csv_options: { col_sep: '\t' })
+      Roo::CSV.new(file_path, csv_options: { col_sep: "\t" })
-    when '.xls'
-      Roo::Excel.new(file_path)
     when '.xlsx'
       Roo::Excelx.new(file_path)
     else

app/models/repository_cell.rb

@@ -1,11 +1,16 @@
 class RepositoryCell < ActiveRecord::Base
+  attr_accessor :skip_on_import
+
   belongs_to :repository_row
   belongs_to :repository_column
   belongs_to :value, polymorphic: true, dependent: :destroy
 
   validates :repository_column, presence: true
+  validates :value, presence: true
   validate :repository_column_data_type
-  validates :repository_row, uniqueness: { scope: :repository_column }
+  validates :repository_row,
+            uniqueness: { scope: :repository_column },
+            unless: :skip_on_import
 
   private
 

app/models/team.rb

@@ -37,16 +37,14 @@ class Team < ActiveRecord::Base
     end
 
     case File.extname(filename)
-    when ".csv" then
+    when '.csv' then
       Roo::CSV.new(file_path, extension: :csv)
-    when ".tdv" then
+    when '.tsv' then
-      Roo::CSV.new(file_path, nil, :ignore, csv_options: {col_sep: "\t"})
+      Roo::CSV.new(file_path, csv_options: { col_sep: "\t" })
-    when ".txt" then
+    when '.txt' then
       # This assumption is based purely on biologist's habits
-      Roo::CSV.new(file_path, csv_options: {col_sep: "\t"})
+      Roo::CSV.new(file_path, csv_options: { col_sep: "\t" })
-    when ".xls" then
+    when '.xlsx' then
-      Roo::Excel.new(file_path)
-    when ".xlsx" then
       Roo::Excelx.new(file_path)
     else
       raise TypeError

app/views/repositories/_parse_records_modal.html.erb

@@ -1,5 +1,5 @@
 <div class="modal fade"
-     id="parse-records_modal"
+     id="parse-records-modal"
      aria-labelledby="parse-modal-title"
      role="dialog">
   <div class="modal-dialog modal-lg">

config/locales/en.yml

@@ -951,7 +951,7 @@ en:
       title: 'Import items'
     modal_import:
       title: 'Import items'
-      notice: 'You may upload .csv file (comma separated) or tab separated file (.txt or .tdv) or Excel file (.xls, .xlsx). First row should include header names, followed by rows with sample data.'
+      notice: 'You may upload .csv file (comma separated) or tab separated file (.txt or .tsv) or Excel file (.xlsx). First row should include header names, followed by rows with sample data.'
       upload: 'Upload file'
     js:
       permission_error: "You don't have permission to edit this item."
@@ -1009,7 +1009,7 @@ en:
       sample_type: "Sample type:"
     modal_import:
       title: "Import samples"
-      notice: "You may upload .csv file (comma separated) or tab separated file (.txt or .tdv) or Excel file (.xls, .xlsx). First row should include header names, followed by rows with sample data."
+      notice: "You may upload .csv file (comma separated) or tab separated file (.txt or .tsv) or Excel file (.xlsx). First row should include header names, followed by rows with sample data."
       no_header_name: 'No column name'
       upload: "Upload file"
     modal_delete: