diff --git a/Gemfile b/Gemfile index 75fadeeee..0a074d9f9 100644 --- a/Gemfile +++ b/Gemfile @@ -59,6 +59,7 @@ gem 'sneaky-save', git: 'https://github.com/einzige/sneaky-save' gem 'rails_autolink', '~> 1.1', '>= 1.1.6' gem 'delayed_paperclip' gem 'rubyzip' +gem 'activerecord-import' gem 'paperclip', '~> 4.3' # File attachment, image attachment library gem 'aws-sdk', '~> 2.2.8' diff --git a/Gemfile.lock b/Gemfile.lock index eb70da5f8..aeeb3b107 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -37,6 +37,8 @@ GEM activemodel (= 4.2.5) activesupport (= 4.2.5) arel (~> 6.0) + activerecord-import (0.19.0) + activerecord (>= 3.2) activesupport (4.2.5) i18n (~> 0.7) json (~> 1.7, >= 1.7.7) @@ -338,6 +340,7 @@ PLATFORMS ruby DEPENDENCIES + activerecord-import ajax-datatables-rails (~> 0.3.1) aspector auto_strip_attributes (~> 2.1) diff --git a/app/assets/javascripts/repositories/import/records_importer.js b/app/assets/javascripts/repositories/import/records_importer.js index 8ccbab01e..9f4ad2a77 100644 --- a/app/assets/javascripts/repositories/import/records_importer.js +++ b/app/assets/javascripts/repositories/import/records_importer.js @@ -31,6 +31,7 @@ disabledOptions = $("option[disabled='disabled']"); disabledOptions.removeAttr('disabled'); loadingRecords = true; + $('#parse-records-modal').modal('hide'); animateSpinner(); }).on('ajax:success', function(ev, data, status) { // Simply reload page to show flash and updated repository records list diff --git a/app/assets/javascripts/repositories/index.js b/app/assets/javascripts/repositories/index.js index 5b22e5486..fed0920ed 100644 --- a/app/assets/javascripts/repositories/index.js +++ b/app/assets/javascripts/repositories/index.js @@ -18,7 +18,7 @@ $('#form-records-file').on('ajax:success', function(ev, data) { $('#modal-import-records').modal('hide'); $(data.html).appendTo('body').promise().done(function() { - $('#parse-records_modal') + $('#parse-records-modal') .modal('show') .on('hidden.bs.modal', function() { animateSpinner(); diff --git a/app/models/repository.rb b/app/models/repository.rb index 0e6c193c3..64058833b 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -110,7 +110,7 @@ class Repository < ActiveRecord::Base # Imports records def import_records(sheet, mappings, user) errors = false - custom_fields = [] + columns = [] name_index = -1 total_nr = 0 nr_of_added = 0 @@ -118,54 +118,71 @@ class Repository < ActiveRecord::Base mappings.each.with_index do |(_k, value), index| if value == '-1' # Fill blank space, so our indices stay the same - custom_fields << nil + columns << nil name_index = index else - cf = repository_columns.find_by_id(value) - custom_fields << cf + columns << repository_columns.find_by_id(value) end end - # Now we can iterate through record data and save stuff into db - (2..sheet.last_row).each do |i| - total_nr += 1 - cell_error = false - record_row = RepositoryRow.new(name: sheet.row(i)[name_index], - repository: self, - created_by: user, - last_modified_by: user) + # Check for duplicate columns + col_compact = columns.compact + unless col_compact.map(&:id).uniq.length == col_compact.length + return { status: :error, nr_of_added: nr_of_added, total_nr: total_nr } + end - unless record_row.valid? - errors = true - next - end - sheet.row(i).each.with_index do |value, index| - if custom_fields[index] && value - rep_column = RepositoryTextValue.new( - data: value, - created_by: user, - last_modified_by: user, - repository_cell_attributes: { - repository_row: record_row, - repository_column: custom_fields[index] - } - ) - cell_error = true unless rep_column.save + # Now we can iterate through record data and save stuff into db + transaction do + (2..sheet.last_row).each do |i| + total_nr += 1 + record_row = RepositoryRow.new(name: sheet.row(i)[name_index], + repository: self, + created_by: user, + last_modified_by: user) + record_row.transaction(requires_new: true) do + unless record_row.save + errors = true + raise ActiveRecord::Rollback + end + + row_cell_values = [] + + sheet.row(i).each.with_index do |value, index| + if columns[index] && value + cell_value = RepositoryTextValue.new( + data: value, + created_by: user, + last_modified_by: user, + repository_cell_attributes: { + repository_row: record_row, + repository_column: columns[index] + } + ) + cell = RepositoryCell.new(repository_row: record_row, + repository_column: columns[index], + value: cell_value) + cell.skip_on_import = true + cell_value.repository_cell = cell + unless cell.valid? && cell_value.valid? + errors = true + raise ActiveRecord::Rollback + end + row_cell_values << cell_value + end + end + if RepositoryTextValue.import(row_cell_values, + recursive: true, + validate: false).failed_instances.any? + errors = true + raise ActiveRecord::Rollback + end + nr_of_added += 1 end end - if cell_error - errors = true - record_row.destroy - else - nr_of_added += 1 - record_row.save - end end if errors - return { status: :error, - nr_of_added: nr_of_added, - total_nr: total_nr } + return { status: :error, nr_of_added: nr_of_added, total_nr: total_nr } end { status: :ok, nr_of_added: nr_of_added, total_nr: total_nr } end @@ -176,13 +193,11 @@ class Repository < ActiveRecord::Base case File.extname(filename) when '.csv' Roo::CSV.new(file_path, extension: :csv) - when '.tdv' - Roo::CSV.new(file_path, nil, :ignore, csv_options: { col_sep: '\t' }) + when '.tsv' + Roo::CSV.new(file_path, csv_options: { col_sep: "\t" }) when '.txt' # This assumption is based purely on biologist's habits - Roo::CSV.new(file_path, csv_options: { col_sep: '\t' }) - when '.xls' - Roo::Excel.new(file_path) + Roo::CSV.new(file_path, csv_options: { col_sep: "\t" }) when '.xlsx' Roo::Excelx.new(file_path) else diff --git a/app/models/repository_cell.rb b/app/models/repository_cell.rb index e89887c94..403362622 100644 --- a/app/models/repository_cell.rb +++ b/app/models/repository_cell.rb @@ -1,11 +1,16 @@ class RepositoryCell < ActiveRecord::Base + attr_accessor :skip_on_import + belongs_to :repository_row belongs_to :repository_column belongs_to :value, polymorphic: true, dependent: :destroy validates :repository_column, presence: true + validates :value, presence: true validate :repository_column_data_type - validates :repository_row, uniqueness: { scope: :repository_column } + validates :repository_row, + uniqueness: { scope: :repository_column }, + unless: :skip_on_import private diff --git a/app/models/team.rb b/app/models/team.rb index c3fae86c2..9238a0f03 100644 --- a/app/models/team.rb +++ b/app/models/team.rb @@ -37,16 +37,14 @@ class Team < ActiveRecord::Base end case File.extname(filename) - when ".csv" then + when '.csv' then Roo::CSV.new(file_path, extension: :csv) - when ".tdv" then - Roo::CSV.new(file_path, nil, :ignore, csv_options: {col_sep: "\t"}) - when ".txt" then + when '.tsv' then + Roo::CSV.new(file_path, csv_options: { col_sep: "\t" }) + when '.txt' then # This assumption is based purely on biologist's habits - Roo::CSV.new(file_path, csv_options: {col_sep: "\t"}) - when ".xls" then - Roo::Excel.new(file_path) - when ".xlsx" then + Roo::CSV.new(file_path, csv_options: { col_sep: "\t" }) + when '.xlsx' then Roo::Excelx.new(file_path) else raise TypeError diff --git a/app/views/repositories/_parse_records_modal.html.erb b/app/views/repositories/_parse_records_modal.html.erb index c582b8bbe..ccab285b1 100644 --- a/app/views/repositories/_parse_records_modal.html.erb +++ b/app/views/repositories/_parse_records_modal.html.erb @@ -1,5 +1,5 @@