scinote-eln/scinote-web
1
1
Fork 0
mirror of https://github.com/scinote-eln/scinote-web.git synced 2025-09-11 15:45:34 +08:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #746 from okriuchykhin/ok_SCI_1487

Browse source 
Improve import of repository records plus tsv file import fixes [SCI-1487]
This commit is contained in:
okriuchykhin 2017-07-21 11:01:31 +02:00 • committed by GitHub
commit a49c35df9f
9 changed files with 79 additions and 56 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
Gemfile
View file

@ -59,6 +59,7 @@ gem 'sneaky-save', git: 'https://github.com/einzige/sneaky-save'
gem 'rails_autolink', '~> 1.1', '>= 1.1.6'
gem 'delayed_paperclip'
gem 'rubyzip'
gem 'activerecord-import'
gem 'paperclip', '~> 4.3' # File attachment, image attachment library
gem 'aws-sdk', '~> 2.2.8'

3
Gemfile.lock
View file

@ -37,6 +37,8 @@ GEM
activemodel (= 4.2.5)
activesupport (= 4.2.5)
arel (~> 6.0)
activerecord-import (0.19.0)
activerecord (>= 3.2)
activesupport (4.2.5)
i18n (~> 0.7)
json (~> 1.7, >= 1.7.7)
@ -338,6 +340,7 @@ PLATFORMS
ruby
DEPENDENCIES
activerecord-import
ajax-datatables-rails (~> 0.3.1)
aspector
auto_strip_attributes (~> 2.1)

1
app/assets/javascripts/repositories/import/records_importer.js
View file

@ -31,6 +31,7 @@
disabledOptions = $("option[disabled='disabled']");
disabledOptions.removeAttr('disabled');
loadingRecords = true;
$('#parse-records-modal').modal('hide');
animateSpinner();
}).on('ajax:success', function(ev, data, status) {
// Simply reload page to show flash and updated repository records list

2
app/assets/javascripts/repositories/index.js
View file

@ -18,7 +18,7 @@
$('#form-records-file').on('ajax:success', function(ev, data) {
$('#modal-import-records').modal('hide');
$(data.html).appendTo('body').promise().done(function() {
$('#parse-records_modal')
$('#parse-records-modal')
.modal('show')
.on('hidden.bs.modal', function() {
animateSpinner();

101
app/models/repository.rb
View file

@ -110,7 +110,7 @@ class Repository < ActiveRecord::Base
# Imports records
def import_records(sheet, mappings, user)
errors = false
custom_fields = []
columns = []
name_index = -1
total_nr = 0
nr_of_added = 0
@ -118,54 +118,71 @@ class Repository < ActiveRecord::Base
mappings.each.with_index do |(_k, value), index|
if value == '-1'
# Fill blank space, so our indices stay the same
custom_fields << nil
columns << nil
name_index = index
else
cf = repository_columns.find_by_id(value)
custom_fields << cf
columns << repository_columns.find_by_id(value)
end
end
# Now we can iterate through record data and save stuff into db
(2..sheet.last_row).each do |i|
total_nr += 1
cell_error = false
record_row = RepositoryRow.new(name: sheet.row(i)[name_index],
repository: self,
created_by: user,
last_modified_by: user)
# Check for duplicate columns
col_compact = columns.compact
unless col_compact.map(&:id).uniq.length == col_compact.length
return { status: :error, nr_of_added: nr_of_added, total_nr: total_nr }
end
unless record_row.valid?
errors = true
next
end
sheet.row(i).each.with_index do |value, index|
if custom_fields[index] && value
rep_column = RepositoryTextValue.new(
data: value,
created_by: user,
last_modified_by: user,
repository_cell_attributes: {
repository_row: record_row,
repository_column: custom_fields[index]
}
)
cell_error = true unless rep_column.save
# Now we can iterate through record data and save stuff into db
transaction do
(2..sheet.last_row).each do |i|
total_nr += 1
record_row = RepositoryRow.new(name: sheet.row(i)[name_index],
repository: self,
created_by: user,
last_modified_by: user)
record_row.transaction(requires_new: true) do
unless record_row.save
errors = true
raise ActiveRecord::Rollback
end
row_cell_values = []
sheet.row(i).each.with_index do |value, index|
if columns[index] && value
cell_value = RepositoryTextValue.new(
data: value,
created_by: user,
last_modified_by: user,
repository_cell_attributes: {
repository_row: record_row,
repository_column: columns[index]
}
)
cell = RepositoryCell.new(repository_row: record_row,
repository_column: columns[index],
value: cell_value)
cell.skip_on_import = true
cell_value.repository_cell = cell
unless cell.valid? && cell_value.valid?
errors = true
raise ActiveRecord::Rollback
end
row_cell_values << cell_value
end
end
if RepositoryTextValue.import(row_cell_values,
recursive: true,
validate: false).failed_instances.any?
errors = true
raise ActiveRecord::Rollback
end
nr_of_added += 1
end
end
if cell_error
errors = true
record_row.destroy
else
nr_of_added += 1
record_row.save
end
end
if errors
return { status: :error,
nr_of_added: nr_of_added,
total_nr: total_nr }
return { status: :error, nr_of_added: nr_of_added, total_nr: total_nr }
end
{ status: :ok, nr_of_added: nr_of_added, total_nr: total_nr }
end
@ -176,13 +193,11 @@ class Repository < ActiveRecord::Base
case File.extname(filename)
when '.csv'
Roo::CSV.new(file_path, extension: :csv)
when '.tdv'
Roo::CSV.new(file_path, nil, :ignore, csv_options: { col_sep: '\t' })
when '.tsv'
Roo::CSV.new(file_path, csv_options: { col_sep: "\t" })
when '.txt'
# This assumption is based purely on biologist's habits
Roo::CSV.new(file_path, csv_options: { col_sep: '\t' })
when '.xls'
Roo::Excel.new(file_path)
Roo::CSV.new(file_path, csv_options: { col_sep: "\t" })
when '.xlsx'
Roo::Excelx.new(file_path)
else

7
app/models/repository_cell.rb
View file

@ -1,11 +1,16 @@
class RepositoryCell < ActiveRecord::Base
attr_accessor :skip_on_import
belongs_to :repository_row
belongs_to :repository_column
belongs_to :value, polymorphic: true, dependent: :destroy
validates :repository_column, presence: true
validates :value, presence: true
validate :repository_column_data_type
validates :repository_row, uniqueness: { scope: :repository_column }
validates :repository_row,
uniqueness: { scope: :repository_column },
unless: :skip_on_import
private

14
app/models/team.rb
View file

@ -37,16 +37,14 @@ class Team < ActiveRecord::Base
end
case File.extname(filename)
when ".csv" then
when '.csv' then
Roo::CSV.new(file_path, extension: :csv)
when ".tdv" then
Roo::CSV.new(file_path, nil, :ignore, csv_options: {col_sep: "\t"})
when ".txt" then
when '.tsv' then
Roo::CSV.new(file_path, csv_options: { col_sep: "\t" })
when '.txt' then
# This assumption is based purely on biologist's habits
Roo::CSV.new(file_path, csv_options: {col_sep: "\t"})
when ".xls" then
Roo::Excel.new(file_path)
when ".xlsx" then
Roo::CSV.new(file_path, csv_options: { col_sep: "\t" })
when '.xlsx' then
Roo::Excelx.new(file_path)
else
raise TypeError

2
app/views/repositories/_parse_records_modal.html.erb
View file

@ -1,5 +1,5 @@
<div class="modal fade"
id="parse-records_modal"
id="parse-records-modal"
aria-labelledby="parse-modal-title"
role="dialog">
<div class="modal-dialog modal-lg">

4
config/locales/en.yml
View file

@ -951,7 +951,7 @@ en:
title: 'Import items'
modal_import:
title: 'Import items'
notice: 'You may upload .csv file (comma separated) or tab separated file (.txt or .tdv) or Excel file (.xls, .xlsx). First row should include header names, followed by rows with sample data.'
notice: 'You may upload .csv file (comma separated) or tab separated file (.txt or .tsv) or Excel file (.xlsx). First row should include header names, followed by rows with sample data.'
upload: 'Upload file'
js:
permission_error: "You don't have permission to edit this item."
@ -1009,7 +1009,7 @@ en:
sample_type: "Sample type:"
modal_import:
title: "Import samples"
notice: "You may upload .csv file (comma separated) or tab separated file (.txt or .tdv) or Excel file (.xls, .xlsx). First row should include header names, followed by rows with sample data."
notice: "You may upload .csv file (comma separated) or tab separated file (.txt or .tsv) or Excel file (.xlsx). First row should include header names, followed by rows with sample data."
no_header_name: 'No column name'
upload: "Upload file"
modal_delete: