scinote-web/app/utilities/repository_import_parser/importer.rb

173 lines
5.4 KiB
Ruby
Raw Normal View History

# frozen_string_literal: true
# handles the import of repository records
# requires 4 parameters:
# @sheet: the csv file with imported rows
# @mappings: mappings for columns
# @user: current_user
# @repository: the repository in which we import the items
module RepositoryImportParser
class Importer
IMPORT_BATCH_SIZE = 500
def initialize(sheet, mappings, user, repository)
@columns = []
@name_index = -1
@total_new_rows = 0
@new_rows_added = 0
@header_skipped = false
@repository = repository
@sheet = sheet
@rows = SpreadsheetParser.spreadsheet_enumerator(@sheet)
@mappings = mappings
@user = user
@repository_columns = @repository.repository_columns
end
def run
2018-03-13 21:47:35 +08:00
fetch_columns
return check_for_duplicate_columns if check_for_duplicate_columns
import_rows!
end
private
2018-03-13 21:47:35 +08:00
def fetch_columns
2020-02-11 23:12:53 +08:00
@mappings.each_with_index do |(_, value), index|
if value == '-1'
# Fill blank space, so our indices stay the same
@columns << nil
@name_index = index
else
2020-02-11 23:12:53 +08:00
@columns << @repository_columns.where(data_type: Extends::REPOSITORY_IMPORTABLE_TYPES)
.preload(Extends::REPOSITORY_IMPORT_COLUMN_PRELOADS)
.find_by(id: value)
end
end
end
def check_for_duplicate_columns
col_compact = @columns.compact
if col_compact.map(&:id).uniq.length != col_compact.length
{ status: :error, nr_of_added: @new_rows_added, total_nr: @total_new_rows }
end
end
def import_rows!
errors = false
@repository.transaction do
batch_counter = 0
full_row_import_batch = []
@rows.each do |row|
# Skip empty rows
2021-07-23 17:56:28 +08:00
next if row.blank?
unless @header_skipped
@header_skipped = true
next
end
@total_new_rows += 1
new_full_row = {}
SpreadsheetParser.parse_row(
row,
@sheet,
date_format: @user.settings['date_format']
).each_with_index do |value, index|
if index == @name_index
new_row =
RepositoryRow.new(name: try_decimal_to_string(value),
repository: @repository,
created_by: @user,
last_modified_by: @user)
unless new_row.valid?
errors = true
break
end
new_full_row[:repository_row] = new_row
next
end
next unless @columns[index]
new_full_row[index] = value
end
if new_full_row[:repository_row].present?
full_row_import_batch << new_full_row
batch_counter += 1
end
2019-03-29 22:41:34 +08:00
next if batch_counter < IMPORT_BATCH_SIZE
import_batch_to_database(full_row_import_batch)
full_row_import_batch = []
batch_counter = 0
end
# Import of the remaining rows
import_batch_to_database(full_row_import_batch) if full_row_import_batch.any?
end
if errors
return { status: :error,
nr_of_added: @new_rows_added,
total_nr: @total_new_rows }
end
{ status: :ok, nr_of_added: @new_rows_added, total_nr: @total_new_rows }
end
def import_batch_to_database(full_row_import_batch)
repository_rows = full_row_import_batch.collect { |row| row[:repository_row] }
@new_rows_added += RepositoryRow.import(repository_rows, recursive: false, validate: false).ids.length
repository_rows.each { |row| row.run_callbacks(:create) }
import_mappings = Hash[@columns.map { |column| column&.data_type&.to_sym }
.compact
.uniq
.map { |data_type| [data_type, []] }]
full_row_import_batch.each do |row|
next unless row[:repository_row].id
2018-04-10 21:45:51 +08:00
row.reject { |k| k == :repository_row }.each do |index, value|
column = @columns[index]
value = try_decimal_to_string(value) unless column.repository_number_value?
2020-05-20 15:56:08 +08:00
next if value.nil?
cell_value_attributes = { created_by: @user,
last_modified_by: @user,
repository_cell_attributes: { repository_row: row[:repository_row],
repository_column: column,
importing: true } }
2020-03-05 21:34:31 +08:00
cell_value = column.data_type.constantize.import_from_text(
value,
cell_value_attributes,
2020-03-13 21:57:26 +08:00
@user.as_json(root: true, only: :settings).deep_symbolize_keys
2020-03-05 21:34:31 +08:00
)
next if cell_value.nil?
cell_value.repository_cell.value = cell_value
import_mappings[column.data_type.to_sym] << cell_value
end
2018-04-10 21:45:51 +08:00
end
import_mappings.each do |data_type, cell_values|
data_type.to_s.constantize.import(cell_values, recursive: true, validate: false)
end
2018-04-10 21:45:51 +08:00
end
def try_decimal_to_string(value)
if value.is_a?(BigDecimal)
value.frac.zero? ? value.to_i.to_s : value.to_s
else
value
end
end
end
end