scinote-eln/scinote-web
1
1
Fork 0
mirror of https://github.com/scinote-eln/scinote-web.git synced 2024-12-27 10:14:17 +08:00
Code Issues Projects Releases Packages Wiki Activity

(dev) Create a POC for Phase 0 of improved importing to inventories [SCI-10222]

Browse source
This commit is contained in:
Gregor Lasnibat 2024-02-23 12:47:57 +01:00
parent f8aae8aca6
commit e8b4e40391
6 changed files with 103 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
app/controllers/repositories_controller.rb
View file

@ -305,10 +305,14 @@ class RepositoriesController < ApplicationController
render_403 unless can_create_repository_rows?(Repository.accessible_by_teams(current_team)
.find_by_id(import_params[:id]))
# Access the checkbox values from params
can_edit_existing_items = params[:edit_existing_items_checkbox]
should_overwrite_with_empty_cells = params[:overwrite_with_empty_cells]
# Check if there exist mapping for repository record (it's mandatory)
if import_params[:mappings].value?('-1')
import_records = repostiory_import_actions
status = import_records.import!
status = import_records.import!(can_edit_existing_items, should_overwrite_with_empty_cells)
if status[:status] == :ok
log_activity(:import_inventory_items,

4
app/models/repository.rb
View file

@ -231,9 +231,9 @@ class Repository < RepositoryBase
new_repo
end
def import_records(sheet, mappings, user)
def import_records(sheet, mappings, user, can_edit_existing_items, should_overwrite_with_empty_cells)
importer = RepositoryImportParser::Importer.new(sheet, mappings, user, self)
importer.run
importer.run(can_edit_existing_items, should_overwrite_with_empty_cells)
end
def assigned_rows(my_module)

10
app/services/import_repository/import_records.rb
View file

@ -8,20 +8,22 @@ module ImportRepository
@user = options.fetch(:user)
end
def import!
status = run_import_actions
def import!(can_edit_existing_items, should_overwrite_with_empty_cells)
status = run_import_actions(can_edit_existing_items, should_overwrite_with_empty_cells)
@temp_file.destroy
status
end
private
def run_import_actions
def run_import_actions(can_edit_existing_items, should_overwrite_with_empty_cells)
@temp_file.file.open do |temp_file|
@repository.import_records(
SpreadsheetParser.open_spreadsheet(temp_file),
@mappings,
@user
@user,
can_edit_existing_items,
should_overwrite_with_empty_cells
)
end
end

97
app/utilities/repository_import_parser/importer.rb
View file

@ -6,6 +6,7 @@
# @mappings: mappings for columns
# @user: current_user
# @repository: the repository in which we import the items
module RepositoryImportParser
class Importer
IMPORT_BATCH_SIZE = 500
@ -24,11 +25,11 @@ module RepositoryImportParser
@repository_columns = @repository.repository_columns
end
def run
def run(can_edit_existing_items, should_overwrite_with_empty_cells)
fetch_columns
return check_for_duplicate_columns if check_for_duplicate_columns
import_rows!
import_rows!(can_edit_existing_items, should_overwrite_with_empty_cells)
end
private
@ -54,7 +55,7 @@ module RepositoryImportParser
end
end
def import_rows!
def import_rows!(can_edit_existing_items, should_overwrite_with_empty_cells)
errors = false
@repository.transaction do
@ -72,12 +73,23 @@ module RepositoryImportParser
@total_new_rows += 1
new_full_row = {}
SpreadsheetParser.parse_row(
incoming_row = SpreadsheetParser.parse_row(
row,
@sheet,
date_format: @user.settings['date_format']
).each_with_index do |value, index|
)
incoming_row.each_with_index do |value, index|
if index == @name_index
# extract row_id
row_id = try_decimal_to_string(value)
# check if row (inventory) already exists
existing_row = RepositoryRow.find_by(name: row_id, repository: @repository)
# if it doesn't exist create it
unless existing_row
new_row =
RepositoryRow.new(name: try_decimal_to_string(value),
repository: @repository,
@ -87,12 +99,24 @@ module RepositoryImportParser
errors = true
break
end
new_full_row[:repository_row] = new_row
next
end
next unless @columns[index]
# if it does exist but shouldn't be edited, error out and break
if existing_row && can_edit_existing_items == '0'
errors = true
break
end
# if it does exist and should be edited, update the existing row
if existing_row && can_edit_existing_items == '1'
# update the existing row with incoming row data
new_full_row[:repository_row] = existing_row
end
end
next unless @columns[index]
new_full_row[index] = value
end
@ -103,13 +127,13 @@ module RepositoryImportParser
next if batch_counter < IMPORT_BATCH_SIZE
import_batch_to_database(full_row_import_batch)
import_batch_to_database(full_row_import_batch, can_edit_existing_items, should_overwrite_with_empty_cells)
full_row_import_batch = []
batch_counter = 0
end
# Import of the remaining rows
import_batch_to_database(full_row_import_batch) if full_row_import_batch.any?
import_batch_to_database(full_row_import_batch, can_edit_existing_items, should_overwrite_with_empty_cells) if full_row_import_batch.any?
end
if errors
@ -120,44 +144,57 @@ module RepositoryImportParser
{ status: :ok, nr_of_added: @new_rows_added, total_nr: @total_new_rows }
end
def import_batch_to_database(full_row_import_batch)
repository_rows = full_row_import_batch.collect { |row| row[:repository_row] }
@new_rows_added += RepositoryRow.import(repository_rows, recursive: false, validate: false).ids.length
repository_rows.each { |row| row.run_callbacks(:create) }
def import_batch_to_database(full_row_import_batch, can_edit_existing_items, should_overwrite_with_empty_cells)
full_row_import_batch.each do |full_row|
full_row[:repository_row].save!(validate: false)
@new_rows_added += 1
import_mappings = Hash[@columns.map { |column| column&.data_type&.to_sym }
.compact
.uniq
.map { |data_type| [data_type, []] }]
full_row_import_batch.each do |row|
next unless row[:repository_row].id
row.reject { |k| k == :repository_row }.each do |index, value|
full_row.reject { |k| k == :repository_row }.each do |index, value|
column = @columns[index]
value = try_decimal_to_string(value) unless column.repository_number_value?
next if value.nil?
cell_value_attributes = { created_by: @user,
cell_value_attributes = {
created_by: @user,
last_modified_by: @user,
repository_cell_attributes: { repository_row: row[:repository_row],
repository_cell_attributes: {
repository_row: full_row[:repository_row],
repository_column: column,
importing: true } }
importing: true
}
}
cell_value = column.data_type.constantize.import_from_text(
value,
cell_value_attributes,
@user.as_json(root: true, only: :settings).deep_symbolize_keys
)
next if cell_value.nil?
existing_cell = full_row[:repository_row].repository_cells.find_by(repository_column: column)
next if cell_value.nil? && existing_cell.nil?
# no existing_cell. Create a new one.
if !existing_cell
cell_value.repository_cell.value = cell_value
cell_value.save!(validate: false)
else
# existing_cell present && !can_edit_existing_items
next if can_edit_existing_items == '0'
import_mappings[column.data_type.to_sym] << cell_value
# existing_cell present && can_edit_existing_items
if can_edit_existing_items == '1'
# if incoming cell is not empty
existing_cell.value.update_data!(cell_value.data, @user) if !cell_value.nil?
# if incoming cell is empty && should_overwrite_with_empty_cells
existing_cell.value.destroy! if cell_value.nil? && should_overwrite_with_empty_cells == '1'
# if incoming cell is empty && !should_overwrite_with_empty_cells
next if cell_value.nil? && should_overwrite_with_empty_cells == '0'
end
end
end
import_mappings.each do |data_type, cell_values|
data_type.to_s.constantize.import(cell_values, recursive: true, validate: false)
end
end

14
app/views/repositories/_parse_records_modal.html.erb
View file

@ -14,6 +14,20 @@
<%= f.hidden_field :team_id, value: current_team.id %>
<div class="modal-body">
<p><%= t("repositories.parse_sheet.help_text") %></p>
<div class="form-check">
<div class="sci-checkbox-container">
<%= f.check_box :edit_existing_items_checkbox, { class: 'sci-checkbox'} %>
<span class="sci-checkbox-label"></span>
</div>
<%= f.label :edit_existing_items_checkbox, t("repositories.parse_sheet.edit_existing_items_checkbox") %>
</div>
<div class="form-check">
<div class="sci-checkbox-container">
<%= f.check_box :overwrite_with_empty_cells, { class: 'sci-checkbox', checked: false} %>
<span class="sci-checkbox-label"></span>
</div>
<%= f.label :overwrite_with_empty_cells, t("repositories.parse_sheet.overwrite_with_empty_cells") %>
</div>
<div style="overflow-x: scroll">
<table class="table parse-records-table" style="display: block">
<thead>

2
config/locales/en.yml
View file

@ -2101,6 +2101,8 @@ en:
file_columns: "Imported columns:"
example_value: "Imported file content:"
do_not_include_column: "Do not include this column"
edit_existing_items_checkbox: "Edit existing items"
overwrite_with_empty_cells: "Overwrite with empty cells"
errors:
invalid_file: "The file you provided is invalid. Make sure the file is encoded using %{encoding}."
invalid_extension: "The file has invalid extension."