refactor repository import, adds RepositoryListType as an option and makes the import class more extensible [fixes SCI-2024]

app/models/repository.rb

@@ -1,5 +1,6 @@
 class Repository < ApplicationRecord
   include SearchableModel
+  include RepositoryImportParser
 
   belongs_to :team, optional: true
   belongs_to :created_by,
@@ -100,87 +101,8 @@ class Repository < ApplicationRecord
     new_repo
   end
 
-  # Imports records
   def import_records(sheet, mappings, user)
-    errors = false
-    columns = []
-    name_index = -1
-    total_nr = 0
-    nr_of_added = 0
-    header_skipped = false
-
-    mappings.each.with_index do |(_k, value), index|
-      if value == '-1'
-        # Fill blank space, so our indices stay the same
-        columns << nil
-        name_index = index
-      else
-        columns << repository_columns.find_by_id(value)
-      end
-    end
-
-    # Check for duplicate columns
-    col_compact = columns.compact
-    unless col_compact.map(&:id).uniq.length == col_compact.length
-      return { status: :error, nr_of_added: nr_of_added, total_nr: total_nr }
-    end
-    rows = SpreadsheetParser.spreadsheet_enumerator(sheet)
-
-    # Now we can iterate through record data and save stuff into db
-    rows.each do |row|
-      # Skip empty rows
-      next if row.empty?
-      unless header_skipped
-        header_skipped = true
-        next
-      end
-      total_nr += 1
-
-      row = SpreadsheetParser.parse_row(row, sheet)
-
-      record_row = RepositoryRow.new(name: row[name_index],
-                                 repository: self,
-                                 created_by: user,
-                                 last_modified_by: user)
-      record_row.transaction do
-        unless record_row.save
-          errors = true
-          raise ActiveRecord::Rollback
-        end
-
-        row_cell_values = []
-
-        row.each.with_index do |value, index|
-          if columns[index] && value
-            cell_value = RepositoryTextValue.new(
-              data: value,
-              created_by: user,
-              last_modified_by: user,
-              repository_cell_attributes: {
-                repository_row: record_row,
-                repository_column: columns[index]
-              }
-            )
-            unless cell_value.valid?
-              errors = true
-              raise ActiveRecord::Rollback
-            end
-            row_cell_values << cell_value
-          end
-        end
-        if RepositoryTextValue.import(row_cell_values,
-                                      recursive: true,
-                                      validate: false).failed_instances.any?
-          errors = true
-          raise ActiveRecord::Rollback
-        end
-        nr_of_added += 1
-      end
-    end
-
-    if errors
-      return { status: :error, nr_of_added: nr_of_added, total_nr: total_nr }
-    end
-    { status: :ok, nr_of_added: nr_of_added, total_nr: total_nr }
+    importer = RepositoryImportParser::Importer.new(sheet, mappings, user, self)
+    importer.run
   end
 end

app/utilities/repository_import_parser/importer.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+# handles the import of repository records
+# requires 4 parameters:
+# @sheet: the csv file with imported rows
+# @mappings: mappings for columns
+# @user: current_user
+# @repository: the repository in which we import the items
+module RepositoryImportParser
+  class Importer
+
+    def initialize(sheet, mappings, user, repository)
+      @columns = []
+      @name_index = -1
+      @total_new_rows = 0
+      @new_rows_added = 0
+      @header_skipped = false
+      @repository = repository
+      @sheet = sheet
+      @rows = SpreadsheetParser.spreadsheet_enumerator(@sheet)
+      @mappings = mappings
+      @user = user
+      @repository_columns = @repository.repository_columns
+    end
+
+    def run
+      get_columns
+      return check_for_duplicate_columns if check_for_duplicate_columns
+      import_rows!
+    end
+
+    private
+
+    def get_columns
+      @mappings.each.with_index do |(_, value), index|
+        if value == '-1'
+          # Fill blank space, so our indices stay the same
+          @columns << nil
+          @name_index = index
+        else
+          @columns << @repository_columns.find_by_id(value)
+        end
+      end
+    end
+
+    def check_for_duplicate_columns
+      col_compact = @columns.compact
+      if col_compact.map(&:id).uniq.length != col_compact.length
+        return { status: :error,
+                 nr_of_added: @new_rows_added,
+                 total_nr: @total_new_rows }
+      end
+    end
+
+    def import_rows!
+      errors = false
+      @rows.each do |row|
+        # Skip empty rows
+        next if row.empty?
+        unless @header_skipped
+          @header_skipped = true
+          next
+        end
+        @total_new_rows += 1
+
+        row = SpreadsheetParser.parse_row(row, @sheet)
+        record_row = new_repository_row(row)
+        record_row.transaction do
+          unless record_row.save
+            errors = true
+            raise ActiveRecord::Rollback
+          end
+
+          row_cell_values = []
+
+          row.each.with_index do |value, index|
+            column = @columns[index]
+            if column && value.present?
+              # uses RepositoryCellValueResolver to retrieve the correct value
+              cell_value_resolver =
+                RepositoryImportParser::RepositoryCellValueResolver.new(
+                  column, @user, @repository
+                )
+              cell_value = cell_value_resolver.get_value(value, record_row)
+              unless cell_value.valid?
+                errors = true
+                raise ActiveRecord::Rollback
+              end
+              row_cell_values << cell_value
+            end
+          end
+
+          unless import_to_database(row_cell_values)
+            errors = true
+            raise ActiveRecord::Rollback
+          end
+          @new_rows_added += 1
+        end
+      end
+
+      if errors
+        return { status: :error,
+                 nr_of_added: @new_rows_added,
+                 total_nr: @total_new_rows }
+      end
+      { status: :ok, nr_of_added: @new_rows_added, total_nr: @total_new_rows }
+    end
+
+    def new_repository_row(row)
+      RepositoryRow.new(name: row[@name_index],
+                        repository: @repository,
+                        created_by: @user,
+                        last_modified_by: @user)
+    end
+
+    def import_to_database(row_cell_values)
+      return false if RepositoryTextValue.import(
+        row_cell_values.select { |element| element.is_a? RepositoryTextValue },
+        recursive: true,
+        validate: false
+      ).failed_instances.any?
+      return false if RepositoryListValue.import(
+        row_cell_values.select { |element| element.is_a? RepositoryListValue },
+        recursive: true,
+        validate: false
+      ).failed_instances.any?
+      true
+    end
+  end
+end

app/utilities/repository_import_parser/repository_cell_value_resolver.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+# this class is used to resolve the column data_type and assign
+# it to the repository_row
+module RepositoryImportParser
+  class RepositoryCellValueResolver
+
+    def initialize(column, user, repository)
+      @column = column
+      @user = user
+      @repository = repository
+    end
+
+    def get_value(value, record_row)
+      return unless @column
+      send("new_#{@column.data_type.underscore}", value, record_row)
+    end
+
+    private
+
+    def new_repository_text_value(value, record_row)
+      RepositoryTextValue.new(data: value,
+                              created_by: @user,
+                              last_modified_by: @user,
+                              repository_cell_attributes: {
+                                repository_row: record_row,
+                                repository_column: @column
+                              })
+    end
+
+    def new_repository_list_value(value, record_row)
+      list_item = @column.repository_list_items.find_by_data(value)
+      list_item ||= create_repository_list_item(value)
+      RepositoryListValue.new(
+        created_by: @user,
+        last_modified_by: @user,
+        repository_list_item: list_item,
+        repository_cell_attributes: {
+          repository_row: record_row,
+          repository_column: @column
+        }
+      )
+    end
+
+    def create_repository_list_item(value)
+      RepositoryListItem.create(
+        data: value,
+        created_by: @user,
+        last_modified_by: @user,
+        repository_column: @column,
+        repository: @repository
+      )
+    end
+  end
+end

spec/fixtures/files/export.csv

@@ -0,0 +1,6 @@
+Name,Added on,Added by,Sample group,Sample type,Custom items
+Sample 5,02.03.2018 09:53,Admin,group 3,,test 3
+Sample 4,02.03.2018 09:53,Admin,group 2,type 1,test 2
+Sample 3,02.03.2018 09:53,Admin,,type 1,
+Sample 2,02.03.2018 09:52,Admin,group 2,type 2,
+Sample 1,02.03.2018 09:52,Admin,group 1,type 2,test test

spec/utilities/repository_import_parser/importer_spec.rb

@@ -0,0 +1,91 @@
+require 'rails_helper'
+
+describe RepositoryImportParser::Importer do
+  let(:user) { create :user }
+  let(:team) { create :team, created_by: user }
+  let(:user_team) { create :user_team, user: user, team: team }
+  let(:repository) { create :repository, team: team, created_by: user }
+  let!(:sample_group_column) do
+    create :repository_column, repository: repository,
+                               created_by: user,
+                               name: 'Sample group',
+                               data_type: 'RepositoryListValue'
+  end
+  let!(:sample_type_column) do
+    create :repository_column, repository: repository,
+                               created_by: user,
+                               name: 'Sample type',
+                               data_type: 'RepositoryListValue'
+  end
+  let!(:custom_column) do
+    create :repository_column, repository: repository,
+                               created_by: user,
+                               name: 'Custom items',
+                               data_type: 'RepositoryTextValue'
+  end
+
+  let(:sheet) do
+    SpreadsheetParser.open_spreadsheet(fixture_file_upload('files/export.csv'))
+  end
+  let(:mappings) do
+    { '0' => '-1',
+      '1' => '',
+      '2' => '',
+      '3' => sample_group_column.id.to_s,
+      '4' => sample_type_column.id.to_s,
+      '5' => custom_column.id.to_s }
+  end
+
+  describe '#run/0' do
+    let(:subject) do
+      RepositoryImportParser::Importer.new(sheet, mappings, user, repository)
+    end
+
+    it 'return a message of imported records' do
+      expect(subject.run).to eq({ status: :ok, nr_of_added: 5, total_nr: 5 })
+    end
+
+    it 'generate 5 new repository rows' do
+      subject.run
+      expect(repository.repository_rows.count).to eq 5
+    end
+
+    it 'generate 3 new repository list items on Sample group column' do
+      subject.run
+      column = repository.repository_columns.find_by_name('Sample group')
+      expect(column.repository_list_items.count).to eq 3
+      column.repository_list_items.each do |repository_item|
+        expect(['group 1', 'group 2', 'group 3']).to include(repository_item.data)
+      end
+    end
+
+    it 'generate 2 new repository list items on Sample type column' do
+      subject.run
+      column = repository.repository_columns.find_by_name('Sample type')
+      expect(column.repository_list_items.count).to eq 2
+      column.repository_list_items.each do |repository_item|
+        expect(['type 1', 'type 2']).to include(repository_item.data)
+      end
+    end
+
+    it 'assign custom columns to imported repository row' do
+      subject.run
+      row = repository.repository_rows.find_by_name('Sample 1')
+      sample_group_cell = row.repository_cells
+                             .find_by_repository_column_id(
+                               sample_group_column.id
+                             )
+      sample_type_cell = row.repository_cells
+                            .find_by_repository_column_id(
+                              sample_type_column.id
+                            )
+      custom_column_cell = row.repository_cells
+                              .find_by_repository_column_id(
+                                custom_column.id
+                              )
+      expect(sample_group_cell.value.formatted).to eq 'group 1'
+      expect(sample_type_cell.value.formatted).to eq 'type 2'
+      expect(custom_column_cell.value.formatted).to eq 'test test'
+    end
+  end
+end

spec/utilities/repository_import_parser/repository_cell_value_resolver_spec.rb

@@ -0,0 +1,66 @@
+require 'rails_helper'
+
+describe RepositoryImportParser::RepositoryCellValueResolver do
+  let(:user) { create :user }
+  let(:team) { create :team, created_by: user }
+  let(:user_team) { create :user_team, user: user, team: team }
+  let(:repository) { create :repository, team: team, created_by: user }
+  let!(:sample_group_column) do
+    create :repository_column, repository: repository,
+                               created_by: user,
+                               name: 'Sample group',
+                               data_type: 'RepositoryListValue'
+  end
+  let!(:custom_column) do
+    create :repository_column, repository: repository,
+                               created_by: user,
+                               name: 'Custom items',
+                               data_type: 'RepositoryTextValue'
+  end
+
+  let!(:repository_row) do
+    create :repository_row, repository: repository,
+                            created_by: user,
+                            name: 'Sample'
+  end
+
+  describe '#ruget_valuen/2' do
+    context 'RepositoryListValue' do
+      let(:subject) do
+        RepositoryImportParser::RepositoryCellValueResolver.new(
+          sample_group_column, user, repository
+        )
+      end
+
+      it 'returns a valid RepositoryListValue object' do
+        value = subject.get_value('leaf', repository_row)
+        expect(value).to be_valid
+        expect(value).to be_a RepositoryListValue
+        expect(value.formatted).to eq 'leaf'
+      end
+
+      it 'creates a new RepositoryListItem' do
+        value = subject.get_value('leaf', repository_row)
+        item = sample_group_column.repository_list_items.find_by_data('leaf')
+        expect(sample_group_column.repository_list_items.count).to eq 1
+        expect(item).to be_present
+        expect(item.data).to eq 'leaf'
+      end
+    end
+
+    context 'RepositoryTextValue' do
+      let(:subject) do
+        RepositoryImportParser::RepositoryCellValueResolver.new(
+          custom_column, user, repository
+        )
+
+        it 'returns a valid RepositoryTextValue object' do
+          value = subject.get_value('blood', repository_row)
+          expect(value).to be_valid
+          expect(value).to be_a RepositoryTextValue
+          expect(value.formatted).to eq 'blood'
+        end
+      end
+    end
+  end
+end