Merge pull request #998 from ZmagoD/zd_SCI_2014

migration script for migrating samples -> custom repositories [SCI-2014]

app/services/tasks/samples_to_repository_migration_service.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+# Helper module for dealing with the migration from samples
+# to custom repositories. We need to query with SQL because probably we will not
+# have the "Sample" and other related models at the time this code will execute
+
+module Tasks
+  module SamplesToRepositoryMigrationService
+    def self.prepare_repository(team, copy_num = 0)
+      repository = Repository.new(
+        name: copy_num > 0 ? "Samples (#{copy_num})" : 'Samples',
+        team: team,
+        created_by: team.created_by
+      )
+      return repository if repository.save
+      prepare_repository(team, copy_num + 1)
+    end
+
+    def self.prepare_text_value_custom_columns(team, repository)
+      custom_columns_sql = <<-SQL
+        SELECT * FROM custom_fields WHERE team_id = #{team.id}
+      SQL
+      # execute query
+      custom_columns = ActiveRecord::Base.connection.execute(custom_columns_sql)
+
+      repository_columns = []
+      custom_columns.each_with_index do |column, index|
+        repository_column = RepositoryColumn.create!(
+          repository: repository,
+          created_by_id: column.fetch('user_id') { repository.created_by_id },
+          data_type: :RepositoryTextValue,
+          name: column.fetch('name') { "Custom column (#{index})" },
+          created_at: column.fetch('created_at') { DateTime.now },
+          updated_at: column.fetch('updated_at') { DateTime.now }
+        )
+        repository_columns << repository_column
+      end
+      repository_columns
+    end
+
+    def self.prepare_list_value_custom_columns_with_list_items(team, repository)
+      sample_types_sql = <<-SQL
+        SELECT name, created_by_id, last_modified_by_id
+        FROM sample_types
+        WHERE team_id = #{team.id}
+      SQL
+      sample_groups_sql = <<-SQL
+        SELECT name, created_by_id, last_modified_by_id
+        FROM sample_groups
+        WHERE team_id = #{team.id}
+      SQL
+      # execute query
+      sample_types = ActiveRecord::Base.connection.execute(sample_types_sql)
+      sample_groups = ActiveRecord::Base.connection.execute(sample_groups_sql)
+
+      sample_group = RepositoryColumn.create!(
+        repository: repository,
+        created_by_id: repository.created_by_id,
+        data_type: :RepositoryListValue,
+        name: 'Sample group'
+      )
+
+      # needs some random string to prevent duplications error
+      sample_group_color = RepositoryColumn.create!(
+        repository: repository,
+        created_by_id: repository.created_by_id,
+        data_type: :RepositoryTextValue,
+        name: 'Sample group color hex'
+      )
+
+      sample_type = RepositoryColumn.create!(
+        repository: repository,
+        created_by_id: repository.created_by_id,
+        data_type: :RepositoryListValue,
+        name: 'Sample type'
+      )
+
+      sample_groups.each_with_index do |item, index|
+        created_by = item['created_by_id'] || team.created_by_id
+        last_modified_by = item['last_modified_by_id'] || team.created_by_id
+        RepositoryListItem.create!(
+          data: item.fetch('name') { "sample group item (#{index})" },
+          created_by_id: created_by,
+          last_modified_by_id: last_modified_by,
+          repository_column: sample_group,
+          repository: repository
+        )
+      end
+
+      sample_types.each_with_index do |item, index|
+        created_by = item['created_by_id'] || team.created_by_id
+        last_modified_by = item['last_modified_by_id'] || team.created_by_id
+        RepositoryListItem.create!(
+          data: item.fetch('name') { "sample group item (#{index})" },
+          created_by_id: created_by,
+          last_modified_by_id: last_modified_by,
+          repository_column: sample_type,
+          repository: repository
+        )
+      end
+
+      [sample_group, sample_type, sample_group_color]
+    end
+
+    def self.get_sample_custom_fields(sample_id)
+      custom_sample_fields_sql = <<-SQL
+        SELECT custom_fields.name AS column_name_reference,
+               sample_custom_fields.value,
+               sample_custom_fields.created_at,
+               sample_custom_fields.updated_at
+        FROM custom_fields
+        INNER JOIN sample_custom_fields
+        ON custom_fields.id = sample_custom_fields.custom_field_id
+        WHERE sample_custom_fields.sample_id = #{sample_id}
+      SQL
+      ActiveRecord::Base.connection.execute(custom_sample_fields_sql).to_a
+    end
+
+    def self.get_assigned_sample_module(sample_id)
+      assigned_samples_sql = <<-SQL
+        SELECT my_module_id, assigned_by_id
+        FROM sample_my_modules
+        WHERE sample_my_modules.sample_id = #{sample_id}
+      SQL
+      ActiveRecord::Base.connection.execute(assigned_samples_sql).to_a
+    end
+
+    def self.fetch_all_team_samples(team)
+      samples_sql = <<-SQL
+        SELECT samples.id AS sample_id,
+               samples.name AS sample_name,
+               samples.user_id AS sample_created_by_id,
+               samples.last_modified_by_id AS sample_last_modified_by_id,
+               samples.created_at AS sample_created_at,
+               samples.updated_at AS sample_updated_at,
+               sample_types.name AS sample_type_name,
+               sample_groups.name AS sample_group_name,
+               sample_groups.color AS sample_group_color
+        FROM samples
+        LEFT OUTER JOIN sample_types
+        ON samples.sample_type_id = sample_types.id
+        LEFT OUTER JOIN sample_groups
+        ON samples.sample_type_id = sample_groups.id
+        WHERE samples.team_id = #{team.id}
+      SQL
+
+      ActiveRecord::Base.connection.execute(samples_sql).to_a
+    end
+
+    def self.get_custom_columns(team, repository)
+      prepare_text_value_custom_columns(team, repository) +
+        prepare_list_value_custom_columns_with_list_items(team, repository)
+    end
+  end
+end

lib/tasks/samples_to_repository_migration.rake

@@ -0,0 +1,117 @@
+require_relative '../../app/services/tasks/samples_to_repository_migration_service'
+
+namespace :samples_to_repository_migration do
+  desc 'Migrates all data from samples to custom repository'
+  task :run, [:last_id] => :environment do |_, args|
+    params = { batch_size: 10 }
+    migration_service = Tasks::SamplesToRepositoryMigrationService
+    if args.present? && args[:last_id].present?
+      params[:start] = args[:last_id].to_i
+    end
+    Team.find_each(params) do |team|
+      puts "******************************* \n\n"
+      puts "Processing Team id => [#{team.id}] \n\n"
+      puts '*******************************'
+
+      ActiveRecord::Base.transaction do
+        team_samples = migration_service.fetch_all_team_samples(team)
+        repository = migration_service.prepare_repository(team)
+        custom_columns = migration_service.get_custom_columns(team, repository)
+
+        team_samples.each do |item|
+          created_by = item['sample_created_by_id'] || team.created_by_id
+          last_modified_by = item['sample_last_modified_by_id']
+          last_modified_by ||= team.created_by_id
+          row = RepositoryRow.create!(
+            name: item['sample_name'],
+            created_at: item['sample_created_at'],
+            updated_at: item['sample_updated_at'],
+            created_by_id: created_by,
+            last_modified_by_id: last_modified_by,
+            repository: repository
+          )
+          # check if sample has sample type assigned
+          if item['sample_type_name']
+            column = custom_columns.detect { |el| el['name'] == 'Sample type' }
+            list_item = column.repository_list_items.where(
+              data: item['sample_type_name']
+            ).take
+            RepositoryListValue.create!(
+              created_by: list_item.created_by,
+              last_modified_by: list_item.last_modified_by,
+              repository_list_item: list_item,
+              repository_cell_attributes: {
+                repository_row: row,
+                repository_column: column
+              }
+            )
+          end
+
+          # check if sample has sample group assigned
+          if item['sample_group_name']
+            column = custom_columns.detect { |el| el['name'] == 'Sample group' }
+            list_item = column.repository_list_items.where(
+              data: item['sample_group_name']
+            ).take
+            RepositoryListValue.create!(
+              created_by: list_item.created_by,
+              last_modified_by: list_item.last_modified_by,
+              repository_list_item: list_item,
+              repository_cell_attributes: {
+                repository_row: row,
+                repository_column: column
+              }
+            )
+
+            # assign sample group color to the sample
+            if item['sample_group_color']
+              column = custom_columns.detect do |el|
+                el['name'] == 'Sample group color hex'
+              end
+              RepositoryTextValue.create!(
+                data: item['sample_group_color'],
+                created_by_id: created_by,
+                last_modified_by_id: last_modified_by,
+                repository_cell_attributes: {
+                  repository_row: row,
+                  repository_column: column
+                }
+              )
+            end
+          end
+
+          # append custom fields
+          custom_fields = migration_service.get_sample_custom_fields(
+            item['sample_id']
+          )
+          custom_fields.each do |field|
+            column = custom_columns.detect do |el|
+              el['name'] == field['column_name_reference']
+            end
+            RepositoryTextValue.create!(
+              data: field['value'],
+              created_by_id: created_by,
+              last_modified_by_id: last_modified_by,
+              repository_cell_attributes: {
+                repository_row: row,
+                repository_column: column
+              }
+            )
+          end
+
+          # assign repository item to a tasks
+          assigned_modules = migration_service.get_assigned_sample_module(
+            item['sample_id']
+          )
+          assigned_modules.each do |element|
+            MyModuleRepositoryRow.create!(
+              my_module_id: element['my_module_id'],
+              repository_row: row,
+              assigned_by_id: element['assigned_by_id'] || created_by
+            )
+          end
+        end
+      end
+    end
+  end
+end

spec/factories/sample_custom_field.rb

@@ -0,0 +1,5 @@
+FactoryBot.define do
+  factory :sample_custom_field do
+    value 'Sample 111'
+  end
+end

spec/lib/tasks/samples_to_repository_migration_spec.rb

@@ -0,0 +1,76 @@
+require 'rails_helper'
+
+describe 'samples_to_repository_migration:run' do
+  include_context 'rake'
+  let!(:user) { create :user, email: 'happy.user@scinote.net' }
+  let!(:team) { create :team, created_by: user }
+  let!(:user_team) { create :user_team, user: user, team: team }
+  let!(:my_module) { create :my_module }
+  let!(:samples_table) { create :samples_table, user: user, team: team }
+  let(:sample_types_names) { %w(type_one type_two type_three) }
+  let(:sample_group_names) { %w(group_one group_two group_three) }
+
+  before do
+    sample_types = []
+    sample_types_names.each do |name|
+      sample_type = create :sample_type, name: name, team: team
+      sample_types << sample_type
+    end
+    sample_groups = []
+    sample_group_names.each do |name|
+      sample_group = create :sample_group, name: name, team: team
+      sample_groups << sample_group
+    end
+    custom_field = create :custom_field, name: 'Banana', team: team, user: user
+    100.times do |index|
+      sample = create :sample, name: "Sample (#{index})", user: user, team: team
+      create :sample_my_module, sample: sample, my_module: my_module
+      sample.sample_type = sample_types[rand(0...2)]
+      sample.sample_group = sample_groups[rand(0...2)]
+      custom_value = create :sample_custom_field,
+                            value: "custom value (#{index})",
+                            custom_field: custom_field,
+                            sample: sample
+      sample.sample_custom_fields << custom_value
+      sample.save
+    end
+  end
+
+  it 'generates a new custom repository with exact copy of samples' do
+    subject.invoke
+    expect(Repository.first.name).to eq 'Samples'
+    expect(RepositoryRow.count).to eq 100
+    RepositoryRow.all.each do |row|
+      row_my_module = MyModuleRepositoryRow.where(repository_row: row,
+                                                  my_module: my_module)
+      expect(row_my_module).to exist
+      expect(row.name).to match(/Sample \([0-9]*\)/)
+      expect(row.created_by).to eq user
+
+      # repository sample_type column
+      sample_type_column = row.repository_cells.first
+      expect(sample_types_names).to include(sample_type_column.value.formatted)
+      expect(sample_type_column.repository_column.name).to eq 'Sample type'
+      expect(sample_type_column.value_type).to eq 'RepositoryListValue'
+
+      # repository sample_group column
+      sample_group_column = row.repository_cells.second
+      expect(sample_group_names).to include(sample_group_column.value.formatted)
+      expect(sample_group_column.repository_column.name).to eq 'Sample group'
+      expect(sample_group_column.value_type).to eq 'RepositoryListValue'
+
+      # repository color column
+      color_column = row.repository_cells.third
+      expect(
+        color_column.repository_column.name
+      ).to eq 'Sample group color hex'
+      expect(color_column.value_type).to eq 'RepositoryTextValue'
+
+      # repository custom column
+      custom_column = row.repository_cells.last
+      expect(custom_column.value.formatted).to match(/custom value \([0-9]*\)/)
+      expect(custom_column.repository_column.name).to eq 'Banana'
+      expect(custom_column.value_type).to eq 'RepositoryTextValue'
+    end
+  end
+end

spec/services/tasks/samples_to_repository_migration_service_spec.rb

@@ -0,0 +1,307 @@
+require 'rails_helper'
+
+describe Tasks::SamplesToRepositoryMigrationService do
+  let(:user) { create :user, email: 'happy.user@scinote.net' }
+  let(:team) { create :team, created_by: user }
+  let(:user_team) { create :user_team, user: user, team: team }
+
+  describe '#prepare_repository/2' do
+    context 'creates and return a new custom repository named' do
+      it '"Samples" for team' do
+        repository = Tasks::SamplesToRepositoryMigrationService
+                     .prepare_repository(team)
+        expect(repository).to be_an_instance_of(Repository)
+        expect(repository.team).to eq team
+        expect(repository.name).to eq 'Samples'
+      end
+
+      it '"Samples (1)" if repository name "Samples" already exists' do
+        create :repository, name: 'Samples', team: team, created_by: user
+        repository = Tasks::SamplesToRepositoryMigrationService
+                     .prepare_repository(team)
+        expect(repository).to be_an_instance_of(Repository)
+        expect(repository.team).to eq team
+        expect(repository.name).to eq 'Samples (1)'
+      end
+    end
+  end
+
+  describe '#prepare_text_value_custom_columns/2' do
+    let(:repository) do
+      create :repository, name: 'Samples', team: team, created_by: user
+    end
+    let(:subject) do
+      Tasks::SamplesToRepositoryMigrationService
+        .prepare_text_value_custom_columns(team, repository)
+    end
+
+    context 'custom columns exists' do
+      before do
+        create :samples_table, user: user, team: team
+        10.times do |index|
+          create :custom_field, name: "My Custom field (#{index})",
+                                user: user,
+                                team: team,
+                                last_modified_by: user
+        end
+      end
+
+      it { is_expected.to be_an Array }
+      it { expect(subject.length).to eq 10 }
+      it { expect(subject.first.name).to eq 'My Custom field (0)' }
+      it { expect(subject.last.name).to eq 'My Custom field (9)' }
+      it { expect(subject.first).to be_an_instance_of RepositoryColumn }
+      it { expect(subject.first.data_type).to eq 'RepositoryTextValue' }
+      it { expect(subject.first.repository).to eq repository }
+    end
+
+    context 'custom columns does not exists' do
+      it { is_expected.to be_an Array }
+      it { is_expected.to be_empty }
+    end
+  end
+
+  describe '#prepare_list_value_custom_columns_with_list_items/2' do
+    let(:repository) do
+      create :repository, name: 'Samples', team: team, created_by: user
+    end
+    let(:subject) do
+      Tasks::SamplesToRepositoryMigrationService
+        .prepare_list_value_custom_columns_with_list_items(team, repository)
+    end
+
+    context 'with samples types' do
+      before do
+        10.times do |index|
+          create :sample_type, name: "Sample Type Item (#{index})",
+                               team: team,
+                               created_by: user,
+                               last_modified_by: user
+        end
+      end
+
+      it { is_expected.to be_an Array }
+      it { expect(subject.length).to eq 3 }
+      it { expect(subject.first).to be_an_instance_of(RepositoryColumn) }
+      it { expect(subject.first.name).to eq 'Sample group' }
+      it { expect(subject.first.data_type).to eq 'RepositoryListValue' }
+      it { expect(subject.second.name).to eq 'Sample type' }
+      it { expect(subject.second.data_type).to eq 'RepositoryListValue' }
+      it { expect(subject.last.name).to eq 'Sample group color hex' }
+      it { expect(subject.last.data_type).to eq 'RepositoryTextValue' }
+
+      describe 'generated list items from sample types' do
+        let!(:generated_list_items) { subject.second.repository_list_items }
+        it { expect(generated_list_items.count).to eq 10 }
+
+        it 'has generated list_items with similar properties' do
+          generated_list_items.each_with_index do |item, index|
+            expect(item.data).to eq "Sample Type Item (#{index})"
+            expect(item).to be_an_instance_of RepositoryListItem
+            expect(item.created_by).to eq user
+            expect(item.last_modified_by).to eq user
+          end
+        end
+      end
+
+      describe 'sample type without created_at/last_modified_by field' do
+        before do
+          team.sample_types.update_all(created_by_id: nil,
+                                       last_modified_by_id: nil)
+        end
+
+        it 'generates valid list_items' do
+          generated_list_items = subject.second.repository_list_items
+          expect(generated_list_items.count).to eq 10
+          generated_list_items.each_with_index do |item, index|
+            expect(item.data).to eq "Sample Type Item (#{index})"
+            expect(item).to be_an_instance_of RepositoryListItem
+            expect(item.created_by).to eq team.created_by
+            expect(item.last_modified_by).to eq team.created_by
+          end
+        end
+      end
+    end
+
+    context 'with samples groups' do
+      before do
+        10.times do |index|
+          create :sample_group, name: "Sample Group Item (#{index})",
+                                color: '#000000',
+                                team: team,
+                                created_by: user,
+                                last_modified_by: user
+        end
+      end
+
+      it { is_expected.to be_an Array }
+      it { expect(subject.length).to eq 3 }
+      it { expect(subject.first).to be_an_instance_of(RepositoryColumn) }
+      it { expect(subject.last).to be_an_instance_of(RepositoryColumn) }
+      it { expect(subject.first.name).to eq 'Sample group' }
+      it { expect(subject.first.data_type).to eq 'RepositoryListValue' }
+      it { expect(subject.second.name).to eq 'Sample type' }
+      it { expect(subject.second.data_type).to eq 'RepositoryListValue' }
+      it { expect(subject.last.name).to eq 'Sample group color hex' }
+      it { expect(subject.last.data_type).to eq 'RepositoryTextValue' }
+
+      describe 'generated list items from sample groups' do
+        let!(:generated_list_items) { subject.first.repository_list_items }
+        it { expect(generated_list_items.count).to eq 10 }
+
+        it 'has generated list_items with similar properties' do
+          generated_list_items.each_with_index do |item, index|
+            expect(item.data).to eq "Sample Group Item (#{index})"
+            expect(item).to be_an_instance_of RepositoryListItem
+            expect(item.created_by).to eq user
+            expect(item.last_modified_by).to eq user
+          end
+        end
+      end
+
+      describe 'sample group without created_at/last_modified_by field' do
+        before do
+          team.sample_groups.update_all(created_by_id: nil,
+                                        last_modified_by_id: nil)
+        end
+
+        it 'generates valid list_items' do
+          generated_list_items = subject.first.repository_list_items
+          expect(generated_list_items.count).to eq 10
+          generated_list_items.each_with_index do |item, index|
+            expect(item.data).to eq "Sample Group Item (#{index})"
+            expect(item).to be_an_instance_of RepositoryListItem
+            expect(item.created_by).to eq team.created_by
+            expect(item.last_modified_by).to eq team.created_by
+          end
+        end
+      end
+    end
+  end
+
+  describe '#get_sample_custom_fields/1' do
+    let(:sample) { create :sample, name: 'My sample', user: user, team: team }
+    let(:custom_field) do
+      create :custom_field, name: 'My Custom column',
+                            user: user,
+                            team: team,
+                            last_modified_by: user
+    end
+
+    let(:subject) do
+      Tasks::SamplesToRepositoryMigrationService
+        .get_sample_custom_fields(sample.id)
+    end
+    context 'sample has custom column assigned' do
+      before do
+        create :samples_table, user: user, team: team
+        create :sample_custom_field, value: 'field value',
+                                     custom_field: custom_field,
+                                     sample: sample
+      end
+
+      it 'returns a hash of sample values' do
+        element = subject.first
+        is_expected.to be_an Array
+        expect(subject.length).to eq 1
+        expect(element.fetch('column_name_reference')).to eq 'My Custom column'
+        expect(element.fetch('value')).to eq 'field value'
+        expect(element.fetch('created_at')).to be_present
+        expect(element.fetch('updated_at')).to be_present
+      end
+    end
+
+    context 'sample does not have custom columns assigned' do
+      it { is_expected.to be_an Array }
+      it { is_expected.to be_empty }
+    end
+  end
+
+  describe '#get_assigned_sample_module/1' do
+    let(:sample) { create :sample, name: 'My sample', user: user, team: team }
+    let(:my_module) { create :my_module }
+    let(:subject) do
+      Tasks::SamplesToRepositoryMigrationService
+        .get_assigned_sample_module(sample.id)
+    end
+
+    context 'sample is assigned to one module' do
+      let!(:sample_my_module) do
+        create :sample_my_module, sample: sample,
+                                  my_module: my_module,
+                                  assigned_by: user
+      end
+
+      it { is_expected.to be_an Array }
+      it { expect(subject.length).to eq 1 }
+      it 'returnes assigned my_module data' do
+        my_module_data = subject.first
+        expect(
+          my_module_data.fetch('my_module_id')
+        ).to eq sample_my_module.my_module_id
+        expect(
+          my_module_data.fetch('assigned_by_id')
+        ).to eq sample_my_module.assigned_by_id
+      end
+    end
+
+    context 'sample is not assigned to module' do
+      it { is_expected.to be_an Array }
+      it { is_expected.to be_empty }
+    end
+
+    context 'sample is assigned to multiple modules' do
+      before do
+        @modules_ids = []
+        10.times do |index|
+          my_module = create :my_module, name: "My module (#{index})"
+          @modules_ids << my_module.id
+        end
+        10.times do |index|
+          create :sample_my_module,
+                 sample: sample,
+                 my_module_id: @modules_ids[index],
+                 assigned_by: user
+        end
+      end
+
+      it { is_expected.to be_an Array }
+      it { expect(subject.length).to eq 10 }
+
+      it 'is expected to return an array of samples_my_modules data' do
+        subject.each do |element|
+          expect(@modules_ids).to include(element.fetch('my_module_id').to_i)
+          expect(element.fetch('assigned_by_id')).to eq user.id
+        end
+      end
+    end
+  end
+
+  describe 'fetch_all_team_samples/1' do
+    let(:subject) do
+      Tasks::SamplesToRepositoryMigrationService.fetch_all_team_samples(team)
+    end
+
+    context 'team has samples' do
+      before do
+        100.times do |index|
+          create :sample, name: "Sample (#{index})", user: user, team: team
+        end
+      end
+
+      it { is_expected.to be_an Array }
+      it { expect(subject.length).to eq 100 }
+
+      it 'returns an array of all team samples' do
+        subject.each_with_index do |element, index|
+          expect(element.fetch('sample_name')). to eq "Sample (#{index})"
+        end
+      end
+    end
+
+    context 'team does not have samples' do
+      it { is_expected.to be_an Array }
+      it { is_expected.to be_empty }
+    end
+  end
+end

spec/support/shared_contexts/rake.rb

@@ -0,0 +1,21 @@
+require 'rake'
+
+shared_context 'rake' do
+  let(:rake)      { Rake::Application.new }
+  let(:task_name) { self.class.top_level_description }
+  let(:task_path) { "lib/tasks/#{task_name.split(':').first}" }
+  subject         { rake[task_name] }
+
+  def loaded_files_excluding_current_rake_file
+    $".reject { |file| file == Rails.root.join("#{task_path}.rake").to_s }
+  end
+
+  before do
+    Rake.application = rake
+    Rake.application.rake_require(task_path,
+                                  [Rails.root.to_s],
+                                  loaded_files_excluding_current_rake_file)
+
+    Rake::Task.define_task(:environment)
+  end
+end