From 75d7cce6b71f46737811b845f11ec43bd131efcc Mon Sep 17 00:00:00 2001 From: Luka Murn Date: Fri, 11 Jan 2019 09:35:25 +0100 Subject: [PATCH] Remove the samples -> repositories migration service/s & specs --- app/helpers/protocols_io_helper.rb | 2 +- ...samples_to_repository_migration_service.rb | 303 ------------------ .../samples_to_repository_migration.rake | 120 ------- 3 files changed, 1 insertion(+), 424 deletions(-) delete mode 100644 app/services/tasks/samples_to_repository_migration_service.rb delete mode 100644 lib/tasks/samples_to_repository_migration.rake diff --git a/app/helpers/protocols_io_helper.rb b/app/helpers/protocols_io_helper.rb index db52f3df1..9c6212fef 100644 --- a/app/helpers/protocols_io_helper.rb +++ b/app/helpers/protocols_io_helper.rb @@ -399,7 +399,7 @@ module ProtocolsIoHelper # (complex mapping with nested hashes) id 8 = software package, # id 9 = dataset, id 15 = command, id 18 = attached sub protocol # id 19= safety information , - # id 20= regents (materials, like scinote samples kind of) + # id 20= regents (materials, like scinote inventories kind of) original_json['steps'] = protocols_io_guid_reorder_step_json( original_json['steps'] diff --git a/app/services/tasks/samples_to_repository_migration_service.rb b/app/services/tasks/samples_to_repository_migration_service.rb deleted file mode 100644 index 0b15a6b65..000000000 --- a/app/services/tasks/samples_to_repository_migration_service.rb +++ /dev/null @@ -1,303 +0,0 @@ -# frozen_string_literal: true - -# Helper module for dealing with the migration from samples -# to custom repositories. We need to query with SQL because probably we will not -# have the "Sample" and other related models at the time this code will execute - -module Tasks - module SamplesToRepositoryMigrationService - include ActiveRecord::Sanitization::ClassMethods - - def self.prepare_repository(team, copy_num = 0) - repository = Repository.new( - name: copy_num > 0 ? "Samples (#{copy_num})" : 'Samples', - team: team, - created_by: team.created_by - ) - return repository if repository.save - prepare_repository(team, copy_num + 1) - end - - def self.prepare_text_value_custom_columns(team, repository) - custom_columns_sql = <<-SQL - SELECT * FROM custom_fields WHERE team_id = #{team.id} - SQL - # execute query - custom_columns = ActiveRecord::Base.connection.execute(custom_columns_sql) - - repository_columns = [] - custom_columns.each_with_index do |column, index| - repository_column = RepositoryColumn.create!( - repository: repository, - created_by_id: column.fetch('user_id') { repository.created_by_id }, - data_type: :RepositoryTextValue, - name: column.fetch('name') { "Custom column (#{index})" }, - created_at: column.fetch('created_at') { DateTime.now }, - updated_at: column.fetch('updated_at') { DateTime.now } - ) - repository_columns << repository_column - end - repository_columns - end - - def self.prepare_list_value_custom_columns_with_list_items(team, repository) - conn = ActiveRecord::Base.connection - sample_types_sql = <<-SQL - SELECT name, created_by_id, last_modified_by_id - FROM sample_types - WHERE team_id = #{team.id} - SQL - sample_groups_sql = <<-SQL - SELECT name, created_by_id, last_modified_by_id - FROM sample_groups - WHERE team_id = #{team.id} - SQL - # execute query - sample_types = conn.execute(sample_types_sql) - sample_groups = conn.execute(sample_groups_sql) - - sample_group = RepositoryColumn.create!( - repository: repository, - created_by_id: repository.created_by_id, - data_type: :RepositoryListValue, - name: 'Sample group' - ) - - # needs some random string to prevent duplications error - sample_group_color = RepositoryColumn.create!( - repository: repository, - created_by_id: repository.created_by_id, - data_type: :RepositoryTextValue, - name: 'Sample group color hex' - ) - - sample_type = RepositoryColumn.create!( - repository: repository, - created_by_id: repository.created_by_id, - data_type: :RepositoryListValue, - name: 'Sample type' - ) - - sample_groups.each_with_index do |item, index| - created_by = item['created_by_id'] || team.created_by_id - last_modified_by = item['last_modified_by_id'] || team.created_by_id - timestamp = conn.quote(Time.now.to_s(:db)) - values = [ - repository.id, - sample_group.id, - conn.quote(item.fetch('name') { "sample group item (#{index})" }), - created_by, - last_modified_by, - timestamp, - timestamp - ] - list_item_sql = <<-SQL - INSERT INTO repository_list_items - (repository_id, - repository_column_id, - data, - created_by_id, - last_modified_by_id, - created_at, updated_at) - VALUES (#{values.join(', ')}) - SQL - conn.execute(list_item_sql) - end - - sample_types.each_with_index do |item, index| - created_by = item['created_by_id'] || team.created_by_id - last_modified_by = item['last_modified_by_id'] || team.created_by_id - timestamp = conn.quote(Time.now.to_s(:db)) - values = [ - repository.id, - sample_type.id, - conn.quote(item.fetch('name') { "sample type item (#{index})" }), - created_by, - last_modified_by, - timestamp, - timestamp - ] - list_item_sql = <<-SQL - INSERT INTO repository_list_items - (repository_id, - repository_column_id, - data, - created_by_id, - last_modified_by_id, - created_at, updated_at) - VALUES (#{values.join(', ')}) - SQL - conn.execute(list_item_sql) - end - - [sample_group, sample_type, sample_group_color] - end - - def self.get_sample_custom_fields(sample_id) - custom_sample_fields_sql = <<-SQL - SELECT custom_fields.name AS column_name_reference, - sample_custom_fields.value, - sample_custom_fields.created_at, - sample_custom_fields.updated_at - FROM custom_fields - INNER JOIN sample_custom_fields - ON custom_fields.id = sample_custom_fields.custom_field_id - WHERE sample_custom_fields.sample_id = #{sample_id} - SQL - ActiveRecord::Base.connection.execute(custom_sample_fields_sql).to_a - end - - def self.get_assigned_sample_module(sample_id) - assigned_samples_sql = <<-SQL - SELECT my_module_id, assigned_by_id - FROM sample_my_modules - WHERE sample_my_modules.sample_id = #{sample_id} - SQL - ActiveRecord::Base.connection.execute(assigned_samples_sql).to_a - end - - def self.fetch_all_team_samples(team) - samples_sql = <<-SQL - SELECT samples.id AS sample_id, - samples.name AS sample_name, - samples.user_id AS sample_created_by_id, - samples.last_modified_by_id AS sample_last_modified_by_id, - samples.created_at AS sample_created_at, - samples.updated_at AS sample_updated_at, - sample_types.name AS sample_type_name, - sample_groups.name AS sample_group_name, - sample_groups.color AS sample_group_color - FROM samples - LEFT OUTER JOIN sample_types - ON samples.sample_type_id = sample_types.id - LEFT OUTER JOIN sample_groups - ON samples.sample_group_id = sample_groups.id - WHERE samples.team_id = #{team.id} - ORDER BY samples.id - SQL - - ActiveRecord::Base.connection.execute(samples_sql).to_a - end - - def self.get_custom_columns(team, repository) - prepare_text_value_custom_columns(team, repository) + - prepare_list_value_custom_columns_with_list_items(team, repository) - end - - def self.create_text_cell(row_id, column_id, data, - created_by_id, last_modified_by_id) - conn = ActiveRecord::Base.connection - timestamp = conn.quote(Time.now.to_s(:db)) - values = [conn.quote(data), created_by_id, last_modified_by_id, timestamp, timestamp] - value_sql = <<-SQL - INSERT INTO repository_text_values - (data, created_by_id, last_modified_by_id, - created_at, updated_at) - VALUES (#{values.join(', ')}) - RETURNING id - SQL - - value_id = conn.execute(value_sql)[0]['id'] - - values = [row_id, column_id, value_id, conn.quote('RepositoryTextValue'), - timestamp, timestamp] - cell_sql = <<-SQL - INSERT INTO repository_cells - (repository_row_id, repository_column_id, value_id, value_type, - created_at, updated_at) - VALUES (#{values.join(', ')}) - SQL - conn.execute(cell_sql) - end - - def self.create_list_cell(row_id, column_id, list_item_id, - created_by_id, last_modified_by_id) - conn = ActiveRecord::Base.connection - timestamp = conn.quote(Time.now.to_s(:db)) - values = [list_item_id, created_by_id, last_modified_by_id, - timestamp, timestamp] - list_value_sql = <<-SQL - INSERT INTO repository_list_values - (repository_list_item_id, created_by_id, last_modified_by_id, - created_at, updated_at) - VALUES (#{values.join(', ')}) - RETURNING id - SQL - value_id = ActiveRecord::Base.connection.execute(list_value_sql)[0]['id'] - - values = [row_id, column_id, value_id, conn.quote('RepositoryListValue'), - timestamp, timestamp] - cell_sql = <<-SQL - INSERT INTO repository_cells - (repository_row_id, repository_column_id, value_id, value_type, - created_at, updated_at) - VALUES (#{values.join(', ')}) - SQL - ActiveRecord::Base.connection.execute(cell_sql) - end - - def self.update_smart_annotations(team, mappings) - team.projects.eager_load(:project_comments).each do |pr| - pr.project_comments.each do |comment| - comment.save! if update_annotation(comment.message, mappings) - end - pr.experiments.each do |exp| - exp.save! if update_annotation(exp.description, mappings) - exp.my_modules.eager_load(:task_comments).each do |task| - task.task_comments.each do |comment| - comment.save! if update_annotation(comment.message, mappings) - end - task.save! if update_annotation(task.description, mappings) - task.protocol.steps.eager_load(:step_comments).each do |step| - step.step_comments.each do |comment| - comment.save! if update_annotation(comment.message, mappings) - end - step.save! if update_annotation(step.description, mappings) - end - task.results.eager_load(:result_comments, :result_text).each do |res| - res.result_comments.each do |comment| - comment.save! if update_annotation(comment.message, mappings) - end - next unless res.result_text - res.save! if update_annotation(res.result_text.text, mappings) - end - end - end - end - team.protocols.where(my_module: nil).each do |protocol| - protocol.steps.eager_load(:step_comments).each do |step| - step.step_comments.each do |comment| - comment.save! if update_annotation(comment.message, mappings) - end - step.save! if update_annotation(step.description, mappings) - end - end - team.repositories.each do |rep| - rep.repository_rows.includes(repository_cells: :repository_text_value) - .where('repository_cells.value_type': 'RepositoryTextValue') - .each do |row| - row.repository_cells.each do |cell| - if update_annotation(cell.repository_text_value.data, mappings) - cell.repository_text_value.save! - end - end - end - end - end - - # Returns true if text was updated - def self.update_annotation(text, sample_mappings) - return false if text.nil? - updated = false - text.scan(/~sam~\w+\]/).each do |text_match| - orig_id_encoded = text_match.match(/~sam~(\w+)\]/)[1] - orig_id = orig_id_encoded.base62_decode - next unless sample_mappings[orig_id] - new_id_encoded = sample_mappings[orig_id].base62_encode - text.sub!("~sam~#{orig_id_encoded}]", "~rep_item~#{new_id_encoded}]") - updated = true - end - updated - end - end -end diff --git a/lib/tasks/samples_to_repository_migration.rake b/lib/tasks/samples_to_repository_migration.rake deleted file mode 100644 index 64575ce10..000000000 --- a/lib/tasks/samples_to_repository_migration.rake +++ /dev/null @@ -1,120 +0,0 @@ -require_relative '../../app/services/tasks/samples_to_repository_migration_service' - -namespace :samples_to_repository_migration do - desc 'Migrates all data from samples to custom repository' - task :run, [:last_id] => :environment do |_, args| - include ActiveRecord::Sanitization::ClassMethods - - params = { batch_size: 10 } - migration_service = Tasks::SamplesToRepositoryMigrationService - conn = ActiveRecord::Base.connection - if args.present? && args[:last_id].present? - params[:start] = args[:last_id].to_i - end - Team.find_each(params) do |team| - puts "******************************* \n\n" - puts "Processing Team id => [#{team.id}] \n\n" - puts '*******************************' - - ActiveRecord::Base.transaction do - team_samples = migration_service.fetch_all_team_samples(team) - repository = migration_service.prepare_repository(team) - custom_columns = migration_service.get_custom_columns(team, repository) - sample_mappings = {} - team_samples.each do |item| - created_by = item['sample_created_by_id'] || team.created_by_id - last_modified_by = item['sample_last_modified_by_id'] - last_modified_by ||= team.created_by_id - timestamp = conn.quote(Time.now.to_s(:db)) - values = [repository.id, created_by, last_modified_by, - conn.quote(item['sample_name']), timestamp, timestamp] - list_item_sql = <<-SQL - INSERT INTO repository_rows - (repository_id, created_by_id, last_modified_by_id, name, - created_at, updated_at) - VALUES (#{values.join(', ')}) - RETURNING id - SQL - result = conn.execute(list_item_sql) - row_id = result[0]['id'] - sample_mappings[item['sample_id']] = row_id - - # check if sample has sample type assigned - if item['sample_type_name'] - column = custom_columns.detect { |el| el['name'] == 'Sample type' } - list_item = column.repository_list_items.where( - data: item['sample_type_name'] - ).take - migration_service.create_list_cell( - row_id, column.id, list_item.id, - list_item.created_by_id, list_item.last_modified_by_id - ) - end - - # check if sample has sample group assigned - if item['sample_group_name'] - column = custom_columns.detect { |el| el['name'] == 'Sample group' } - list_item = column.repository_list_items.where( - data: item['sample_group_name'] - ).take - migration_service.create_list_cell( - row_id, column.id, list_item.id, - list_item.created_by_id, list_item.last_modified_by_id - ) - - # assign sample group color to the sample - if item['sample_group_color'] - column = custom_columns.detect do |el| - el['name'] == 'Sample group color hex' - end - migration_service.create_text_cell( - row_id, column.id, item['sample_group_color'], - created_by, last_modified_by - ) - end - end - - # append custom fields - custom_fields = migration_service.get_sample_custom_fields( - item['sample_id'] - ) - custom_fields.each do |field| - column = custom_columns.detect do |el| - el['name'] == field['column_name_reference'] - end - migration_service - .create_text_cell(row_id, column.id, field['value'], - created_by, last_modified_by) - end - - # assign repository item to a tasks - assigned_modules = migration_service.get_assigned_sample_module( - item['sample_id'] - ) - assigned_modules.each do |element| - assigned_by = element['assigned_by_id'] || created_by - values = [row_id, element['my_module_id'], assigned_by, - timestamp, timestamp] - cell_sql = <<-SQL - INSERT INTO my_module_repository_rows - (repository_row_id, my_module_id, assigned_by_id, - created_at, updated_at) - VALUES (#{values.join(', ')}) - SQL - conn.execute(cell_sql) - end - end - - # Update report elements - team.reports.each do |r| - r.report_elements.where(type_of: 7).each do |e| - e.update_attributes(type_of: 17, repository_id: repository.id) - end - end - - # Now update smart annotations - migration_service.update_smart_annotations(team, sample_mappings) - end - end - end -end