diff --git a/app/services/tasks/samples_to_repository_migration_service.rb b/app/services/tasks/samples_to_repository_migration_service.rb index 1cce142f0..1889a371a 100644 --- a/app/services/tasks/samples_to_repository_migration_service.rb +++ b/app/services/tasks/samples_to_repository_migration_service.rb @@ -6,6 +6,8 @@ module Tasks module SamplesToRepositoryMigrationService + include ActiveRecord::Sanitization::ClassMethods + def self.prepare_repository(team, copy_num = 0) repository = Repository.new( name: copy_num > 0 ? "Samples (#{copy_num})" : 'Samples', @@ -39,6 +41,7 @@ module Tasks end def self.prepare_list_value_custom_columns_with_list_items(team, repository) + conn = ActiveRecord::Base.connection sample_types_sql = <<-SQL SELECT name, created_by_id, last_modified_by_id FROM sample_types @@ -50,8 +53,8 @@ module Tasks WHERE team_id = #{team.id} SQL # execute query - sample_types = ActiveRecord::Base.connection.execute(sample_types_sql) - sample_groups = ActiveRecord::Base.connection.execute(sample_groups_sql) + sample_types = conn.execute(sample_types_sql) + sample_groups = conn.execute(sample_groups_sql) sample_group = RepositoryColumn.create!( repository: repository, @@ -78,25 +81,53 @@ module Tasks sample_groups.each_with_index do |item, index| created_by = item['created_by_id'] || team.created_by_id last_modified_by = item['last_modified_by_id'] || team.created_by_id - RepositoryListItem.create!( - data: item.fetch('name') { "sample group item (#{index})" }, - created_by_id: created_by, - last_modified_by_id: last_modified_by, - repository_column: sample_group, - repository: repository - ) + timestamp = conn.quote(Time.now.to_s(:db)) + values = [ + repository.id, + sample_group.id, + conn.quote(item.fetch('name') { "sample group item (#{index})" }), + created_by, + last_modified_by, + timestamp, + timestamp + ] + list_item_sql = <<-SQL + INSERT INTO repository_list_items + (repository_id, + repository_column_id, + data, + created_by_id, + last_modified_by_id, + created_at, updated_at) + VALUES (#{values.join(', ')}) + SQL + conn.execute(list_item_sql) end sample_types.each_with_index do |item, index| created_by = item['created_by_id'] || team.created_by_id last_modified_by = item['last_modified_by_id'] || team.created_by_id - RepositoryListItem.create!( - data: item.fetch('name') { "sample group item (#{index})" }, - created_by_id: created_by, - last_modified_by_id: last_modified_by, - repository_column: sample_type, - repository: repository - ) + timestamp = conn.quote(Time.now.to_s(:db)) + values = [ + repository.id, + sample_type.id, + conn.quote(item.fetch('name') { "sample type item (#{index})" }), + created_by, + last_modified_by, + timestamp, + timestamp + ] + list_item_sql = <<-SQL + INSERT INTO repository_list_items + (repository_id, + repository_column_id, + data, + created_by_id, + last_modified_by_id, + created_at, updated_at) + VALUES (#{values.join(', ')}) + SQL + conn.execute(list_item_sql) end [sample_group, sample_type, sample_group_color] @@ -140,7 +171,7 @@ module Tasks LEFT OUTER JOIN sample_types ON samples.sample_type_id = sample_types.id LEFT OUTER JOIN sample_groups - ON samples.sample_type_id = sample_groups.id + ON samples.sample_group_id = sample_groups.id WHERE samples.team_id = #{team.id} SQL @@ -151,5 +182,121 @@ module Tasks prepare_text_value_custom_columns(team, repository) + prepare_list_value_custom_columns_with_list_items(team, repository) end + + def self.create_text_cell(row_id, column_id, data, + created_by_id, last_modified_by_id) + conn = ActiveRecord::Base.connection + timestamp = conn.quote(Time.now.to_s(:db)) + values = [conn.quote(data), created_by_id, last_modified_by_id, timestamp, timestamp] + value_sql = <<-SQL + INSERT INTO repository_text_values + (data, created_by_id, last_modified_by_id, + created_at, updated_at) + VALUES (#{values.join(', ')}) + RETURNING id + SQL + + value_id = conn.execute(value_sql)[0]['id'] + + values = [row_id, column_id, value_id, conn.quote('RepositoryTextValue'), + timestamp, timestamp] + cell_sql = <<-SQL + INSERT INTO repository_cells + (repository_row_id, repository_column_id, value_id, value_type, + created_at, updated_at) + VALUES (#{values.join(', ')}) + SQL + conn.execute(cell_sql) + end + + def self.create_list_cell(row_id, column_id, list_item_id, + created_by_id, last_modified_by_id) + conn = ActiveRecord::Base.connection + timestamp = conn.quote(Time.now.to_s(:db)) + values = [list_item_id, created_by_id, last_modified_by_id, + timestamp, timestamp] + list_value_sql = <<-SQL + INSERT INTO repository_list_values + (repository_list_item_id, created_by_id, last_modified_by_id, + created_at, updated_at) + VALUES (#{values.join(', ')}) + RETURNING id + SQL + value_id = ActiveRecord::Base.connection.execute(list_value_sql)[0]['id'] + + values = [row_id, column_id, value_id, conn.quote('RepositoryListValue'), + timestamp, timestamp] + cell_sql = <<-SQL + INSERT INTO repository_cells + (repository_row_id, repository_column_id, value_id, value_type, + created_at, updated_at) + VALUES (#{values.join(', ')}) + SQL + ActiveRecord::Base.connection.execute(cell_sql) + end + + def self.update_smart_annotations(team, mappings) + team.projects.eager_load(:project_comments).each do |pr| + pr.project_comments.each do |comment| + comment.save! if update_annotation(comment.message, mappings) + end + pr.experiments.each do |exp| + exp.save! if update_annotation(exp.description, mappings) + exp.my_modules.eager_load(:task_comments).each do |task| + task.task_comments.each do |comment| + comment.save! if update_annotation(comment.message, mappings) + end + task.save! if update_annotation(task.description, mappings) + task.protocol.steps.eager_load(:step_comments).each do |step| + step.step_comments.each do |comment| + comment.save! if update_annotation(comment.message, mappings) + end + step.save! if update_annotation(step.description, mappings) + end + task.results.eager_load(:result_comments, :result_text).each do |res| + res.result_comments.each do |comment| + comment.save! if update_annotation(comment.message, mappings) + end + next unless res.result_text + res.save! if update_annotation(res.result_text.text, mappings) + end + end + end + end + team.protocols.where(my_module: nil).each do |protocol| + protocol.steps.eager_load(:step_comments).each do |step| + step.step_comments.each do |comment| + comment.save! if update_annotation(comment.message, mappings) + end + step.save! if update_annotation(step.description, mappings) + end + end + team.repositories.each do |rep| + rep.repository_rows.includes(repository_cells: :repository_text_value) + .where('repository_cells.value_type': 'RepositoryTextValue') + .each do |row| + row.repository_cells.each do |cell| + if update_annotation(cell.repository_text_value.data, mappings) + cell.repository_text_value.save! + end + end + end + end + end + + # Returns true if text was updated + def self.update_annotation(text, sample_mappings) + return false if text.nil? + updated = false + text.scan(/~sam~\w+\]/).each do |text_match| + orig_id_encoded = text_match.match(/~sam~(\w+)\]/)[1] + orig_id = orig_id_encoded.base62_decode + next unless sample_mappings[orig_id] + new_id_encoded = sample_mappings[orig_id].base62_encode + text.sub!("~sam~#{orig_id_encoded}]", "~rep_item~#{new_id_encoded}]") + updated = true + end + updated + end end end diff --git a/lib/tasks/samples_to_repository_migration.rake b/lib/tasks/samples_to_repository_migration.rake index bf179c76d..cbd663363 100644 --- a/lib/tasks/samples_to_repository_migration.rake +++ b/lib/tasks/samples_to_repository_migration.rake @@ -3,8 +3,11 @@ require_relative '../../app/services/tasks/samples_to_repository_migration_servi namespace :samples_to_repository_migration do desc 'Migrates all data from samples to custom repository' task :run, [:last_id] => :environment do |_, args| + include ActiveRecord::Sanitization::ClassMethods + params = { batch_size: 10 } migration_service = Tasks::SamplesToRepositoryMigrationService + conn = ActiveRecord::Base.connection if args.present? && args[:last_id].present? params[:start] = args[:last_id].to_i end @@ -17,33 +20,34 @@ namespace :samples_to_repository_migration do team_samples = migration_service.fetch_all_team_samples(team) repository = migration_service.prepare_repository(team) custom_columns = migration_service.get_custom_columns(team, repository) - + sample_mappings = {} team_samples.each do |item| created_by = item['sample_created_by_id'] || team.created_by_id last_modified_by = item['sample_last_modified_by_id'] last_modified_by ||= team.created_by_id - row = RepositoryRow.create!( - name: item['sample_name'], - created_at: item['sample_created_at'], - updated_at: item['sample_updated_at'], - created_by_id: created_by, - last_modified_by_id: last_modified_by, - repository: repository - ) + timestamp = conn.quote(Time.now.to_s(:db)) + values = [repository.id, created_by, last_modified_by, + conn.quote(item['sample_name']), timestamp, timestamp] + list_item_sql = <<-SQL + INSERT INTO repository_rows + (repository_id, created_by_id, last_modified_by_id, name, + created_at, updated_at) + VALUES (#{values.join(', ')}) + RETURNING id + SQL + result = conn.execute(list_item_sql) + row_id = result[0]['id'] + sample_mappings[item['sample_id']] = row_id + # check if sample has sample type assigned if item['sample_type_name'] column = custom_columns.detect { |el| el['name'] == 'Sample type' } list_item = column.repository_list_items.where( data: item['sample_type_name'] ).take - RepositoryListValue.create!( - created_by: list_item.created_by, - last_modified_by: list_item.last_modified_by, - repository_list_item: list_item, - repository_cell_attributes: { - repository_row: row, - repository_column: column - } + migration_service.create_list_cell( + row_id, column.id, list_item.id, + list_item.created_by_id, list_item.last_modified_by_id ) end @@ -53,14 +57,9 @@ namespace :samples_to_repository_migration do list_item = column.repository_list_items.where( data: item['sample_group_name'] ).take - RepositoryListValue.create!( - created_by: list_item.created_by, - last_modified_by: list_item.last_modified_by, - repository_list_item: list_item, - repository_cell_attributes: { - repository_row: row, - repository_column: column - } + migration_service.create_list_cell( + row_id, column.id, list_item.id, + list_item.created_by_id, list_item.last_modified_by_id ) # assign sample group color to the sample @@ -68,14 +67,9 @@ namespace :samples_to_repository_migration do column = custom_columns.detect do |el| el['name'] == 'Sample group color hex' end - RepositoryTextValue.create!( - data: item['sample_group_color'], - created_by_id: created_by, - last_modified_by_id: last_modified_by, - repository_cell_attributes: { - repository_row: row, - repository_column: column - } + migration_service.create_text_cell( + row_id, column.id, item['sample_group_color'], + created_by, last_modified_by ) end end @@ -88,15 +82,9 @@ namespace :samples_to_repository_migration do column = custom_columns.detect do |el| el['name'] == field['column_name_reference'] end - RepositoryTextValue.create!( - data: field['value'], - created_by_id: created_by, - last_modified_by_id: last_modified_by, - repository_cell_attributes: { - repository_row: row, - repository_column: column - } - ) + migration_service + .create_text_cell(row_id, column.id, field['value'], + created_by, last_modified_by) end # assign repository item to a tasks @@ -104,13 +92,21 @@ namespace :samples_to_repository_migration do item['sample_id'] ) assigned_modules.each do |element| - MyModuleRepositoryRow.create!( - my_module_id: element['my_module_id'], - repository_row: row, - assigned_by_id: element['assigned_by_id'] || created_by - ) + assigned_by = element['assigned_by_id'] || created_by + values = [row_id, element['my_module_id'], assigned_by, + timestamp, timestamp] + cell_sql = <<-SQL + INSERT INTO my_module_repository_rows + (repository_row_id, my_module_id, assigned_by_id, + created_at, updated_at) + VALUES (#{values.join(', ')}) + SQL + conn.execute(cell_sql) end end + + # Now update smart annotations + migration_service.update_smart_annotations(team, sample_mappings) end end end