scinote-eln/scinote-web
1
1
Fork 0
mirror of https://github.com/scinote-eln/scinote-web.git synced 2024-09-21 07:26:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1030 from okriuchykhin/ok_SCI_2016

Browse source 
Add annotations migrations and optimize queries [SCI-2016]
This commit is contained in:
okriuchykhin 2018-03-13 13:07:31 +01:00 committed by GitHub
parent 9f5513ff3d ac009a29cb
commit dfb0d923b5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 207 additions and 64 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

181
app/services/tasks/samples_to_repository_migration_service.rb
View file

@ -6,6 +6,8 @@
module Tasks
module SamplesToRepositoryMigrationService
include ActiveRecord::Sanitization::ClassMethods
def self.prepare_repository(team, copy_num = 0)
repository = Repository.new(
name: copy_num > 0 ? "Samples (#{copy_num})" : 'Samples',
@ -39,6 +41,7 @@ module Tasks
end
def self.prepare_list_value_custom_columns_with_list_items(team, repository)
conn = ActiveRecord::Base.connection
sample_types_sql = <<-SQL
SELECT name, created_by_id, last_modified_by_id
FROM sample_types
@ -50,8 +53,8 @@ module Tasks
WHERE team_id = #{team.id}
SQL
# execute query
sample_types = ActiveRecord::Base.connection.execute(sample_types_sql)
sample_groups = ActiveRecord::Base.connection.execute(sample_groups_sql)
sample_types = conn.execute(sample_types_sql)
sample_groups = conn.execute(sample_groups_sql)
sample_group = RepositoryColumn.create!(
repository: repository,
@ -78,25 +81,53 @@ module Tasks
sample_groups.each_with_index do |item, index|
created_by = item['created_by_id'] || team.created_by_id
last_modified_by = item['last_modified_by_id'] || team.created_by_id
RepositoryListItem.create!(
data: item.fetch('name') { "sample group item (#{index})" },
created_by_id: created_by,
last_modified_by_id: last_modified_by,
repository_column: sample_group,
repository: repository
)
timestamp = conn.quote(Time.now.to_s(:db))
values = [
repository.id,
sample_group.id,
conn.quote(item.fetch('name') { "sample group item (#{index})" }),
created_by,
last_modified_by,
timestamp,
timestamp
]
list_item_sql = <<-SQL
INSERT INTO repository_list_items
(repository_id,
repository_column_id,
data,
created_by_id,
last_modified_by_id,
created_at, updated_at)
VALUES (#{values.join(', ')})
SQL
conn.execute(list_item_sql)
end
sample_types.each_with_index do |item, index|
created_by = item['created_by_id'] || team.created_by_id
last_modified_by = item['last_modified_by_id'] || team.created_by_id
RepositoryListItem.create!(
data: item.fetch('name') { "sample group item (#{index})" },
created_by_id: created_by,
last_modified_by_id: last_modified_by,
repository_column: sample_type,
repository: repository
)
timestamp = conn.quote(Time.now.to_s(:db))
values = [
repository.id,
sample_type.id,
conn.quote(item.fetch('name') { "sample type item (#{index})" }),
created_by,
last_modified_by,
timestamp,
timestamp
]
list_item_sql = <<-SQL
INSERT INTO repository_list_items
(repository_id,
repository_column_id,
data,
created_by_id,
last_modified_by_id,
created_at, updated_at)
VALUES (#{values.join(', ')})
SQL
conn.execute(list_item_sql)
end
[sample_group, sample_type, sample_group_color]
@ -140,7 +171,7 @@ module Tasks
LEFT OUTER JOIN sample_types
ON samples.sample_type_id = sample_types.id
LEFT OUTER JOIN sample_groups
ON samples.sample_type_id = sample_groups.id
ON samples.sample_group_id = sample_groups.id
WHERE samples.team_id = #{team.id}
SQL
@ -151,5 +182,121 @@ module Tasks
prepare_text_value_custom_columns(team, repository) +
prepare_list_value_custom_columns_with_list_items(team, repository)
end
def self.create_text_cell(row_id, column_id, data,
created_by_id, last_modified_by_id)
conn = ActiveRecord::Base.connection
timestamp = conn.quote(Time.now.to_s(:db))
values = [conn.quote(data), created_by_id, last_modified_by_id, timestamp, timestamp]
value_sql = <<-SQL
INSERT INTO repository_text_values
(data, created_by_id, last_modified_by_id,
created_at, updated_at)
VALUES (#{values.join(', ')})
RETURNING id
SQL
value_id = conn.execute(value_sql)[0]['id']
values = [row_id, column_id, value_id, conn.quote('RepositoryTextValue'),
timestamp, timestamp]
cell_sql = <<-SQL
INSERT INTO repository_cells
(repository_row_id, repository_column_id, value_id, value_type,
created_at, updated_at)
VALUES (#{values.join(', ')})
SQL
conn.execute(cell_sql)
end
def self.create_list_cell(row_id, column_id, list_item_id,
created_by_id, last_modified_by_id)
conn = ActiveRecord::Base.connection
timestamp = conn.quote(Time.now.to_s(:db))
values = [list_item_id, created_by_id, last_modified_by_id,
timestamp, timestamp]
list_value_sql = <<-SQL
INSERT INTO repository_list_values
(repository_list_item_id, created_by_id, last_modified_by_id,
created_at, updated_at)
VALUES (#{values.join(', ')})
RETURNING id
SQL
value_id = ActiveRecord::Base.connection.execute(list_value_sql)[0]['id']
values = [row_id, column_id, value_id, conn.quote('RepositoryListValue'),
timestamp, timestamp]
cell_sql = <<-SQL
INSERT INTO repository_cells
(repository_row_id, repository_column_id, value_id, value_type,
created_at, updated_at)
VALUES (#{values.join(', ')})
SQL
ActiveRecord::Base.connection.execute(cell_sql)
end
def self.update_smart_annotations(team, mappings)
team.projects.eager_load(:project_comments).each do |pr|
pr.project_comments.each do |comment|
comment.save! if update_annotation(comment.message, mappings)
end
pr.experiments.each do |exp|
exp.save! if update_annotation(exp.description, mappings)
exp.my_modules.eager_load(:task_comments).each do |task|
task.task_comments.each do |comment|
comment.save! if update_annotation(comment.message, mappings)
end
task.save! if update_annotation(task.description, mappings)
task.protocol.steps.eager_load(:step_comments).each do |step|
step.step_comments.each do |comment|
comment.save! if update_annotation(comment.message, mappings)
end
step.save! if update_annotation(step.description, mappings)
end
task.results.eager_load(:result_comments, :result_text).each do |res|
res.result_comments.each do |comment|
comment.save! if update_annotation(comment.message, mappings)
end
next unless res.result_text
res.save! if update_annotation(res.result_text.text, mappings)
end
end
end
end
team.protocols.where(my_module: nil).each do |protocol|
protocol.steps.eager_load(:step_comments).each do |step|
step.step_comments.each do |comment|
comment.save! if update_annotation(comment.message, mappings)
end
step.save! if update_annotation(step.description, mappings)
end
end
team.repositories.each do |rep|
rep.repository_rows.includes(repository_cells: :repository_text_value)
.where('repository_cells.value_type': 'RepositoryTextValue')
.each do |row|
row.repository_cells.each do |cell|
if update_annotation(cell.repository_text_value.data, mappings)
cell.repository_text_value.save!
end
end
end
end
end
# Returns true if text was updated
def self.update_annotation(text, sample_mappings)
return false if text.nil?
updated = false
text.scan(/~sam~\w+\]/).each do |text_match|
orig_id_encoded = text_match.match(/~sam~(\w+)\]/)[1]
orig_id = orig_id_encoded.base62_decode
next unless sample_mappings[orig_id]
new_id_encoded = sample_mappings[orig_id].base62_encode
text.sub!("~sam~#{orig_id_encoded}]", "~rep_item~#{new_id_encoded}]")
updated = true
end
updated
end
end
end

90
lib/tasks/samples_to_repository_migration.rake
View file

@ -3,8 +3,11 @@ require_relative '../../app/services/tasks/samples_to_repository_migration_servi
namespace :samples_to_repository_migration do
desc 'Migrates all data from samples to custom repository'
task :run, [:last_id] => :environment do |_, args|
include ActiveRecord::Sanitization::ClassMethods
params = { batch_size: 10 }
migration_service = Tasks::SamplesToRepositoryMigrationService
conn = ActiveRecord::Base.connection
if args.present? && args[:last_id].present?
params[:start] = args[:last_id].to_i
end
@ -17,33 +20,34 @@ namespace :samples_to_repository_migration do
team_samples = migration_service.fetch_all_team_samples(team)
repository = migration_service.prepare_repository(team)
custom_columns = migration_service.get_custom_columns(team, repository)
sample_mappings = {}
team_samples.each do |item|
created_by = item['sample_created_by_id'] || team.created_by_id
last_modified_by = item['sample_last_modified_by_id']
last_modified_by ||= team.created_by_id
row = RepositoryRow.create!(
name: item['sample_name'],
created_at: item['sample_created_at'],
updated_at: item['sample_updated_at'],
created_by_id: created_by,
last_modified_by_id: last_modified_by,
repository: repository
)
timestamp = conn.quote(Time.now.to_s(:db))
values = [repository.id, created_by, last_modified_by,
conn.quote(item['sample_name']), timestamp, timestamp]
list_item_sql = <<-SQL
INSERT INTO repository_rows
(repository_id, created_by_id, last_modified_by_id, name,
created_at, updated_at)
VALUES (#{values.join(', ')})
RETURNING id
SQL
result = conn.execute(list_item_sql)
row_id = result[0]['id']
sample_mappings[item['sample_id']] = row_id
# check if sample has sample type assigned
if item['sample_type_name']
column = custom_columns.detect { |el| el['name'] == 'Sample type' }
list_item = column.repository_list_items.where(
data: item['sample_type_name']
).take
RepositoryListValue.create!(
created_by: list_item.created_by,
last_modified_by: list_item.last_modified_by,
repository_list_item: list_item,
repository_cell_attributes: {
repository_row: row,
repository_column: column
}
migration_service.create_list_cell(
row_id, column.id, list_item.id,
list_item.created_by_id, list_item.last_modified_by_id
)
end
@ -53,14 +57,9 @@ namespace :samples_to_repository_migration do
list_item = column.repository_list_items.where(
data: item['sample_group_name']
).take
RepositoryListValue.create!(
created_by: list_item.created_by,
last_modified_by: list_item.last_modified_by,
repository_list_item: list_item,
repository_cell_attributes: {
repository_row: row,
repository_column: column
}
migration_service.create_list_cell(
row_id, column.id, list_item.id,
list_item.created_by_id, list_item.last_modified_by_id
)
# assign sample group color to the sample
@ -68,14 +67,9 @@ namespace :samples_to_repository_migration do
column = custom_columns.detect do |el|
el['name'] == 'Sample group color hex'
end
RepositoryTextValue.create!(
data: item['sample_group_color'],
created_by_id: created_by,
last_modified_by_id: last_modified_by,
repository_cell_attributes: {
repository_row: row,
repository_column: column
}
migration_service.create_text_cell(
row_id, column.id, item['sample_group_color'],
created_by, last_modified_by
)
end
end
@ -88,15 +82,9 @@ namespace :samples_to_repository_migration do
column = custom_columns.detect do |el|
el['name'] == field['column_name_reference']
end
RepositoryTextValue.create!(
data: field['value'],
created_by_id: created_by,
last_modified_by_id: last_modified_by,
repository_cell_attributes: {
repository_row: row,
repository_column: column
}
)
migration_service
.create_text_cell(row_id, column.id, field['value'],
created_by, last_modified_by)
end
# assign repository item to a tasks
@ -104,13 +92,21 @@ namespace :samples_to_repository_migration do
item['sample_id']
)
assigned_modules.each do |element|
MyModuleRepositoryRow.create!(
my_module_id: element['my_module_id'],
repository_row: row,
assigned_by_id: element['assigned_by_id'] || created_by
)
assigned_by = element['assigned_by_id'] || created_by
values = [row_id, element['my_module_id'], assigned_by,
timestamp, timestamp]
cell_sql = <<-SQL
INSERT INTO my_module_repository_rows
(repository_row_id, my_module_id, assigned_by_id,
created_at, updated_at)
VALUES (#{values.join(', ')})
SQL
conn.execute(cell_sql)
end
end
# Now update smart annotations
migration_service.update_smart_annotations(team, sample_mappings)
end
end
end