From f2c0b3aa925fb5a6e7fe52c18bff6db027f6953c Mon Sep 17 00:00:00 2001 From: zmagod Date: Wed, 13 Jun 2018 12:45:48 +0200 Subject: [PATCH 1/6] removes samples, adds list_items --- app/services/team_exporter.rb | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/app/services/team_exporter.rb b/app/services/team_exporter.rb index e1c8f28d8..43c68ab28 100644 --- a/app/services/team_exporter.rb +++ b/app/services/team_exporter.rb @@ -77,9 +77,6 @@ class TeamExporter .includes(:user_notifications) .where('user_notifications.user_id': team.users) .map { |n| notification(n) }, - samples: team.samples.map { |s| sample(s) }, - sample_groups: team.sample_groups, - sample_types: team.sample_types, custom_fields: team.custom_fields, repositories: team.repositories.map { |r| repository(r) }, tiny_mce_assets: team.tiny_mce_assets, @@ -110,7 +107,6 @@ class TeamExporter user: user_json, user_notifications: user.user_notifications, user_identities: user.user_identities, - samples_tables: user.samples_tables.where(team: @team), repository_table_states: user.repository_table_states.where(repository: @team.repositories) } @@ -150,7 +146,6 @@ class TeamExporter my_module_tags: my_module.my_module_tags, task_comments: my_module.task_comments, my_module_repository_rows: my_module.my_module_repository_rows, - sample_my_modules: my_module.sample_my_modules, user_my_modules: my_module.user_my_modules, protocols: my_module.protocols.map { |pr| protocol(pr) }, results: my_module.results.map { |res| result(res) } @@ -204,17 +199,13 @@ class TeamExporter } end - def sample(sample) - { - sample: sample, - sample_custom_fields: sample.sample_custom_fields - } - end - def repository(repository) + columns = repository.repository_columns { repository: repository, - repository_columns: repository.repository_columns, + repository_columns: repository.repository_columns.map do |c| + repository_columns(c) + end, repository_rows: repository.repository_rows.map do |r| repository_row(r) end @@ -237,4 +228,11 @@ class TeamExporter repository_value: cell.value } end + + def repository_columns(column) + { + repository_column: column, + repository_list_items: column.repository_list_items + } + end end From 4a82d1e7ac364547a2e3fcaefb7353150dd43dd2 Mon Sep 17 00:00:00 2001 From: zmagod Date: Thu, 14 Jun 2018 14:40:43 +0200 Subject: [PATCH 2/6] fixes repository items import --- app/services/team_importer.rb | 142 +++++++++++----------------------- 1 file changed, 44 insertions(+), 98 deletions(-) diff --git a/app/services/team_importer.rb b/app/services/team_importer.rb index a70705d6b..22eed8fca 100644 --- a/app/services/team_importer.rb +++ b/app/services/team_importer.rb @@ -3,10 +3,7 @@ class TeamImporter @user_mappings = {} @notification_mappings = {} @repository_mappings = {} - @sample_group_mappings = {} - @sample_type_mappings = {} @custom_field_mappings = {} - @sample_mappings = {} @project_mappings = {} @repository_column_mappings = {} @experiment_mappings = {} @@ -19,6 +16,7 @@ class TeamImporter @tag_mappings = {} @result_text_mappings = {} @repository_row_mappings = {} + @repository_list_item_mappings = {} @result_mappings = {} @checklist_mappings = {} @table_mappings = {} @@ -28,7 +26,6 @@ class TeamImporter @repository_row_counter = 0 @protocol_counter = 0 @step_counter = 0 - @sample_counter = 0 @report_counter = 0 @my_module_counter = 0 @notification_counter = 0 @@ -70,10 +67,7 @@ class TeamImporter user_team.save! end - create_sample_groups(team_json['sample_groups'], team) - create_sample_types(team_json['sample_types'], team) create_custom_fields(team_json['custom_fields'], team) - create_samples(team_json['samples'], team) create_protocol_keywords(team_json['protocol_keywords'], team) create_protocols(team_json['protocols'], nil, team) create_projects(team_json['projects'], team) @@ -132,7 +126,6 @@ class TeamImporter puts "Imported reports: #{@report_counter}" puts "Imported repositories: #{@repository_counter}" puts "Imported repository rows: #{@repository_row_counter}" - puts "Imported samples: #{@sample_counter}" puts "Imported tasks: #{@my_module_counter}" puts "Imported protocols: #{@protocol_counter}" puts "Imported steps: #{@step_counter}" @@ -184,11 +177,6 @@ class TeamImporter step.save! if update_annotation(step.description) end end - team.samples.each do |sample| - sample.sample_custom_fields.find_each do |cf| - cf.save! if update_annotation(cf.value) - end - end team.repositories.each do |rep| rep.repository_rows.find_each do |row| row.repository_cells.each do |cell| @@ -202,7 +190,7 @@ class TeamImporter def update_annotation(text) return false if text.nil? updated = false - %w(prj exp tsk sam).each do |name| + %w(prj exp tsk).each do |name| text.scan(/~#{name}~\w+\]/).each do |text_match| orig_id_encoded = text_match.match(/~#{name}~(\w+)\]/)[1] orig_id = orig_id_encoded.base62_decode @@ -214,8 +202,6 @@ class TeamImporter @experiment_mappings[orig_id] when 'tsk' @my_module_mappings[orig_id] - when 'sam' - @sample_mappings[orig_id] end next unless new_id new_id_encoded = new_id.base62_encode @@ -317,13 +303,6 @@ class TeamImporter user_identity.user_id = user.id user_identity.save! end - user_json['samples_tables'].each do |samples_table_json| - samples_table = SamplesTable.new(samples_table_json) - samples_table.id = nil - samples_table.user_id = user.id - samples_table.team_id = team.id - samples_table.save! - end end end @@ -353,7 +332,10 @@ class TeamImporter @repository_mappings[orig_repository_id] = repository.id @repository_counter += 1 repository_json['repository_columns'].each do |repository_column_json| - repository_column = RepositoryColumn.new(repository_column_json) + + repository_column = RepositoryColumn.new( + repository_column_json['repository_column'] + ) orig_rep_col_id = repository_column.id repository_column.id = nil repository_column.repository = repository @@ -361,6 +343,18 @@ class TeamImporter find_user(repository_column.created_by_id) repository_column.save! @repository_column_mappings[orig_rep_col_id] = repository_column.id + next unless repository_column.data_type == 'RepositoryListValue' + repository_column_json['repository_list_items'].each do |list_item| + created_by_id = find_user(repository_column.created_by_id) + repository_list_item = RepositoryListItem.new(data: list_item['data']) + repository_list_item.repository_column = repository_column + repository_list_item.repository = repository + repository_list_item.created_by_id = created_by_id + repository_list_item.last_modified_by_id = created_by_id + repository_list_item.save! + @repository_list_item_mappings[list_item['id']] = + repository_list_item.id + end end create_repository_rows(repository_json['repository_rows'], repository) end @@ -399,44 +393,8 @@ class TeamImporter @repository_column_mappings[repository_cell.repository_column_id] repository_cell.repository_row_id = @repository_row_mappings[repository_cell.repository_row_id] - repository_value = - RepositoryTextValue.new(repository_cell_json['repository_value']) - repository_value.id = nil - repository_value.created_by_id = find_user(repository_value.created_by_id) - repository_value.last_modified_by_id = - find_user(repository_value.last_modified_by_id) - repository_value.repository_cell = repository_cell - repository_value.save! - end - end - - def create_sample_groups(sample_groups_json, team) - puts 'Creating sample groups...' - sample_groups_json.each do |sample_group_json| - sample_group = SampleGroup.new(sample_group_json) - orig_group_id = sample_group.id - sample_group.id = nil - sample_group.team = team - sample_group.created_by_id = find_user(sample_group.created_by_id) - sample_group.last_modified_by_id = - find_user(sample_group.last_modified_by_id) - sample_group.save! - @sample_group_mappings[orig_group_id] = sample_group.id - end - end - - def create_sample_types(sample_types_json, team) - puts 'Creating sample types...' - sample_types_json.each do |sample_type_json| - sample_type = SampleType.new(sample_type_json) - orig_type_id = sample_type.id - sample_type.id = nil - sample_type.team = team - sample_type.created_by_id = find_user(sample_type.created_by_id) - sample_type.last_modified_by_id = - find_user(sample_type.last_modified_by_id) - sample_type.save! - @sample_type_mappings[orig_type_id] = sample_type.id + create_cell_value(repository_cell, + repository_cell_json['repository_value']) end end @@ -455,31 +413,6 @@ class TeamImporter end end - def create_samples(samples_json, team) - puts 'Creating samples...' - samples_json.each do |sample_json| - sample = Sample.new(sample_json['sample']) - orig_sample_id = sample.id - sample.id = nil - sample.team = team - sample.user_id = find_user(sample.user_id) - sample.sample_group_id = @sample_group_mappings[sample.sample_group_id] - sample.sample_type_id = @sample_type_mappings[sample.sample_type_id] - sample.last_modified_by_id = find_user(sample.last_modified_by_id) - sample.save! - @sample_mappings[orig_sample_id] = sample.id - @sample_counter += 1 - sample_json['sample_custom_fields'].each do |s_custom_field_json| - s_custom_field = SampleCustomField.new(s_custom_field_json) - s_custom_field.id = nil - s_custom_field.sample = sample - s_custom_field.custom_field_id = - @custom_field_mappings[s_custom_field.custom_field_id] - s_custom_field.save! - end - end - end - def create_protocol_keywords(protocol_keywords_json, team) puts 'Creating keywords...' protocol_keywords_json.each do |protocol_keyword_json| @@ -607,17 +540,6 @@ class TeamImporter task_comment.save! end - my_module_json['sample_my_modules'].each do |sample_module_json| - sample_module = SampleMyModule.new(sample_module_json) - sample_module.id = nil - sample_module.sample_id = - @sample_mappings[sample_module.sample_id] - sample_module.my_module = my_module - sample_module.assigned_by_id = - find_user(sample_module.assigned_by_id) - sample_module.save! - end - my_module_json['user_my_modules'].each do |user_module_json| user_module = UserMyModule.new(user_module_json) user_module.id = nil @@ -875,4 +797,28 @@ class TeamImporter return nil if user_id.nil? @user_mappings[user_id] ? @user_mappings[user_id] : @admin_id end + + def find_list_item_id(list_item_id) + @repository_list_item_mappings[list_item_id] + end + + def create_cell_value(repository_cell, value_json) + case repository_cell.value_type + when 'RepositoryListValue' + list_item_id = find_list_item_id(value_json['repository_list_item_id']) + repository_value = RepositoryListValue.new( + repository_list_item_id: list_item_id.to_i + ) + when 'RepositoryTextValue' + repository_value = RepositoryTextValue.new(value_json) + when 'RepositoryAssetValue' + repository_value = RepositoryAssetValue.new() + end + repository_value.id = nil + repository_value.created_by_id = find_user(value_json['created_by_id']) + repository_value.last_modified_by_id = + find_user(value_json['last_modified_by_id']) + repository_value.repository_cell = repository_cell + repository_value.save! + end end From 3d7591100a24bf1b7cf4fea002a122e6ba556b97 Mon Sep 17 00:00:00 2001 From: zmagod Date: Thu, 14 Jun 2018 15:20:57 +0200 Subject: [PATCH 3/6] adds repository row assets to team import/export [fixes SCI-2165] --- app/services/team_exporter.rb | 9 ++++++++- app/services/team_importer.rb | 21 ++++++++++++--------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/app/services/team_exporter.rb b/app/services/team_exporter.rb index 43c68ab28..e6bf33807 100644 --- a/app/services/team_exporter.rb +++ b/app/services/team_exporter.rb @@ -225,7 +225,8 @@ class TeamExporter def repository_cell(cell) { repository_cell: cell, - repository_value: cell.value + repository_value: cell.value, + repository_value_asset: get_cell_value_asset(cell) } end @@ -235,4 +236,10 @@ class TeamExporter repository_list_items: column.repository_list_items } end + + def get_cell_value_asset(cell) + return unless cell.value_type == 'RepositoryAssetValue' + @assets_to_copy.push(cell.value.asset) + cell.value.asset + end end diff --git a/app/services/team_importer.rb b/app/services/team_importer.rb index 22eed8fca..494ea674f 100644 --- a/app/services/team_importer.rb +++ b/app/services/team_importer.rb @@ -108,7 +108,7 @@ class TeamImporter team_json['repositories'].each do |repository_json| repository_json['repository_rows'].each do |repository_row_json| - create_repository_cells(repository_row_json['repository_cells']) + create_repository_cells(repository_row_json['repository_cells'], team) end end @@ -384,7 +384,7 @@ class TeamImporter end end - def create_repository_cells(repository_cells_json) + def create_repository_cells(repository_cells_json, team) repository_cells_json.each do |repository_cell_json| repository_cell = RepositoryCell.new(repository_cell_json['repository_cell']) @@ -394,7 +394,8 @@ class TeamImporter repository_cell.repository_row_id = @repository_row_mappings[repository_cell.repository_row_id] create_cell_value(repository_cell, - repository_cell_json['repository_value']) + repository_cell_json, + team) end end @@ -802,22 +803,24 @@ class TeamImporter @repository_list_item_mappings[list_item_id] end - def create_cell_value(repository_cell, value_json) + def create_cell_value(repository_cell, value_json, team) + cell_json = value_json['repository_value'] case repository_cell.value_type when 'RepositoryListValue' - list_item_id = find_list_item_id(value_json['repository_list_item_id']) + list_item_id = find_list_item_id(cell_json['repository_list_item_id']) repository_value = RepositoryListValue.new( repository_list_item_id: list_item_id.to_i ) when 'RepositoryTextValue' - repository_value = RepositoryTextValue.new(value_json) + repository_value = RepositoryTextValue.new(cell_json) when 'RepositoryAssetValue' - repository_value = RepositoryAssetValue.new() + asset = create_asset(value_json['repository_value_asset'], team) + repository_value = RepositoryAssetValue.new(asset: asset) end repository_value.id = nil - repository_value.created_by_id = find_user(value_json['created_by_id']) + repository_value.created_by_id = find_user(cell_json['created_by_id']) repository_value.last_modified_by_id = - find_user(value_json['last_modified_by_id']) + find_user(cell_json['last_modified_by_id']) repository_value.repository_cell = repository_cell repository_value.save! end From fbcc7885977b89b306f3769021afba438f34848e Mon Sep 17 00:00:00 2001 From: zmagod Date: Mon, 18 Jun 2018 14:00:49 +0200 Subject: [PATCH 4/6] fixes per @duczOr request --- app/services/team_exporter.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/app/services/team_exporter.rb b/app/services/team_exporter.rb index e6bf33807..b27020b4f 100644 --- a/app/services/team_exporter.rb +++ b/app/services/team_exporter.rb @@ -200,11 +200,10 @@ class TeamExporter end def repository(repository) - columns = repository.repository_columns { repository: repository, repository_columns: repository.repository_columns.map do |c| - repository_columns(c) + repository_column(c) end, repository_rows: repository.repository_rows.map do |r| repository_row(r) @@ -230,7 +229,7 @@ class TeamExporter } end - def repository_columns(column) + def repository_column(column) { repository_column: column, repository_list_items: column.repository_list_items From d68ae63c402ebc2f8359164c42be735701d3fa27 Mon Sep 17 00:00:00 2001 From: zmagod Date: Tue, 19 Jun 2018 11:58:22 +0200 Subject: [PATCH 5/6] adds repository_items to smart annotation import --- app/services/team_exporter.rb | 2 +- app/services/team_importer.rb | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/app/services/team_exporter.rb b/app/services/team_exporter.rb index b27020b4f..54ee46985 100644 --- a/app/services/team_exporter.rb +++ b/app/services/team_exporter.rb @@ -48,7 +48,7 @@ class TeamExporter end yield if block_given? dir = FileUtils.mkdir_p(File.join(dir_name, a.id.to_s)).first - if defined?(S3_BUCKET) + if ENV['S3_BUCKET'] s3_asset = S3_BUCKET.object(a.public_send(attachment_name).path.remove(%r{^/})) file_name = a.public_send(attachment_name).original_filename diff --git a/app/services/team_importer.rb b/app/services/team_importer.rb index 494ea674f..62e94c173 100644 --- a/app/services/team_importer.rb +++ b/app/services/team_importer.rb @@ -202,6 +202,8 @@ class TeamImporter @experiment_mappings[orig_id] when 'tsk' @my_module_mappings[orig_id] + when 'rep_item' + @repository_row_mappings[orig_id] end next unless new_id new_id_encoded = new_id.base62_encode From 29c379dbae1ebdd9f036482854dbe1b1b3b77f25 Mon Sep 17 00:00:00 2001 From: Zmago Devetak Date: Wed, 27 Jun 2018 15:17:02 +0200 Subject: [PATCH 6/6] add rep_item to update_annotation function --- app/services/team_importer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/team_importer.rb b/app/services/team_importer.rb index 62e94c173..0970933b6 100644 --- a/app/services/team_importer.rb +++ b/app/services/team_importer.rb @@ -190,7 +190,7 @@ class TeamImporter def update_annotation(text) return false if text.nil? updated = false - %w(prj exp tsk).each do |name| + %w(prj exp tsk rep_item).each do |name| text.scan(/~#{name}~\w+\]/).each do |text_match| orig_id_encoded = text_match.match(/~#{name}~(\w+)\]/)[1] orig_id = orig_id_encoded.base62_decode