From 0184c97c61855f1ba2612d5c901842d1920373fd Mon Sep 17 00:00:00 2001 From: Jure Grabnar Date: Wed, 16 Jan 2019 08:00:01 +0100 Subject: [PATCH] Refactor TeamExporter into ModelExporter --- .../model_exporters/experiment_exporter.rb | 80 ++++++ .../model_exporters/model_exporter.rb | 35 +++ app/services/model_exporters/team_exporter.rb | 163 ++++++++++++ app/services/team_exporter.rb | 244 ------------------ lib/tasks/data.rake | 2 +- 5 files changed, 279 insertions(+), 245 deletions(-) create mode 100644 app/services/model_exporters/experiment_exporter.rb create mode 100644 app/services/model_exporters/model_exporter.rb create mode 100644 app/services/model_exporters/team_exporter.rb delete mode 100644 app/services/team_exporter.rb diff --git a/app/services/model_exporters/experiment_exporter.rb b/app/services/model_exporters/experiment_exporter.rb new file mode 100644 index 000000000..d3aa72059 --- /dev/null +++ b/app/services/model_exporters/experiment_exporter.rb @@ -0,0 +1,80 @@ +require 'fileutils' + +module ModelExporters + class ExperimentExporter + def initialize(experiment) + @experiment = experiment + raise StandardError, 'Can not load experiment' unless @experiment + + @assets_to_copy = [] + end + + def experiment() + return { + experiment: @experiment, + my_modules: @experiment.my_modules.map { |m| my_module(m) }, + my_module_groups: @experiment.my_module_groups + }, @assets_to_copy + end + + def my_module(my_module) + { + my_module: my_module, + outputs: my_module.outputs, + my_module_tags: my_module.my_module_tags, + task_comments: my_module.task_comments, + my_module_repository_rows: my_module.my_module_repository_rows, + user_my_modules: my_module.user_my_modules, + protocols: my_module.protocols.map { |pr| protocol(pr) }, + results: my_module.results.map { |res| result(res) } + } + end + + def protocol(protocol) + { + protocol: protocol, + protocol_protocol_keywords: protocol.protocol_protocol_keywords, + steps: protocol.steps.map { |s| step(s) } + } + end + + def step(step) + @assets_to_copy.push(step.assets.to_a) if step.assets.present? + { + step: step, + checklists: step.checklists.map { |c| checklist(c) }, + step_comments: step.step_comments, + step_assets: step.step_assets, + assets: step.assets, + step_tables: step.step_tables, + tables: step.tables.map { |t| table(t) } + } + end + + def checklist(checklist) + { + checklist: checklist, + checklist_items: checklist.checklist_items + } + end + + def table(table) + return {} if table.nil? + table_json = table.as_json(except: %i(contents data_vector)) + table_json['contents'] = Base64.encode64(table.contents) + table_json['data_vector'] = Base64.encode64(table.data_vector) + table_json + end + + def result(result) + @assets_to_copy.push(result.asset) if result.asset.present? + { + result: result, + result_comments: result.result_comments, + asset: result.asset, + table: table(result.table), + result_text: result.result_text + } + end + end +end diff --git a/app/services/model_exporters/model_exporter.rb b/app/services/model_exporters/model_exporter.rb new file mode 100644 index 000000000..367493eae --- /dev/null +++ b/app/services/model_exporters/model_exporter.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module ModelExporters + class ModelExporter + def copy_files(assets, attachment_name, dir_name) + assets.flatten.each do |a| + next unless a.public_send(attachment_name).present? + + unless a.public_send(attachment_name).exists? + raise StandardError, + "File id:#{a.id} of type #{attachment_name} is missing" + end + yield if block_given? + dir = FileUtils.mkdir_p(File.join(dir_name, a.id.to_s)).first + if defined?(S3_BUCKET) + s3_asset = + S3_BUCKET.object(a.public_send(attachment_name).path.remove(%r{^/})) + file_name = a.public_send(attachment_name).original_filename + File.open(File.join(dir, file_name), 'wb') do |f| + s3_asset.get(response_target: f) + end + else + FileUtils.cp( + a.public_send(attachment_name).path, + File.join(dir, a.public_send(attachment_name).original_filename) + ) + end + end + end + + def export_to_dir + raise NotImplementedError, '#export_to_dir method not implemented.' + end + end +end diff --git a/app/services/model_exporters/team_exporter.rb b/app/services/model_exporters/team_exporter.rb new file mode 100644 index 000000000..f99bb3f33 --- /dev/null +++ b/app/services/model_exporters/team_exporter.rb @@ -0,0 +1,163 @@ +# frozen_string_literal: true + +require 'fileutils' + +module ModelExporters + class TeamExporter < ModelExporter + def initialize(team_id) + @team = Team.includes(:user_teams).find_by_id(team_id) + raise StandardError, 'Can not load team' unless @team + + @assets_to_copy = [] + @tiny_mce_assets_to_copy = [] + end + + def export_to_dir + @asset_counter = 0 + @team.transaction(isolation: :serializable) do + @dir_to_export = FileUtils.mkdir_p( + File.join("tmp/team_#{@team.id}_export_#{Time.now.to_i}") + ).first + + # Writing JSON file with team structure + File.write( + File.join(@dir_to_export, 'team_export.json'), + team(@team).to_json + ) + # Copying assets + copy_files(@assets_to_copy, :file, File.join(@dir_to_export, 'assets')) do + @asset_counter += 1 + end + # Copying tiny_mce_assets + copy_files(@tiny_mce_assets_to_copy, + :image, + File.join(@dir_to_export, 'tiny_mce_assets')) + puts "Exported assets: #{@asset_counter}" + puts "Exported tinyMCE assets: #{@team.tiny_mce_assets.count}" + puts "Exported users: #{@team.users.count}" + puts "Exported repositories: #{@team.repositories.count}" + puts "Exported projects: #{@team.projects.count}" + puts 'Done!' + end + end + + private + + def team(team) + if team.tiny_mce_assets.present? + @tiny_mce_assets_to_copy.push(team.tiny_mce_assets) + end + { + team: team, + default_admin_id: team.user_teams.where(role: 2).first.user.id, + users: team.users.map { |u| user(u) }, + user_teams: team.user_teams, + notifications: Notification + .includes(:user_notifications) + .where('user_notifications.user_id': team.users) + .map { |n| notification(n) }, + custom_fields: team.custom_fields, + repositories: team.repositories.map { |r| repository(r) }, + tiny_mce_assets: team.tiny_mce_assets, + protocols: team.protocols.where(my_module: nil).map { |pr| protocol(pr) }, + protocol_keywords: team.protocol_keywords, + projects: team.projects.map { |p| project(p) } + } + end + + def notification(notification) + notification_json = notification.as_json + notification_json['type_of'] = Extends::NOTIFICATIONS_TYPES + .key(notification.read_attribute('type_of')) + .to_s + notification_json + end + + def user(user) + user_json = user.as_json + # Looks like Devise doesn't export some fields to JSON, so add it manually + user_json['encrypted_password'] = user.encrypted_password + user_json['confirmed_at'] = user.confirmed_at + user_json['sign_in_count'] = user.sign_in_count + user_json['last_sign_in_at'] = user.last_sign_in_at + user_json['last_sign_in_ip'] = user.last_sign_in_ip + copy_files([user], :avatar, File.join(@dir_to_export, 'avatars')) + { + user: user_json, + user_notifications: user.user_notifications, + user_identities: user.user_identities, + repository_table_states: + user.repository_table_states.where(repository: @team.repositories) + } + end + + def project(project) + experiments = project.experiments.map do |e| + experiment, assets = ExperimentExporter.new(e).experiment + @assets_to_copy << assets + experiment + end + + { + project: project, + user_projects: project.user_projects, + activities: project.activities, + project_comments: project.project_comments, + reports: project.reports.map { |r| report(r) }, + experiments: experiments, + tags: project.tags + } + end + + def report(report) + { + report: report, + report_elements: report.report_elements + } + end + + def repository(repository) + { + repository: repository, + repository_columns: repository.repository_columns.map do |c| + repository_column(c) + end, + repository_rows: repository.repository_rows.map do |r| + repository_row(r) + end + } + end + + def repository_row(repository_row) + { + repository_row: repository_row, + my_module_repository_rows: repository_row.my_module_repository_rows, + repository_cells: repository_row.repository_cells.map do |c| + repository_cell(c) + end + } + end + + def repository_cell(cell) + { + repository_cell: cell, + repository_value: cell.value, + repository_value_asset: get_cell_value_asset(cell) + } + end + + def repository_column(column) + { + repository_column: column, + repository_list_items: column.repository_list_items + } + end + + def get_cell_value_asset(cell) + return unless cell.value_type == 'RepositoryAssetValue' + + @assets_to_copy.push(cell.value.asset) + cell.value.asset + end + end +end diff --git a/app/services/team_exporter.rb b/app/services/team_exporter.rb deleted file mode 100644 index b27020b4f..000000000 --- a/app/services/team_exporter.rb +++ /dev/null @@ -1,244 +0,0 @@ -require 'fileutils' - -class TeamExporter - def initialize(team_id) - @team = Team.includes(:user_teams).find_by_id(team_id) - raise StandardError, 'Can not load team' unless @team - @assets_to_copy = [] - @tiny_mce_assets_to_copy = [] - end - - def export_to_dir - @asset_counter = 0 - @team.transaction(isolation: :serializable) do - @dir_to_export = FileUtils.mkdir_p( - File.join("tmp/team_#{@team.id}_export_#{Time.now.to_i}") - ).first - - # Writing JSON file with team structure - File.write( - File.join(@dir_to_export, 'team_export.json'), - team(@team).to_json - ) - # Copying assets - copy_files(@assets_to_copy, :file, File.join(@dir_to_export, 'assets')) do - @asset_counter += 1 - end - # Copying tiny_mce_assets - copy_files(@tiny_mce_assets_to_copy, - :image, - File.join(@dir_to_export, 'tiny_mce_assets')) - puts "Exported assets: #{@asset_counter}" - puts "Exported tinyMCE assets: #{@team.tiny_mce_assets.count}" - puts "Exported users: #{@team.users.count}" - puts "Exported repositories: #{@team.repositories.count}" - puts "Exported projects: #{@team.projects.count}" - puts 'Done!' - end - end - - private - - def copy_files(assets, attachment_name, dir_name) - assets.flatten.each do |a| - next unless a.public_send(attachment_name).present? - unless a.public_send(attachment_name).exists? - raise StandardError, - "File id:#{a.id} of type #{attachment_name} is missing" - end - yield if block_given? - dir = FileUtils.mkdir_p(File.join(dir_name, a.id.to_s)).first - if defined?(S3_BUCKET) - s3_asset = - S3_BUCKET.object(a.public_send(attachment_name).path.remove(%r{^/})) - file_name = a.public_send(attachment_name).original_filename - File.open(File.join(dir, file_name), 'wb') do |f| - s3_asset.get(response_target: f) - end - else - FileUtils.cp( - a.public_send(attachment_name).path, - File.join(dir, a.public_send(attachment_name).original_filename) - ) - end - end - end - - def team(team) - if team.tiny_mce_assets.present? - @tiny_mce_assets_to_copy.push(team.tiny_mce_assets) - end - { - team: team, - default_admin_id: team.user_teams.where(role: 2).first.user.id, - users: team.users.map { |u| user(u) }, - user_teams: team.user_teams, - notifications: Notification - .includes(:user_notifications) - .where('user_notifications.user_id': team.users) - .map { |n| notification(n) }, - custom_fields: team.custom_fields, - repositories: team.repositories.map { |r| repository(r) }, - tiny_mce_assets: team.tiny_mce_assets, - protocols: team.protocols.where(my_module: nil).map { |pr| protocol(pr) }, - protocol_keywords: team.protocol_keywords, - projects: team.projects.map { |p| project(p) } - } - end - - def notification(notification) - notification_json = notification.as_json - notification_json['type_of'] = Extends::NOTIFICATIONS_TYPES - .key(notification.read_attribute('type_of')) - .to_s - notification_json - end - - def user(user) - user_json = user.as_json - # Looks like Devise doesn't export some fields to JSON, so add it manually - user_json['encrypted_password'] = user.encrypted_password - user_json['confirmed_at'] = user.confirmed_at - user_json['sign_in_count'] = user.sign_in_count - user_json['last_sign_in_at'] = user.last_sign_in_at - user_json['last_sign_in_ip'] = user.last_sign_in_ip - copy_files([user], :avatar, File.join(@dir_to_export, 'avatars')) - { - user: user_json, - user_notifications: user.user_notifications, - user_identities: user.user_identities, - repository_table_states: - user.repository_table_states.where(repository: @team.repositories) - } - end - - def project(project) - { - project: project, - user_projects: project.user_projects, - activities: project.activities, - project_comments: project.project_comments, - reports: project.reports.map { |r| report(r) }, - experiments: project.experiments.map { |e| experiment(e) }, - tags: project.tags - } - end - - def report(report) - { - report: report, - report_elements: report.report_elements - } - end - - def experiment(experiment) - { - experiment: experiment, - my_modules: experiment.my_modules.map { |m| my_module(m) }, - my_module_groups: experiment.my_module_groups - } - end - - def my_module(my_module) - { - my_module: my_module, - outputs: my_module.outputs, - my_module_tags: my_module.my_module_tags, - task_comments: my_module.task_comments, - my_module_repository_rows: my_module.my_module_repository_rows, - user_my_modules: my_module.user_my_modules, - protocols: my_module.protocols.map { |pr| protocol(pr) }, - results: my_module.results.map { |res| result(res) } - } - end - - def protocol(protocol) - { - protocol: protocol, - protocol_protocol_keywords: protocol.protocol_protocol_keywords, - steps: protocol.steps.map { |s| step(s) } - } - end - - def step(step) - @assets_to_copy.push(step.assets.to_a) if step.assets.present? - { - step: step, - checklists: step.checklists.map { |c| checklist(c) }, - step_comments: step.step_comments, - step_assets: step.step_assets, - assets: step.assets, - step_tables: step.step_tables, - tables: step.tables.map { |t| table(t) } - } - end - - def checklist(checklist) - { - checklist: checklist, - checklist_items: checklist.checklist_items - } - end - - def table(table) - return {} if table.nil? - table_json = table.as_json(except: %i(contents data_vector)) - table_json['contents'] = Base64.encode64(table.contents) - table_json['data_vector'] = Base64.encode64(table.data_vector) - table_json - end - - def result(result) - @assets_to_copy.push(result.asset) if result.asset.present? - { - result: result, - result_comments: result.result_comments, - asset: result.asset, - table: table(result.table), - result_text: result.result_text - } - end - - def repository(repository) - { - repository: repository, - repository_columns: repository.repository_columns.map do |c| - repository_column(c) - end, - repository_rows: repository.repository_rows.map do |r| - repository_row(r) - end - } - end - - def repository_row(repository_row) - { - repository_row: repository_row, - my_module_repository_rows: repository_row.my_module_repository_rows, - repository_cells: repository_row.repository_cells.map do |c| - repository_cell(c) - end - } - end - - def repository_cell(cell) - { - repository_cell: cell, - repository_value: cell.value, - repository_value_asset: get_cell_value_asset(cell) - } - end - - def repository_column(column) - { - repository_column: column, - repository_list_items: column.repository_list_items - } - end - - def get_cell_value_asset(cell) - return unless cell.value_type == 'RepositoryAssetValue' - @assets_to_copy.push(cell.value.asset) - cell.value.asset - end -end diff --git a/lib/tasks/data.rake b/lib/tasks/data.rake index 6140447dc..dbb553ed1 100644 --- a/lib/tasks/data.rake +++ b/lib/tasks/data.rake @@ -85,7 +85,7 @@ namespace :data do Rails.logger.info( "Exporting team with ID:#{args[:team_id]} to directory in tmp" ) - te = TeamExporter.new(args[:team_id]) + te = ModelExporters::TeamExporter.new(args[:team_id]) te.export_to_dir if te end