From e6ad5047e021958ba49f42478940877cd99059ec Mon Sep 17 00:00:00 2001 From: Oleksii Kriuchykhin Date: Tue, 28 Apr 2020 12:03:23 +0200 Subject: [PATCH 1/2] Implement background processing of repository snapshots [SCI-4552] --- .../javascripts/my_modules/repositories.js | 85 +++++++++++++------ ..._module_repository_snapshots_controller.rb | 38 ++++++--- app/jobs/application_job.rb | 9 ++ .../repository_snapshot_provisioning_job.rb | 11 +++ app/models/repository_snapshot.rb | 3 + .../my_module_assigning_snapshot_service.rb | 67 --------------- .../snapshot_provisioning_service.rb | 62 ++++++++++++++ .../repositories/_full_view_version.html.erb | 34 ++++++++ .../_full_view_versions_sidebar.html.erb | 24 +----- config/initializers/delayed_job_config.rb | 4 + config/locales/en.yml | 1 + config/routes.rb | 3 +- ...20200331183640_add_repository_snapshots.rb | 1 + db/structure.sql | 1 + 14 files changed, 215 insertions(+), 128 deletions(-) create mode 100644 app/jobs/application_job.rb create mode 100644 app/jobs/repository_snapshot_provisioning_job.rb delete mode 100644 app/services/repositories/my_module_assigning_snapshot_service.rb create mode 100644 app/services/repositories/snapshot_provisioning_service.rb create mode 100644 app/views/my_modules/repositories/_full_view_version.html.erb diff --git a/app/assets/javascripts/my_modules/repositories.js b/app/assets/javascripts/my_modules/repositories.js index be3de75f9..7f3c03366 100644 --- a/app/assets/javascripts/my_modules/repositories.js +++ b/app/assets/javascripts/my_modules/repositories.js @@ -3,6 +3,7 @@ var MyModuleRepositories = (function() { const FULL_VIEW_MODAL = $('#myModuleRepositoryFullViewModal'); + const STATUS_POLLING_INTERVAL = 30000; var SIMPLE_TABLE; var FULL_VIEW_TABLE; var FULL_VIEW_TABLE_SCROLLBAR; @@ -165,27 +166,10 @@ var MyModuleRepositories = (function() { versionsSidebar.find(`[data-id="${currentId}"]`).addClass('active'); } - function createDestroySnapshot(actionPath, requestType) { - animateSpinner(null, true); - $.ajax({ - url: actionPath, - type: requestType, - dataType: 'json', - success: function(data) { - FULL_VIEW_MODAL.find('.repository-versions-sidebar').html(data.html); - setSelectedItem(); - animateSpinner(null, false); - }, - error: function() { - // TODO - } - }); - } - function reloadTable(tableUrl) { animateSpinner(null, true); if (FULL_VIEW_TABLE) FULL_VIEW_TABLE.destroy(); - $.get(tableUrl, (data) => { + $.getJSON(tableUrl, (data) => { FULL_VIEW_MODAL.find('.table-container').html(data.html); renderFullViewTable(FULL_VIEW_MODAL.find('.table')); setSelectedItem(); @@ -193,6 +177,20 @@ var MyModuleRepositories = (function() { }); } + function checkSnapshotStatus(snapshotItem) { + $.getJSON(snapshotItem.data('status-url'), (statusData) => { + if (statusData.status === 'ready') { + $.getJSON(snapshotItem.data('item-url'), (itemData) => { + snapshotItem.replaceWith(itemData.html); + }); + } else { + setTimeout(function() { + checkSnapshotStatus(snapshotItem); + }, STATUS_POLLING_INTERVAL); + } + }); + } + function initSimpleTable() { $('#assigned-items-container').on('show.bs.collapse', '.assigned-repository-container', function() { var repositoryContainer = $(this); @@ -203,28 +201,61 @@ var MyModuleRepositories = (function() { }); } + function initVersionsStatusCheck() { + let sidebar = FULL_VIEW_MODAL.find('.repository-versions-sidebar'); + sidebar.find('.repository-snapshot-item.provisioning').each(function() { + var snapshotItem = $(this); + setTimeout(function() { + checkSnapshotStatus(snapshotItem); + }, STATUS_POLLING_INTERVAL); + }); + } + function initVersionsSidebarActions() { FULL_VIEW_MODAL.on('click', '#showVersionsSidebar', function(e) { - $.get(FULL_VIEW_MODAL.find('.table').data('versions-sidebar-url'), (data) => { + $.getJSON(FULL_VIEW_MODAL.find('.table').data('versions-sidebar-url'), (data) => { FULL_VIEW_MODAL.find('.repository-versions-sidebar').html(data.html); setSelectedItem(); FULL_VIEW_MODAL.find('.table-container').addClass('collapsed'); FULL_VIEW_MODAL.find('.repository-versions-sidebar').removeClass('collapsed'); + initVersionsStatusCheck(); }); e.stopPropagation(); }); FULL_VIEW_MODAL.on('click', '#createRepositorySnapshotButton', function(e) { - createDestroySnapshot($(this).data('action-path'), 'POST'); + animateSpinner(null, true); + $.ajax({ + url: $(this).data('action-path'), + type: 'POST', + dataType: 'json', + success: function(data) { + let snapshotItem = $(data.html); + FULL_VIEW_MODAL.find('.repository-versions-list').append(snapshotItem); + setTimeout(function() { + checkSnapshotStatus(snapshotItem); + }, STATUS_POLLING_INTERVAL); + animateSpinner(null, false); + } + }); e.stopPropagation(); }); FULL_VIEW_MODAL.on('click', '.delete-snapshot-button', function(e) { - let snapshotId = $(this).closest('.repository-snapshot-item').data('id'); - createDestroySnapshot($(this).data('action-path'), 'DELETE'); - if (snapshotId === FULL_VIEW_MODAL.find('.table').data('id')) { - reloadTable(FULL_VIEW_MODAL.find('#selectLiveVersionButton').data('table-url')); - } + let snapshotItem = $(this).closest('.repository-snapshot-item'); + animateSpinner(null, true); + $.ajax({ + url: $(this).data('action-path'), + type: 'DELETE', + dataType: 'json', + success: function() { + if (snapshotItem.data('id') === FULL_VIEW_MODAL.find('.table').data('id')) { + reloadTable(FULL_VIEW_MODAL.find('#selectLiveVersionButton').data('table-url')); + } + snapshotItem.remove(); + animateSpinner(null, false); + } + }); e.stopPropagation(); }); @@ -253,7 +284,7 @@ var MyModuleRepositories = (function() { FULL_VIEW_MODAL.find('.repository-name').html(repositoryNameObject); FULL_VIEW_MODAL.modal('show'); - $.get($(this).data('table-url'), (data) => { + $.getJSON($(this).data('table-url'), (data) => { FULL_VIEW_MODAL.find('.table-container').html(data.html); renderFullViewTable(FULL_VIEW_MODAL.find('.table')); }); @@ -264,7 +295,7 @@ var MyModuleRepositories = (function() { function initRepositoriesDropdown() { $('.repositories-assign-container').on('show.bs.dropdown', function() { var dropdownContainer = $(this); - $.get(dropdownContainer.data('repositories-url'), function(result) { + $.getJSON(dropdownContainer.data('repositories-url'), function(result) { dropdownContainer.find('.repositories-dropdown-menu').html(result.html); }); }); diff --git a/app/controllers/my_module_repository_snapshots_controller.rb b/app/controllers/my_module_repository_snapshots_controller.rb index 82486a372..9733431b8 100644 --- a/app/controllers/my_module_repository_snapshots_controller.rb +++ b/app/controllers/my_module_repository_snapshots_controller.rb @@ -30,22 +30,38 @@ class MyModuleRepositorySnapshotsController < ApplicationController end def create - service = Repositories::MyModuleAssigningSnapshotService.call(repository: @repository, - my_module: @my_module, - user: current_user) + repository_snapshot = @repository.dup.becomes(RepositorySnapshot) + repository_snapshot.assign_attributes(type: RepositorySnapshot.name, + original_repository: @repository, + my_module: @my_module, + created_by: current_user) + repository_snapshot.provisioning! + repository_snapshot.reload - if service.succeed? - @repository_snapshots = @my_module.repository_snapshots.where(original_repository: @repository) - render json: { html: render_to_string(partial: 'my_modules/repositories/full_view_versions_sidebar') } - else - render json: service.errors, status: :unprocessable_entity - end + RepositorySnapshotProvisioningJob.perform_later(repository_snapshot) + + render json: { + html: render_to_string(partial: 'my_modules/repositories/full_view_version', + locals: { repository_snapshot: repository_snapshot }) + } + end + + def status + render json: { + status: @repository_snapshot.status + } + end + + def show + render json: { + html: render_to_string(partial: 'my_modules/repositories/full_view_version', + locals: { repository_snapshot: @repository_snapshot }) + } end def destroy @repository_snapshot.destroy! - @repository_snapshots = @my_module.repository_snapshots.where(original_repository: @repository) - render json: { html: render_to_string(partial: 'my_modules/repositories/full_view_versions_sidebar') } + render json: {} end def full_view_table diff --git a/app/jobs/application_job.rb b/app/jobs/application_job.rb new file mode 100644 index 000000000..43fd60b59 --- /dev/null +++ b/app/jobs/application_job.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class ApplicationJob < ActiveJob::Base + # Automatically retry jobs that encountered a deadlock + retry_on ActiveRecord::Deadlocked + + # Most jobs are safe to ignore if the underlying records are no longer available + discard_on ActiveJob::DeserializationError +end diff --git a/app/jobs/repository_snapshot_provisioning_job.rb b/app/jobs/repository_snapshot_provisioning_job.rb new file mode 100644 index 000000000..724053931 --- /dev/null +++ b/app/jobs/repository_snapshot_provisioning_job.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class RepositorySnapshotProvisioningJob < ApplicationJob + queue_as :high_priority + + def perform(repository_snapshot) + service = Repositories::SnapshotProvisioningService.call(repository_snapshot: repository_snapshot) + + repository_snapshot.failed! unless service.succeed? + end +end diff --git a/app/models/repository_snapshot.rb b/app/models/repository_snapshot.rb index 2ba1da66e..e5951bda4 100644 --- a/app/models/repository_snapshot.rb +++ b/app/models/repository_snapshot.rb @@ -1,10 +1,13 @@ # frozen_string_literal: true class RepositorySnapshot < RepositoryBase + enum status: { provisioning: 0, ready: 1, failed: 2 } + belongs_to :original_repository, foreign_key: :parent_id, class_name: 'Repository', inverse_of: :repository_snapshots belongs_to :my_module, optional: true validates :name, presence: true, length: { maximum: Constants::NAME_MAX_LENGTH } + validates :status, presence: true def default_columns_count Constants::REPOSITORY_SNAPSHOT_TABLE_DEFAULT_STATE['length'] diff --git a/app/services/repositories/my_module_assigning_snapshot_service.rb b/app/services/repositories/my_module_assigning_snapshot_service.rb deleted file mode 100644 index 9dad90a91..000000000 --- a/app/services/repositories/my_module_assigning_snapshot_service.rb +++ /dev/null @@ -1,67 +0,0 @@ -# frozen_string_literal: true - -module Repositories - class MyModuleAssigningSnapshotService - extend Service - - attr_reader :repository, :my_module, :user, :errors - - def initialize(repository:, my_module:, user:) - @repository = repository - @my_module = my_module - @user = user - @errors = {} - end - - def call - return self unless valid? - - ActiveRecord::Base.transaction do - repository_snapshot = @repository.dup.becomes(RepositorySnapshot) - repository_snapshot.type = RepositorySnapshot.name - repository_snapshot.original_repository = @repository - repository_snapshot.my_module = @my_module - repository_snapshot.save! - - @repository.repository_columns.each do |column| - column.snapshot!(repository_snapshot) - end - - repository_rows = @repository.repository_rows - .joins(:my_module_repository_rows) - .where(my_module_repository_rows: { my_module: @my_module }) - - repository_rows.find_each do |original_row| - original_row.snapshot!(repository_snapshot) - end - rescue ActiveRecord::RecordInvalid => e - @errors[e.record.class.name.underscore] = e.record.errors.full_messages - raise ActiveRecord::Rollback - end - - self - end - - def succeed? - @errors.none? - end - - private - - def valid? - unless @repository && @my_module && @user - @errors[:invalid_arguments] = - { 'repository': @repository, - 'my_module': @my_module, - 'user': @user } - .map do |key, value| - if value.nil? - I18n.t('repositories.my_module_assigned_snapshot_service.invalid_arguments', key: key.capitalize) - end - end.compact - return false - end - true - end - end -end diff --git a/app/services/repositories/snapshot_provisioning_service.rb b/app/services/repositories/snapshot_provisioning_service.rb new file mode 100644 index 000000000..5d37672db --- /dev/null +++ b/app/services/repositories/snapshot_provisioning_service.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +module Repositories + class SnapshotProvisioningService + extend Service + + attr_reader :repository_snapshot, :errors + + def initialize(repository_snapshot:) + @repository_snapshot = repository_snapshot + @errors = {} + end + + def call + return self unless valid? + + ActiveRecord::Base.transaction do + repository = @repository_snapshot.original_repository + + repository.repository_columns.each do |column| + column.snapshot!(@repository_snapshot) + end + + repository_rows = repository.repository_rows + .joins(:my_module_repository_rows) + .where(my_module_repository_rows: { my_module: @repository_snapshot.my_module }) + + repository_rows.find_each do |original_row| + original_row.snapshot!(@repository_snapshot) + end + + @repository_snapshot.ready! + rescue ActiveRecord::RecordInvalid => e + @errors[e.record.class.name.underscore] = e.record.errors.full_messages + Rails.logger.error e.message + raise ActiveRecord::Rollback + end + + self + end + + def succeed? + @errors.none? + end + + private + + def valid? + unless @repository_snapshot + @errors[:invalid_arguments] = + { 'repository_snapshot': @repository_snapshot } + .map do |key, value| + if value.nil? + I18n.t('repositories.my_module_assigned_snapshot_service.invalid_arguments', key: key.capitalize) + end + end.compact + return false + end + true + end + end +end diff --git a/app/views/my_modules/repositories/_full_view_version.html.erb b/app/views/my_modules/repositories/_full_view_version.html.erb new file mode 100644 index 000000000..59e99fc2d --- /dev/null +++ b/app/views/my_modules/repositories/_full_view_version.html.erb @@ -0,0 +1,34 @@ +
+
+
+ <% if repository_snapshot.status == 'provisioning' %> +

+ + <%= t('my_modules.repository.snapshots.full_view.provisioning') %> +

+

+ <%= t('my_modules.repository.snapshots.full_view.created_by', full_name: repository_snapshot.created_by.full_name) %> +

+ <% else %> + +

+ <%= l(repository_snapshot.updated_at, format: :full) %> +

+

+ <%= t('my_modules.repository.snapshots.full_view.created_by', full_name: repository_snapshot.created_by.full_name) %> +

+ + <% end %> +
+
+ + + +
+
+
diff --git a/app/views/my_modules/repositories/_full_view_versions_sidebar.html.erb b/app/views/my_modules/repositories/_full_view_versions_sidebar.html.erb index 53df8d39a..6c48fd15e 100644 --- a/app/views/my_modules/repositories/_full_view_versions_sidebar.html.erb +++ b/app/views/my_modules/repositories/_full_view_versions_sidebar.html.erb @@ -5,7 +5,7 @@ -
+

@@ -30,25 +30,5 @@

- <% @repository_snapshots.each do |repository_snapshot| %> -
-
-
- -

- <%= l(repository_snapshot.created_at, format: :full) %> -

-

- <%= t('my_modules.repository.snapshots.full_view.created_by', full_name: repository_snapshot.created_by.full_name) %> -

- -
-
- - - -
-
-
- <% end %> + <%= render partial: 'my_modules/repositories/full_view_version', collection: @repository_snapshots, as: :repository_snapshot %>
diff --git a/config/initializers/delayed_job_config.rb b/config/initializers/delayed_job_config.rb index 57ba970ff..4f51127e5 100644 --- a/config/initializers/delayed_job_config.rb +++ b/config/initializers/delayed_job_config.rb @@ -63,3 +63,7 @@ Delayed::Worker.max_attempts = DelayedWorkerConfig.max_attempts Delayed::Worker.max_run_time = DelayedWorkerConfig.max_run_time Delayed::Worker.read_ahead = DelayedWorkerConfig.read_ahead Delayed::Worker.default_queue_name = DelayedWorkerConfig.default_queue_name +Delayed::Worker.queue_attributes = { + high_priority: { priority: -10 }, + low_priority: { priority: 10 } +} diff --git a/config/locales/en.yml b/config/locales/en.yml index b87821703..04391e673 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -803,6 +803,7 @@ en: versions_sidebar_button: 'View versions' create_button: 'Create snapshot' created_by: 'by %{full_name}' + provisioning: 'Provisioning' modals: assign_repository_record: title: 'Assign %{repository_name} items to task %{my_module_name}' diff --git a/config/routes.rb b/config/routes.rb index 7c734b76d..e50f83fbc 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -394,10 +394,11 @@ Rails.application.routes.draw do resources :repository_snapshots, controller: 'my_module_repository_snapshots', as: :snapshots, - only: %i(create destroy) do + only: %i(create show destroy) do member do get :full_view_table, to: 'my_module_repository_snapshots#full_view_table' post :index_dt, to: 'my_module_repository_snapshots#index_dt' + get :status, to: 'my_module_repository_snapshots#status' end end diff --git a/db/migrate/20200331183640_add_repository_snapshots.rb b/db/migrate/20200331183640_add_repository_snapshots.rb index cd05dbff7..771e70131 100644 --- a/db/migrate/20200331183640_add_repository_snapshots.rb +++ b/db/migrate/20200331183640_add_repository_snapshots.rb @@ -3,6 +3,7 @@ class AddRepositorySnapshots < ActiveRecord::Migration[6.0] def up add_column :repositories, :parent_id, :bigint, null: true + add_column :repositories, :status, :integer, null: true add_reference :repositories, :my_module add_column :repositories, :type, :string diff --git a/db/structure.sql b/db/structure.sql index d6a9dcd33..b29d33308 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -1101,6 +1101,7 @@ CREATE TABLE public.repositories ( discarded_at timestamp without time zone, permission_level integer DEFAULT 0 NOT NULL, parent_id bigint, + status integer, my_module_id bigint, type character varying ); From 307a5d7e1babe1fb279701815bb0b684d046250a Mon Sep 17 00:00:00 2001 From: Oleksii Kriuchykhin Date: Wed, 29 Apr 2020 10:42:00 +0200 Subject: [PATCH 2/2] Improve code style in migration [SCI-4552] --- db/migrate/20200331183640_add_repository_snapshots.rb | 9 ++++++--- db/structure.sql | 4 ++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/db/migrate/20200331183640_add_repository_snapshots.rb b/db/migrate/20200331183640_add_repository_snapshots.rb index 771e70131..2273279f7 100644 --- a/db/migrate/20200331183640_add_repository_snapshots.rb +++ b/db/migrate/20200331183640_add_repository_snapshots.rb @@ -2,10 +2,13 @@ class AddRepositorySnapshots < ActiveRecord::Migration[6.0] def up - add_column :repositories, :parent_id, :bigint, null: true - add_column :repositories, :status, :integer, null: true + change_table :repositories, bulk: true do |t| + t.string :type + t.bigint :parent_id, null: true + t.integer :status, null: true + end + add_reference :repositories, :my_module - add_column :repositories, :type, :string execute "UPDATE \"repositories\" SET \"type\" = 'Repository'" execute "UPDATE \"activities\" SET \"subject_type\" = 'RepositoryBase' WHERE \"subject_type\" = 'Repository'" diff --git a/db/structure.sql b/db/structure.sql index b29d33308..8758c90e6 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -1100,10 +1100,10 @@ CREATE TABLE public.repositories ( updated_at timestamp without time zone, discarded_at timestamp without time zone, permission_level integer DEFAULT 0 NOT NULL, + type character varying, parent_id bigint, status integer, - my_module_id bigint, - type character varying + my_module_id bigint );