diff --git a/Gemfile b/Gemfile index 4946767cd..b523846fa 100644 --- a/Gemfile +++ b/Gemfile @@ -12,7 +12,7 @@ gem 'bootstrap_form', '~> 2.7.0' gem 'devise', '~> 4.6.2' gem 'devise_invitable' gem 'figaro' -gem 'pg', '~> 0.18' +gem 'pg', '~> 1.1' gem 'pg_search' # PostgreSQL full text search gem 'rails', '~> 5.2.3' gem 'recaptcha', require: 'recaptcha/rails' diff --git a/Gemfile.lock b/Gemfile.lock index e2e5e6687..c2297e6d7 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -364,7 +364,9 @@ GEM parallel (1.17.0) parser (2.6.3.0) ast (~> 2.4.0) - pg (0.21.0) + pg (1.1.4) + activerecord (>= 4.2) + activesupport (>= 4.2) pg_search (2.2.0) activerecord (>= 4.2) activesupport (>= 4.2) @@ -627,7 +629,7 @@ DEPENDENCIES omniauth-linkedin-oauth2 overcommit paperclip (~> 6.1) - pg (~> 0.18) + pg (~> 1.1) pg_search phantomjs poltergeist diff --git a/app/assets/javascripts/assets.js b/app/assets/javascripts/assets.js deleted file mode 100644 index 31d19b6ab..000000000 --- a/app/assets/javascripts/assets.js +++ /dev/null @@ -1,76 +0,0 @@ -/* global animateSpinner FilePreviewModal */ - -var Assets = (function() { - var DELAY = 5000; - var REPETITIONS_LIMIT = 30; - - function finalizeAsset(element) { - var $el = $(element); - - $el.attr('data-status', 'asset-failed'); - if ($el.data('filename')) { - $el.html($el.data('filename')); - } - } - - function refreshAsset(element) { - var $el = $(element); - - if ($el.data('refreshCounter') > REPETITIONS_LIMIT) { - finalizeAsset($el); - return; - } - - // Perform an AJAX call to present URL - // to check if file already exists - $.ajax({ - url: $el.data('present-url'), - type: 'GET', - dataType: 'json', - success: function(data) { - if (data.processing === true) { - setTimeout(function() { - refreshAsset($el); - }, DELAY); - } - - if (data.processing === false) { - $el.html(data.placeholder_html); - $el.attr('data-status', 'asset-loaded'); - animateSpinner($el, false); - FilePreviewModal.init(); - } - }, - error: function() { - animateSpinner(null, false); - } - }); - - $el.data().refreshCounter += 1; - } - - function setupAssetsLoading() { - var elements = $("[data-status='asset-loading']"); - - if (!elements.length) { - return; - } - - $.each(elements, function(_, el) { - var $el = $(el); - - if ($el.data('refreshIsRunning') === true) return; - - $el.data('refreshCounter', 1); - $el.data('refreshIsRunning', true); - - setTimeout(function() { - refreshAsset($el); - }, DELAY); - }); - } - - return Object.freeze({ - setupAssetsLoading: setupAssetsLoading - }); -}()); diff --git a/app/assets/javascripts/my_modules/results.js.erb b/app/assets/javascripts/my_modules/results.js.erb index 73261444d..2248a5d52 100644 --- a/app/assets/javascripts/my_modules/results.js.erb +++ b/app/assets/javascripts/my_modules/results.js.erb @@ -1,5 +1,3 @@ -//= require assets - (function(global) { 'use strict'; @@ -112,7 +110,6 @@ }); _renderTable($(result).find('div.step-result-hot-table')); animateSpinner(null, false); - Assets.setupAssetsLoading(); } function _renderTable(table) { diff --git a/app/assets/javascripts/protocols/steps.js.erb b/app/assets/javascripts/protocols/steps.js.erb index a1cc278c2..8f5e1fceb 100644 --- a/app/assets/javascripts/protocols/steps.js.erb +++ b/app/assets/javascripts/protocols/steps.js.erb @@ -1,6 +1,5 @@ //= require Sortable.min //= require canvas-to-blob.min -//= require assets (function(global) { 'use strict'; @@ -246,7 +245,6 @@ $new_step.find("[data-role='step-hot-table']").each(function() { renderTable($(this)); }); - Assets.setupAssetsLoading(); }) .on("ajax:error", function(e, xhr, status, error) { $form.renderFormErrors('step', xhr.responseJSON, true, e); @@ -609,7 +607,10 @@ $(this).handsontable("render"); }); animateSpinner(null, false); +<<<<<<< HEAD Assets.setupAssetsLoading(); +======= +>>>>>>> activestorage_migration DragNDropSteps.clearFiles(); FilePreviewModal.init(); $.initTooltips(); @@ -688,7 +689,6 @@ initCallBacks(); initHandsOnTable($(document)); expandAllSteps(); - Assets.setupAssetsLoading(); FilePreviewModal.init(); TinyMCE.highlight(); SmartAnnotation.preventPropagation('.atwho-user-popover'); diff --git a/app/assets/javascripts/repositories/forms/repository_item_edit.js b/app/assets/javascripts/repositories/forms/repository_item_edit.js index 0a36e2d9e..b6b1d5a0d 100644 --- a/app/assets/javascripts/repositories/forms/repository_item_edit.js +++ b/app/assets/javascripts/repositories/forms/repository_item_edit.js @@ -71,7 +71,7 @@ $.each(itemData.repository_row.repository_cells, function(i, cell) { var tableCellId = 'colId-' + cell.cell_column_id; if(cell.type === 'RepositoryAssetValue') { - formBindingsData[tableCellId] = new File([""], cell.value.file_file_name); + formBindingsData[tableCellId] = new File([""], cell.value.file_name); } else { formBindingsData[tableCellId] = cell.value; } @@ -193,7 +193,7 @@ * @returns (String) */ function changeToInputFileField(object, name, value, id) { - var fileName = (value.file_file_name) ? value.file_file_name : I18n.t('general.file.no_file_chosen'); + var fileName = (value.file_name) ? value.file_name : I18n.t('general.file.no_file_chosen'); var buttonLabel = I18n.t('general.file.choose'); var html = "
" + "
" + buttonLabel + "

" + truncateLongString(fileName, 20) + "

"; - if(value.file_file_name) { + if(value.file_name) { html += "
@asset.id, 'type' => (@asset.image? ? 'image' : 'file'), - 'filename' => truncate(escape_input(@asset.file_file_name), + 'filename' => truncate(escape_input(@asset.file_name), length: Constants::FILENAME_TRUNCATION_LENGTH), 'download-url' => download_asset_path(@asset, timestamp: Time.now.to_i) } @@ -63,22 +36,18 @@ class AssetsController < ApplicationController end if @asset.image? - if ['image/jpeg', 'image/pjpeg'].include? @asset.file.content_type + if ['image/jpeg', 'image/pjpeg'].include? @asset.content_type response_json['quality'] = @asset.file_image_quality || 90 end response_json.merge!( 'editable' => @asset.editable_image? && can_edit, 'mime-type' => @asset.file.content_type, - 'processing' => @asset.file.processing?, - 'large-preview-url' => @asset.url(:large), - 'processing-img' => image_tag('medium/processing.gif') + 'large-preview-url' => @asset.large_preview ) else - response_json['processing'] = @asset.file.processing? - response_json['preview-icon'] = render_to_string( - partial: 'shared/file_preview_icon.html.erb', - locals: { asset: @asset } - ) + + response_json['preview-icon'] = render_to_string(partial: 'shared/file_preview_icon.html.erb', + locals: { asset: @asset }) end if wopi_enabled? && wopi_file?(@asset) @@ -102,7 +71,7 @@ class AssetsController < ApplicationController # Check whether the wopi file can be edited and return appropriate response def wopi_file_edit_button_status - file_ext = @asset.file_file_name.split('.').last + file_ext = @asset.file_name.split('.').last if Constants::WOPI_EDITABLE_FORMATS.include?(file_ext) edit_supported = true title = '' @@ -120,20 +89,16 @@ class AssetsController < ApplicationController end def download - if !@asset.file_present - render_404 and return - elsif @asset.file.is_stored_on_s3? - redirect_to @asset.presigned_url(download: true), status: 307 + if !@asset.file.attached? + render_404 else - send_file @asset.file.path, filename: URI.unescape(@asset.file_file_name), - type: @asset.file_content_type + redirect_to rails_blob_path(@asset.file, disposition: 'attachment') end end def edit - action = @asset.file_file_size.zero? && !@asset.locked? ? 'editnew' : 'edit' - @action_url = append_wd_params(@asset - .get_action_url(current_user, action, false)) + action = @asset.file_size.zero? && !@asset.locked? ? 'editnew' : 'edit' + @action_url = append_wd_params(@asset.get_action_url(current_user, action, false)) @favicon_url = @asset.favicon_url('edit') tkn = current_user.get_wopi_token @token = tkn.token @@ -146,8 +111,7 @@ class AssetsController < ApplicationController end def view - @action_url = append_wd_params(@asset - .get_action_url(current_user, 'view', false)) + @action_url = append_wd_params(@asset.get_action_url(current_user, 'view', false)) @favicon_url = @asset.favicon_url('view') tkn = current_user.get_wopi_token @token = tkn.token @@ -162,12 +126,11 @@ class AssetsController < ApplicationController def update_image @asset = Asset.find(params[:id]) - orig_file_size = @asset.file_file_size - orig_file_name = @asset.file_file_name + orig_file_size = @asset.file_size + orig_file_name = @asset.file_name return render_403 unless can_read_team?(@asset.team) - @asset.file = params[:image] - @asset.file_file_name = orig_file_name + @asset.file.attach(io: params.require(:image), filename: orig_file_name) @asset.save! create_edit_image_activity(@asset, current_user, :finish_editing) # release previous image space @@ -212,10 +175,10 @@ class AssetsController < ApplicationController render_403 && return unless %w(docx xlsx pptx).include?(params[:file_type]) # Asset validation - file = Paperclip.io_adapters.for(StringIO.new) - file.original_filename = "#{params[:file_name]}.#{params[:file_type]}" - file.content_type = wopi_content_type(params[:file_type]) - asset = Asset.new(file: file, created_by: current_user, file_present: true) + asset = Asset.new(created_by: current_user) + asset.file.attach(io: StringIO.new, + filename: "#{params[:file_name]}.#{params[:file_type]}", + content_type: wopi_content_type(params[:file_type])) unless asset.valid?(:wopi_file_creation) render json: { @@ -302,9 +265,7 @@ class AssetsController < ApplicationController end def asset_params - params.permit( - :file - ) + params.permit(:file) end def asset_data_type(asset) diff --git a/app/controllers/experiments_controller.rb b/app/controllers/experiments_controller.rb index 7ff2856af..4b2e45bc4 100644 --- a/app/controllers/experiments_controller.rb +++ b/app/controllers/experiments_controller.rb @@ -238,14 +238,13 @@ class ExperimentsController < ApplicationController end def updated_img - if @experiment.workflowimg.present? && !@experiment.workflowimg.exists? - @experiment.workflowimg = nil - @experiment.save + if @experiment.workflowimg.attached? && !@experiment.workflowimg_exists? + @experiment.workflowimg.purge @experiment.generate_workflow_img end respond_to do |format| format.json do - if @experiment.workflowimg.present? + if @experiment.workflowimg.attached? render json: {}, status: 200 else render json: {}, status: 404 diff --git a/app/controllers/protocols_controller.rb b/app/controllers/protocols_controller.rb index 714d79013..c35cf182b 100644 --- a/app/controllers/protocols_controller.rb +++ b/app/controllers/protocols_controller.rb @@ -794,14 +794,11 @@ class ProtocolsController < ApplicationController step_dir = "#{protocol_dir}/#{step_guid}" if step.assets.exists? step.assets.order(:id).each do |asset| - next unless asset.file.exists? + next unless asset.file.attached? asset_guid = get_guid(asset.id) - asset_file_name = asset_guid.to_s + - File.extname(asset.file_file_name).to_s + asset_file_name = asset_guid.to_s + File.extname(asset.file_name).to_s ostream.put_next_entry("#{step_dir}/#{asset_file_name}") - input_file = asset.open - ostream.print(input_file.read) - input_file.close + ostream.print(asset.file.download) end end ostream = step.tiny_mce_assets.save_to_eln(ostream, step_dir) diff --git a/app/controllers/result_assets_controller.rb b/app/controllers/result_assets_controller.rb index 3520626b2..092d32084 100644 --- a/app/controllers/result_assets_controller.rb +++ b/app/controllers/result_assets_controller.rb @@ -194,10 +194,10 @@ class ResultAssetsController < ApplicationController success = true results = [] params[:results_files].values.each_with_index do |file, index| - asset = Asset.new(file: file, - created_by: current_user, + asset = Asset.new(created_by: current_user, last_modified_by: current_user, team: current_team) + asset.file.attach(file) result = Result.new(user: current_user, my_module: @my_module, name: params[:results_names][index.to_s], diff --git a/app/controllers/teams_controller.rb b/app/controllers/teams_controller.rb index 425053d4e..46f3422f4 100644 --- a/app/controllers/teams_controller.rb +++ b/app/controllers/teams_controller.rb @@ -1,13 +1,6 @@ class TeamsController < ApplicationController - before_action :load_vars, only: %i(parse_sheet import_samples - export_samples export_projects - export_projects_modal) - - before_action :check_create_samples_permissions, only: %i(parse_sheet - import_samples) - before_action :check_view_samples_permission, only: [:export_samples] - before_action :check_export_projects_permissions, - only: %i(export_projects_modal export_projects) + before_action :load_vars, only: %i(parse_sheet export_projects export_projects_modal) + before_action :check_export_projects_permissions, only: %i(export_projects_modal export_projects) def parse_sheet session[:return_to] ||= request.referer @@ -66,167 +59,6 @@ class TeamsController < ApplicationController end end - def import_samples - session[:return_to] ||= request.referer - - respond_to do |format| - if import_params[:file_id] - @temp_file = TempFile.find_by_id(import_params[:file_id]) - - if @temp_file - # Check if session_id is equal to prevent file stealing - if @temp_file.session_id == session.id - # Check if mappings exists or else we don't have anything to parse - if import_params[:mappings] - @sheet = SpreadsheetParser.open_spreadsheet(@temp_file.file) - - # Check for duplicated values - h1 = import_params[:mappings].clone.delete_if { |_, v| v.empty? } - if h1.length == h1.invert.length - - # Check if there exist mapping for sample name (it's mandatory) - if import_params[:mappings].value?('-1') - result = @team.import_samples(@sheet, - import_params[:mappings], - current_user) - nr_of_added = result[:nr_of_added] - total_nr = result[:total_nr] - - if result[:status] == :ok - # If no errors are present, redirect back - # to samples table - flash[:success] = t( - "teams.import_samples.success_flash", - nr: nr_of_added, - samples: t( - "teams.import_samples.sample", - count: total_nr - ) - ) - @temp_file.destroy - format.html { - redirect_to session.delete(:return_to) - } - format.json { - flash.keep(:success) - render json: { status: :ok } - } - else - # Otherwise, also redirect back, - # but display different message - flash[:alert] = t( - "teams.import_samples.partial_success_flash", - nr: nr_of_added, - samples: t( - "teams.import_samples.sample", - count: total_nr - ) - ) - @temp_file.destroy - format.html { - redirect_to session.delete(:return_to) - } - format.json { - flash.keep(:alert) - render json: { status: :unprocessable_entity } - } - end - else - # This is currently the only AJAX error response - flash_alert = t( - "teams.import_samples.errors.no_sample_name") - format.html { - flash[:alert] = flash_alert - redirect_to session.delete(:return_to) - } - format.json { - render json: { - html: render_to_string({ - partial: "parse_error.html.erb", - locals: { error: flash_alert } - }) - }, - status: :unprocessable_entity - } - end - else - # This code should never execute unless user tampers with - # JS (selects same column in more than one dropdown) - flash_alert = t( - "teams.import_samples.errors.duplicated_values") - format.html { - flash[:alert] = flash_alert - redirect_to session.delete(:return_to) - } - format.json { - render json: { - html: render_to_string({ - partial: "parse_error.html.erb", - locals: { error: flash_alert } - }) - }, - status: :unprocessable_entity - } - end - else - @temp_file.destroy - flash[:alert] = t( - "teams.import_samples.errors.no_data_to_parse") - format.html { - redirect_to session.delete(:return_to) - } - format.json { - flash.keep(:alert) - render json: { status: :unprocessable_entity } - } - end - else - @temp_file.destroy - flash[:alert] = t( - "teams.import_samples.errors.session_expired") - format.html { - redirect_to session.delete(:return_to) - } - format.json { - flash.keep(:alert) - render json: { status: :unprocessable_entity } - } - end - else - # No temp file to begin with, so no need to destroy it - flash[:alert] = t( - "teams.import_samples.errors.temp_file_not_found") - format.html { - redirect_to session.delete(:return_to) - } - format.json { - flash.keep(:alert) - render json: { status: :unprocessable_entity } - } - end - else - flash[:alert] = t( - "teams.import_samples.errors.temp_file_not_found") - format.html { - redirect_to session.delete(:return_to) - } - format.json { - flash.keep(:alert) - render json: { status: :unprocessable_entity } - } - end - end - end - - def export_samples - if export_params[:sample_ids] && export_params[:header_ids] - generate_samples_zip - else - flash[:alert] = t('zip_export.export_error') - end - redirect_back(fallback_location: root_path) - end - def export_projects if current_user.has_available_exports? current_user.increase_daily_exports_counter! @@ -318,12 +150,6 @@ class TeamsController < ApplicationController render_403 unless can_create_samples?(@team) end - def check_view_samples_permission - unless can_read_team?(@team) - render_403 - end - end - def check_export_projects_permissions render_403 unless can_read_team?(@team) diff --git a/app/controllers/tiny_mce_assets_controller.rb b/app/controllers/tiny_mce_assets_controller.rb index 88a96a847..2e64a4248 100644 --- a/app/controllers/tiny_mce_assets_controller.rb +++ b/app/controllers/tiny_mce_assets_controller.rb @@ -3,13 +3,17 @@ class TinyMceAssetsController < ApplicationController def create image = params.fetch(:file) { render_404 } - tiny_img = TinyMceAsset.new(image: image, - team_id: current_team.id, - saved: false) - if tiny_img.save + tiny_img = TinyMceAsset.new(team_id: current_team.id, saved: false) + + tiny_img.transaction do + tiny_img.save! + tiny_img.image.attach(io: image, filename: image.original_filename) + end + + if tiny_img.persisted? render json: { image: { - url: view_context.image_url(tiny_img.url(:large)), + url: url_for(tiny_img.image), token: Base62.encode(tiny_img.id) } }, content_type: 'text/html' @@ -19,10 +23,4 @@ class TinyMceAssetsController < ApplicationController }, status: :unprocessable_entity end end - - def update - image = TinyMceAsset.find_by_id(Base62.decode(params[:id])) - image.update(image: params[:image], image_file_name: image.image_file_name) - render json: { url: view_context.image_url(image.url) } - end end diff --git a/app/controllers/users/registrations_controller.rb b/app/controllers/users/registrations_controller.rb index 58ac7bb8f..d36d90945 100644 --- a/app/controllers/users/registrations_controller.rb +++ b/app/controllers/users/registrations_controller.rb @@ -8,40 +8,8 @@ class Users::RegistrationsController < Devise::RegistrationsController def avatar user = User.find_by_id(params[:id]) || current_user - style = params[:style] || "icon_small" - # TODO: Maybe avatar should be an Asset, so it's methods could be used, - # e.g. presigned_url in this case - redirect_to user.avatar.url(style.to_sym), status: 307 - end - - # Validates asset and then generates S3 upload posts, because - # otherwise untracked files could be uploaded to S3 - def signature - respond_to do |format| - format.json { - - # Changed avatar values are only used for pre-generating S3 key - # and user object is not persisted with this values. - current_user.empty_avatar avatar_params[:file].original_filename, - avatar_params[:file].size - - validation_asset = Asset.new(avatar_params) - if current_user.valid? && validation_asset.valid? - render json: { - posts: generate_upload_posts - } - else - if validation_asset.errors[:file].any? - # Add file content error - current_user.errors[:avatar] << validation_asset.errors[:file].first - end - render json: { - status: 'error', - errors: current_user.errors - }, status: :bad_request - end - } - end + style = params[:style] || :icon_small + redirect_to user.avatar_url(style) end def update_resource(resource, params) @@ -50,12 +18,10 @@ class Users::RegistrationsController < Devise::RegistrationsController if params.include? :change_password # Special handling if changing password params.delete(:change_password) - if ( - resource.valid_password?(params[:current_password]) and - params.include? :password and - params.include? :password_confirmation and - params[:password].blank? - ) then + if resource.valid_password?(params[:current_password]) && + params.include?(:password) && + params.include?(:password_confirmation) && + params[:password].blank? # If new password is blank and we're in process of changing # password, add error to the resource and return false resource.errors.add(:password, :blank) @@ -65,16 +31,26 @@ class Users::RegistrationsController < Devise::RegistrationsController end elsif params.include? :change_avatar params.delete(:change_avatar) - unless params.include? :avatar + if !params.include?(:avatar) resource.errors.add(:avatar, :blank) false else + temp_file = Tempfile.new('avatar', Rails.root.join('tmp')) + begin + temp_file.binmode + temp_file.write(Base64.decode64(params[:avatar].sub(%r{^data:image\/jpeg\;base64,}, ''))) + temp_file.rewind + resource.avatar.attach(io: temp_file, filename: 'avatar.jpg') + ensure + temp_file.close + temp_file.unlink + end + params.delete(:avatar) resource.update_without_password(params) end - elsif params.include? :email or params.include? :password + elsif params.include?(:email) || params.include?(:password) # For changing email or password, validate current_password resource.update_with_password(params) - else # For changing some attributes, no current_password validation # is required @@ -235,7 +211,6 @@ class Users::RegistrationsController < Devise::RegistrationsController :full_name, :initials, :avatar, - :avatar_file_name, :email, :password, :password_confirmation, @@ -251,45 +226,6 @@ class Users::RegistrationsController < Devise::RegistrationsController ) end - # Generates posts for uploading files (many sizes of same file) - # to S3 server - def generate_upload_posts - posts = [] - file_size = current_user.avatar_file_size - content_type = current_user.avatar_content_type - s3_post = S3_BUCKET.presigned_post( - key: current_user.avatar.path[1..-1], - success_action_status: '201', - acl: 'private', - storage_class: "STANDARD", - content_length_range: file_size..file_size, - content_type: content_type - ) - posts.push({ - url: s3_post.url, - fields: s3_post.fields - }) - - current_user.avatar.options[:styles].each do |style, option| - s3_post = S3_BUCKET.presigned_post( - key: current_user.avatar.path(style)[1..-1], - success_action_status: '201', - acl: 'public-read', - storage_class: "REDUCED_REDUNDANCY", - content_length_range: 1..Rails.configuration.x.file_max_size_mb.megabytes, - content_type: content_type - ) - posts.push({ - url: s3_post.url, - fields: s3_post.fields, - style_option: option, - mime_type: content_type - }) - end - - posts - end - private def layout diff --git a/app/controllers/wopi_controller.rb b/app/controllers/wopi_controller.rb index 183403aef..e081cb8b5 100644 --- a/app/controllers/wopi_controller.rb +++ b/app/controllers/wopi_controller.rb @@ -52,9 +52,9 @@ class WopiController < ActionController::Base asset_owner_id = @asset.created_by_id.to_s if @asset.created_by_id msg = { - BaseFileName: @asset.file_file_name, + BaseFileName: @asset.file_name, OwnerId: asset_owner_id, - Size: @asset.file_file_size, + Size: @asset.file_size, UserId: @user.id.to_s, Version: @asset.version.to_s, SupportsExtendedLockLength: true, @@ -218,7 +218,7 @@ class WopiController < ActionController::Base response.headers['X-WOPI-Lock'] = @asset.lock render body: nil, status: 409 and return end - elsif !@asset.file_file_size.nil? && @asset.file_file_size.zero? + elsif !@asset.file_size.nil? && @asset.file_size.zero? logger.warn 'WOPI: initializing empty file' @team.release_space(@asset.estimated_size) diff --git a/app/controllers/zip_exports_controller.rb b/app/controllers/zip_exports_controller.rb index e30f1300a..5c8d5e669 100644 --- a/app/controllers/zip_exports_controller.rb +++ b/app/controllers/zip_exports_controller.rb @@ -5,12 +5,10 @@ class ZipExportsController < ApplicationController before_action :check_download_permissions, except: :file_expired def download - if @zip_export.stored_on_s3? - redirect_to @zip_export.presigned_url(download: true), status: 307 + if !@zip_export.zip_file.attached? + render_404 else - send_file @zip_export.zip_file.path, - filename: URI.unescape(@zip_export.zip_file_file_name), - type: 'application/zip' + redirect_to rails_blob_path(@zip_export.zip_file, disposition: 'attachment') end end diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index e093d5cdc..d7b26e533 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -196,7 +196,12 @@ module ApplicationHelper # No more dirty hack def user_avatar_absolute_url(user, style) begin - return user.avatar_base64(style) unless missing_avatar(user, style) + unless missing_avatar(user, style) + image = File.open(user.avatar_variant(style)) + encoded_data = Base64.strict_encode64(image) + avatar_base64 = "data:#{user.avatar_content_type};base64,#{encoded_data}" + return avatar_base64 + end rescue StandardError => e Rails.logger.error e.message end diff --git a/app/helpers/file_icons_helper.rb b/app/helpers/file_icons_helper.rb index 021303b06..eaeca001d 100644 --- a/app/helpers/file_icons_helper.rb +++ b/app/helpers/file_icons_helper.rb @@ -1,11 +1,12 @@ module FileIconsHelper def wopi_file?(asset) - file_ext = asset.file_file_name.split('.').last - %w(csv ods xls xlsb xlsm xlsx odp pot potm potx pps ppsm ppsx ppt pptm pptx doc docm docx dot dotm dotx odt rtf).include?(file_ext) + file_ext = asset.file_name.split('.').last + %w(csv ods xls xlsb xlsm xlsx odp pot potm potx pps ppsm + ppsx ppt pptm pptx doc docm docx dot dotm dotx odt rtf).include?(file_ext) end def file_fa_icon_class(asset) - file_ext = asset.file_file_name.split('.').last + file_ext = asset.file_name.split('.').last if Extends::FILE_FA_ICON_MAPPINGS[file_ext] # Check for custom mappings or possible overrides return Extends::FILE_FA_ICON_MAPPINGS[file_ext] @@ -28,7 +29,7 @@ module FileIconsHelper # For showing next to file def file_extension_icon(asset) - file_ext = asset.file_file_name.split('.').last + file_ext = asset.file_name.split('.').last if Constants::FILE_TEXT_FORMATS.include?(file_ext) image_link = 'office/Word-docx_20x20x32.png' elsif Constants::FILE_TABLE_FORMATS.include?(file_ext) @@ -38,9 +39,7 @@ module FileIconsHelper end # Now check for custom mappings or possible overrides - if Extends::FILE_ICON_MAPPINGS[file_ext] - image_link = Extends::FILE_ICON_MAPPINGS[file_ext] - end + image_link = Extends::FILE_ICON_MAPPINGS[file_ext] if Extends::FILE_ICON_MAPPINGS[file_ext] if image_link image_tag image_link @@ -51,7 +50,7 @@ module FileIconsHelper # For showing in view/edit buttons (WOPI) def file_application_icon(asset) - file_ext = asset.file_file_name.split('.').last + file_ext = asset.file_name.split('.').last if Constants::FILE_TEXT_FORMATS.include?(file_ext) image_link = 'office/Word-color_16x16x32.png' elsif Constants::FILE_TABLE_FORMATS.include?(file_ext) @@ -69,7 +68,7 @@ module FileIconsHelper # Shows correct WOPI application text (Word Online/Excel ..) def wopi_button_text(asset, action) - file_ext = asset.file_file_name.split('.').last + file_ext = asset.file_name.split('.').last if Constants::FILE_TEXT_FORMATS.include?(file_ext) app = 'Word Online' elsif Constants::FILE_TABLE_FORMATS.include?(file_ext) diff --git a/app/helpers/my_modules_helper.rb b/app/helpers/my_modules_helper.rb index 5dda6a077..e578df1ea 100644 --- a/app/helpers/my_modules_helper.rb +++ b/app/helpers/my_modules_helper.rb @@ -30,12 +30,11 @@ module MyModulesHelper end def az_ordered_assets_index(step, asset_id) - assets = [] - assets += step.assets - assets += step.marvin_js_assets - assets.sort! do |a, b| - (a[asset_name_sort_field(a)]).downcase <=> (b[asset_name_sort_field(b)]).downcase - end.pluck(:id).index(asset_id) + step.assets + .joins(file_attachment: :blob) + .order(Arel.sql('LOWER(active_storage_blobs.filename)')) + .pluck(:id) + .index(asset_id) end def number_of_samples(my_module) diff --git a/app/helpers/reports_helper.rb b/app/helpers/reports_helper.rb index 0cae132a5..6be90ef63 100644 --- a/app/helpers/reports_helper.rb +++ b/app/helpers/reports_helper.rb @@ -97,8 +97,9 @@ module ReportsHelper end # "Hack" to omit file preview URL because of WKHTML issues - def report_image_asset_url(asset, _type = :asset, klass = nil) - image_tag(asset.generate_base64(:medium), class: klass) + def report_image_asset_url(asset, type = :asset, klass = nil) + url = type == :tiny_mce_asset ? asset.preview : asset.large_preview + image_tag(url, class: klass) end # "Hack" to load Glyphicons css directly from the CDN @@ -147,7 +148,7 @@ module ReportsHelper def obj_name_to_filename(obj, filename_suffix = '') obj_name = if obj.class == Asset - obj_name, extension = obj.file_file_name.split('.') + obj_name, extension = obj.file_name.split('.') extension&.prepend('.') obj_name elsif obj.class.in? [Table, Result, Repository] diff --git a/app/models/asset.rb b/app/models/asset.rb index 057feb8c0..1e1c2e84a 100644 --- a/app/models/asset.rb +++ b/app/models/asset.rb @@ -1,4 +1,5 @@ class Asset < ApplicationRecord + include ActiveStorage::Downloading include SearchableModel include DatabaseHelper include Encryptor @@ -78,30 +79,9 @@ class Asset < ApplicationRecord has_many :report_elements, inverse_of: :asset, dependent: :destroy has_one :asset_text_datum, inverse_of: :asset, dependent: :destroy - # Specific file errors propagate to "file" error hash key, - # so use just these errors - after_validation :filter_paperclip_errors - # Needed because Paperclip validatates on creation - after_initialize :filter_paperclip_errors, if: :new_record? - before_destroy :paperclip_delete, prepend: true after_save { result&.touch; step&.touch } - attr_accessor :file_content, :file_info, :preview_cached, :in_template - - def file_empty(name, size) - file_ext = name.split(".").last - self.file_file_name = name - self.file_content_type = Rack::Mime.mime_type(".#{file_ext}") - self.file_file_size = size - self.file_updated_at = DateTime.now - self.file_present = false - end - - def self.new_empty(name, size) - asset = self.new - asset.file_empty name, size - asset - end + attr_accessor :file_content, :file_info, :in_template def self.search( user, @@ -136,6 +116,7 @@ class Asset < ApplicationRecord assets_in_steps, assets_in_results, assets_in_inventories) new_query = Asset.left_outer_joins(:asset_text_datum) + .left_outer_joins(file_attachment: :blob) .from(assets, 'assets') a_query = s_query = '' @@ -160,7 +141,7 @@ class Asset < ApplicationRecord s_query = s_query.tr('\'', '"') new_query = new_query.where( - "(trim_html_tags(assets.file_file_name) #{like} ? " \ + "(active_storage_blobs.filename #{like} ? " \ "OR asset_text_data.data_vector @@ to_tsquery(?))", a_query, s_query @@ -181,7 +162,7 @@ class Asset < ApplicationRecord .join('|') .tr('\'', '"') new_query = new_query.where( - "(trim_html_tags(assets.file_file_name) #{like} ANY (array[?]) " \ + "(active_storage_blobs.filename #{like} ANY (array[?]) " \ "OR asset_text_data.data_vector @@ to_tsquery(?))", a_query, s_query @@ -203,34 +184,69 @@ class Asset < ApplicationRecord end end + def blob + file&.blob + end + def previewable? + return false unless file.attached? + previewable_document? || previewable_image? end def medium_preview return file.variant(resize: Constants::MEDIUM_PIC_FORMAT) if previewable_image? - 'medium/processing.gif' + '/images/medium/processing.gif' # file.preview(resize: Constants::MEDIUM_PIC_FORMAT) end def large_preview return file.variant(resize: Constants::LARGE_PIC_FORMAT) if previewable_image? - 'large/processing.gif' + '/images/large/processing.gif' # file.preview(resize: Constants::LARGE_PIC_FORMAT) end def file_name + return '' unless file.attached? + file.blob&.filename&.to_s end def file_size + return 0 unless file.attached? + file.blob&.byte_size end + def content_type + file&.blob&.content_type + end + + def file_size + file&.blob&.byte_size + end + + def duplicate + new_asset = dup + return unless new_asset.save + + return new_asset unless file.attached? + + duplicate_file(new_asset) + new_asset + end + + def duplicate_file(to_asset) + download_blob_to_tempfile do |tmp_file| + to_asset.file.attach(io: tmp_file.open, filename: file_name) + end + new_asset.post_process_file(new_asset.team) + end + def extract_image_quality - return unless ['image/jpeg', 'image/pjpeg'].include? file_content_type + return unless ['image/jpeg', 'image/pjpeg'].include? content_type tempfile = file.queued_for_write[:original] unless tempfile.nil? @@ -242,30 +258,18 @@ class Asset < ApplicationRecord end def image? - file.blob.content_type == %r{^image/#{Regexp.union(Constants::WHITELISTED_IMAGE_TYPES)}} + content_type == %r{^image/#{Regexp.union(Constants::WHITELISTED_IMAGE_TYPES)}} end def text? Constants::TEXT_EXTRACT_FILE_TYPES.any? do |v| - file_content_type.start_with? v + file&.blob&.content_type&.start_with? v end end - # TODO: get the current_user - # before_save do - # if current_user - # self.created_by ||= current_user - # self.last_modified_by = current_user if self.changed? - # end - # end - - def is_stored_on_s3? - file.options[:storage].to_sym == :s3 - end - def post_process_file(team = nil) # Update self.empty - self.update(file_present: true) + update(file_present: true) # Extract asset text if it's of correct type if text? @@ -273,80 +277,56 @@ class Asset < ApplicationRecord # The extract_asset_text also includes # estimated size calculation Asset.delay(queue: :assets, run_at: 20.minutes.from_now) - .extract_asset_text(id, in_template) + .extract_asset_text_delayed(id, in_template) else # Update asset's estimated size immediately update_estimated_size(team) end end - def self.extract_asset_text(asset_id, in_template = false) + def self.extract_asset_text_delayed(asset_id, in_template = false) asset = find_by_id(asset_id) - return unless asset.present? && asset.file.present? - asset.in_template = in_template + return unless asset.present? && asset.file.attached? - begin - file_path = asset.file.path + asset.extract_asset_text(in_template) + end - if asset.file.is_stored_on_s3? - fa = asset.file.fetch - file_path = fa.path - end + def extract_asset_text(in_template = false) + self.in_template = in_template + download_blob_to_tempfile do |tmp_file| # Start Tika as a server - if !ENV['NO_TIKA_SERVER'] && Yomu.class_variable_get(:@@server_pid).nil? - Yomu.server(:text) - end + Yomu.server(:text) if !ENV['NO_TIKA_SERVER'] && Yomu.class_variable_get(:@@server_pid).nil? - y = Yomu.new file_path - text_data = y.text + text_data = Yomu.new(tmp_file.path).text - if asset.asset_text_datum.present? + if asset_text_datum.present? # Update existing text datum if it exists - asset.asset_text_datum.update(data: text_data) + asset_text_datum.update(data: text_data) else # Create new text datum - AssetTextDatum.create(data: text_data, asset: asset) + AssetTextDatum.create(data: text_data, asset: self) end - Rails.logger.info "Asset #{asset.id}: Asset file successfully extracted" + Rails.logger.info "Asset #{id}: Asset file successfully extracted" # Finally, update asset's estimated size to include # the data vector - asset.update_estimated_size(asset.team) + update_estimated_size(team) rescue StandardError => e Rails.logger.fatal( - "Asset #{asset.id}: Error extracting contents from asset "\ - "file #{asset.file.path}: #{e.message}" + "Asset #{id}: Error extracting contents from asset "\ + "file #{file.blob.key}: #{e.message}" ) - ensure - File.delete file_path if fa end end - # Workaround for making Paperclip work with asset deletion (might be because - # of how the asset models are implemented) - def paperclip_delete - report_elements.destroy_all - asset_text_datum.destroy if asset_text_datum.present? - # Nullify needed to force paperclip file deletion - file = nil - save && reload - end - - def destroy - super() - # Needed, otherwise the object isn't deleted, because of how the asset - # models are implemented - delete - end - # If team is provided, its space_taken # is updated as well def update_estimated_size(team = nil) - return if file_file_size.blank? || in_template + return if file_size.blank? || in_template - es = file_file_size + es = file_size if asset_text_datum.present? && asset_text_datum.persisted? asset_text_datum.reload es += get_octet_length_record(asset_text_datum, :data) @@ -363,66 +343,18 @@ class Asset < ApplicationRecord end end - def url(style = :original, timeout: Constants::URL_SHORT_EXPIRE_TIME) - if file.is_stored_on_s3? && !file.processing? - presigned_url(style, timeout: timeout) - else - file.url(style) - end - end - - # When using S3 file upload, we can limit file accessibility with url signing - def presigned_url(style = :original, - download: false, - timeout: Constants::URL_SHORT_EXPIRE_TIME) - if file.is_stored_on_s3? - if download - download_arg = 'attachment; filename=' + URI.escape(file_file_name) - else - download_arg = nil - end - - signer = Aws::S3::Presigner.new(client: S3_BUCKET.client) - signer.presigned_url(:get_object, - bucket: S3_BUCKET.name, - key: file.path(style)[1..-1], - expires_in: timeout, - # this response header forces object download - response_content_disposition: download_arg) - end - end - - def open - if file.is_stored_on_s3? - Kernel.open(presigned_url, "rb") - else - File.open(file.path, "rb") - end - end - - # Preserving attachments (on client-side) between failed validations - # (only usable for small/few files!). - # Needs to be called before save method and view needs to have - # :file_content and :file_info hidden field. - # If file is an image, it can be viewed on front-end - # using @preview_cached with image_tag tag. - def preserve(file_data) - if file_data[:file_content].present? - restore_cached(file_data[:file_content], file_data[:file_info]) - end - cache - end - def can_perform_action(action) if ENV['WOPI_ENABLED'] == 'true' - file_ext = file_file_name.split('.').last + file_ext = file_name.split('.').last if file_ext == 'wopitest' && (!ENV['WOPI_TEST_ENABLED'] || ENV['WOPI_TEST_ENABLED'] == 'false') return false end + action = get_action(file_ext, action) return false if action.nil? + true else false @@ -430,7 +362,7 @@ class Asset < ApplicationRecord end def get_action_url(user, action, with_tokens = true) - file_ext = file_file_name.split('.').last + file_ext = file_name.split('.').last action = get_action(file_ext, action) if !action.nil? action_url = action.urlsrc @@ -453,7 +385,7 @@ class Asset < ApplicationRecord ) action_url += "WOPISrc=#{rest_url}" if with_tokens - token = user.get_wopi_token + token = user.get_wopi_token action_url + "&access_token=#{token.token}"\ "&access_token_ttl=#{(token.ttl * 1000)}" else @@ -465,7 +397,7 @@ class Asset < ApplicationRecord end def favicon_url(action) - file_ext = file_file_name.split('.').last + file_ext = file_name.split('.').last action = get_action(file_ext, action) action.wopi_app.icon if action.try(:wopi_app) end @@ -507,8 +439,8 @@ class Asset < ApplicationRecord def update_contents(new_file) new_file.class.class_eval { attr_accessor :original_filename } - new_file.original_filename = file_file_name - self.file = new_file + new_file.original_filename = file_name + file.attach(io: new_file, filename: original_filename) self.version = version.nil? ? 1 : version + 1 save end @@ -517,6 +449,7 @@ class Asset < ApplicationRecord !locked? && %r{^image/#{Regexp.union(Constants::WHITELISTED_IMAGE_TYPES_EDITABLE)}} =~ file.content_type end +<<<<<<< HEAD def generate_base64(style) image = if file.options[:storage].to_sym == :s3 URI.parse(url(style)).open.to_a.join @@ -528,22 +461,19 @@ class Asset < ApplicationRecord end protected +======= + private +>>>>>>> activestorage_migration - # Checks if attachments is an image (in post processing imagemagick will - # generate styles) - def allow_styles_on_images - if !(file.content_type =~ %r{^(image|(x-)?application)/(x-png|pjpeg|jpeg|jpg|png|gif)$}) - return false - end + def tempdir + Rails.root.join('tmp') end - private - def previewable_document? - previewable = Constants::PREVIEWABLE_FILE_TYPES.include?(file.blob&.content_type) + previewable = Constants::PREVIEWABLE_FILE_TYPES.include?(file.content_type) - filename = file.blob&.filename - content_type = file.blob&.content_type + filename = file.filename.to_s + content_type = file.content_type extensions = %w(.xlsx .docx .pptx .xls .doc .ppt) # Mimetype sometimes recognizes Office files as zip files @@ -562,29 +492,6 @@ class Asset < ApplicationRecord file.blob&.content_type =~ %r{^image/#{Regexp.union(Constants::WHITELISTED_IMAGE_TYPES)}} end - def filter_paperclip_errors - if errors.size > 1 - temp_errors = errors[:file] - errors.clear - errors.add(:file, temp_errors) - end - end - - def file_changed? - previous_changes.present? and - ( - ( - previous_changes.key? "file_file_name" and - previous_changes["file_file_name"].first != - previous_changes["file_file_name"].last - ) or ( - previous_changes.key? "file_file_size" and - previous_changes["file_file_size"].first != - previous_changes["file_file_size"].last - ) - ) - end - def step_or_result_or_repository_asset_value # We must allow both step and result to be blank because of GUI # (even though it's not really a "valid" asset) @@ -617,30 +524,4 @@ class Asset < ApplicationRecord ) end end - - def cache - fetched_file = file.fetch - file_content = fetched_file.read - @file_content = encrypt(file_content) - @file_info = %Q[{"content_type" : "#{file.content_type}", "original_filename" : "#{file.original_filename}"}] - @file_info = encrypt(@file_info) - if !(file.content_type =~ /^image/).nil? - @preview_cached = "data:image/png;base64," + Base64.encode64(file_content) - end - end - - def restore_cached(file_content, file_info) - decoded_data = decrypt(file_content) - decoded_data_info = decrypt(file_info) - decoded_data_info = JSON.parse decoded_data_info - - data = StringIO.new(decoded_data) - data.class_eval do - attr_accessor :content_type, :original_filename - end - data.content_type = decoded_data_info['content_type'] - data.original_filename = File.basename(decoded_data_info['original_filename']) - - self.file = data - end end diff --git a/app/models/concerns/tiny_mce_images.rb b/app/models/concerns/tiny_mce_images.rb index a7965d4e8..b5cc990f2 100644 --- a/app/models/concerns/tiny_mce_images.rb +++ b/app/models/concerns/tiny_mce_images.rb @@ -18,23 +18,17 @@ module TinyMceImages description = TinyMceAsset.update_old_tinymce(description, self) tiny_mce_assets.each do |tm_asset| - tmp_f = Tempfile.open(tm_asset.image_file_name, Rails.root.join('tmp')) - begin - tm_asset.image.copy_to_local_file(:large, tmp_f.path) - encoded_tm_asset = Base64.strict_encode64(tmp_f.read) - new_tm_asset_src = "data:image/jpg;base64,#{encoded_tm_asset}" - html_description = Nokogiri::HTML(description) - tm_asset_to_update = html_description.css( - "img[data-mce-token=\"#{Base62.encode(tm_asset.id)}\"]" - )[0] - next unless tm_asset_to_update + tm_asset_key = tm_asset.image.preview.key + encoded_tm_asset = Base64.strict_encode64(tm_asset.image.service.download(tm_asset_key)) + new_tm_asset_src = "data:image/jpg;base64,#{encoded_tm_asset}" + html_description = Nokogiri::HTML(description) + tm_asset_to_update = html_description.css( + "img[data-mce-token=\"#{Base62.encode(tm_asset.id)}\"]" + )[0] + next unless tm_asset_to_update - tm_asset_to_update.attributes['src'].value = new_tm_asset_src - description = html_description.css('body').inner_html.to_s - ensure - tmp_f.close - tmp_f.unlink - end + tm_asset_to_update.attributes['src'].value = new_tm_asset_src + description = html_description.css('body').inner_html.to_s end description end @@ -72,12 +66,15 @@ module TinyMceImages cloned_img_ids = [] tiny_mce_assets.each do |tiny_img| tiny_img_clone = TinyMceAsset.new( - image: tiny_img.image, estimated_size: tiny_img.estimated_size, object: target, team: team ) - tiny_img_clone.save! + + tiny_img_clone.transaction do + tiny_img_clone.save! + tiny_img_clone.image.attach(io: tiny_img.image.open, filename: tiny_img.image.filename.to_s) + end target.tiny_mce_assets << tiny_img_clone cloned_img_ids << [tiny_img.id, tiny_img_clone.id] @@ -86,7 +83,7 @@ module TinyMceImages end def copy_unknown_tiny_mce_images - asset_team_id = Team.find_by_object(self) + asset_team_id = Team.find_by_object(self).id return unless asset_team_id object_field = Extends::RICH_TEXT_FIELD_MAPPINGS[self.class.name] @@ -97,19 +94,26 @@ module TinyMceImages if image['data-mce-token'] asset = TinyMceAsset.find_by_id(Base62.decode(image['data-mce-token'])) - next if asset && asset.object == self + next if asset && asset.object == self && asset.team_id != asset_team_id new_image = asset.image + new_image_filename = new_image.file_name else - new_image = URI.parse(image['src']) + # We need implement size and type checks here + new_image = URI.parse(image['src']).open + new_image_filename = asset.class.generate_unique_secure_token + '.jpg' end new_asset = TinyMceAsset.create( - image: new_image, object: self, team_id: asset_team_id ) + new_asset.transaction do + new_asset.save! + new_asset.image.attach(io: new_image, filename: new_image_filename) + end + image['src'] = '' image['class'] = 'img-responsive' image['data-mce-token'] = Base62.encode(new_asset.id) diff --git a/app/models/experiment.rb b/app/models/experiment.rb index d730b06d7..5d5bbb40b 100644 --- a/app/models/experiment.rb +++ b/app/models/experiment.rb @@ -28,10 +28,7 @@ class Experiment < ApplicationRecord # Associations for old activity type has_many :activities, inverse_of: :experiment - has_attached_file :workflowimg - validates_attachment :workflowimg, - content_type: { content_type: ['image/png'] }, - if: :workflowimg_check + has_one_attached :workflowimg auto_strip_attributes :name, :description, nullify: false validates :name, @@ -220,13 +217,14 @@ class Experiment < ApplicationRecord end def generate_workflow_img - if workflowimg.present? - self.workflowimg = nil - save - end + workflowimg.purge if workflowimg.attached? Experiments::GenerateWorkflowImageService.delay.call(experiment_id: id) end + def workflowimg_exists? + workflowimg.service.exist?(workflowimg.blob.key) + end + # Get projects where user is either owner or user in the same team # as this experiment def projects_with_role_above_user(current_user) @@ -613,12 +611,6 @@ class Experiment < ApplicationRecord true end - def workflowimg_check - workflowimg_content_type - rescue - false - end - def log_activity(type_of, current_user, my_module) Activities::CreateActivityService .call(activity_type: type_of, diff --git a/app/models/protocol.rb b/app/models/protocol.rb index f57b8b05d..d0009893d 100644 --- a/app/models/protocol.rb +++ b/app/models/protocol.rb @@ -234,7 +234,7 @@ class Protocol < ApplicationRecord end # Deep-clone given array of assets - def self.deep_clone_assets(assets_to_clone, team) + def self.deep_clone_assets(assets_to_clone) assets_to_clone.each do |src_id, dest_id| src = Asset.find_by_id(src_id) dest = Asset.find_by_id(dest_id) @@ -242,34 +242,7 @@ class Protocol < ApplicationRecord next unless src.present? && dest.present? # Clone file - dest.file = src.file - dest.save! - - if dest.image? - dest.file.reprocess!(:large) - dest.file.reprocess!(:medium) - end - - # Clone extracted text data if it exists - if (atd = src.asset_text_datum).present? - atd2 = AssetTextDatum.new( - data: atd.data, - asset: dest - ) - atd2.save! - end - - # Update estimated size of cloned asset - # (& file_present flag) - dest.update( - estimated_size: src.estimated_size, - file_present: true - ) - - # Update team's space taken - team.reload - team.take_space(dest.estimated_size) - team.save! + src.duplicate_file(dst) end end @@ -327,16 +300,8 @@ class Protocol < ApplicationRecord # "Shallow" Copy assets step.assets.each do |asset| - asset2 = Asset.new_empty( - asset.file_file_name, - asset.file_file_size - ) - asset2.created_by = current_user - asset2.team = dest.team - asset2.last_modified_by = current_user - asset2.file_processing = true if asset.image? + asset2 = asset.dup asset2.save! - step2.assets << asset2 assets_to_clone << [asset.id, asset2.id] end @@ -357,10 +322,7 @@ class Protocol < ApplicationRecord step.clone_tinymce_assets(step2, dest.team) end # Call clone helper - Protocol.delay(queue: :assets).deep_clone_assets( - assets_to_clone, - dest.team - ) + Protocol.delay(queue: :assets).deep_clone_assets(assets_to_clone) end def in_repository_active? diff --git a/app/models/repository_asset_value.rb b/app/models/repository_asset_value.rb index c4c5f9594..1b175ac12 100644 --- a/app/models/repository_asset_value.rb +++ b/app/models/repository_asset_value.rb @@ -21,11 +21,11 @@ class RepositoryAssetValue < ApplicationRecord SORTABLE_VALUE_INCLUDE = { repository_asset_value: :asset }.freeze def formatted - asset.file_file_name + asset.file_name end def data - asset.file_file_name + asset.file_name end def data_changed?(_new_data) @@ -33,9 +33,8 @@ class RepositoryAssetValue < ApplicationRecord end def update_data!(new_data, user) - file = Paperclip.io_adapters.for(new_data[:file_data]) file.original_filename = new_data[:file_name] - asset.file = file + asset.file.attach(io: new_data[:file_data], filename: new_data[:file_name]) asset.last_modified_by = user self.last_modified_by = user asset.save! && save! diff --git a/app/models/team.rb b/app/models/team.rb index 5fcbbdc70..ef328dd6b 100644 --- a/app/models/team.rb +++ b/app/models/team.rb @@ -71,124 +71,6 @@ class Team < ApplicationRecord .where('full_name ILIKE ? OR email ILIKE ?', a_query, a_query) end - # Imports samples into db - # -1 == sample_name, - # -2 == sample_type, - # -3 == sample_group - # TODO: use constants - def import_samples(sheet, mappings, user) - errors = false - nr_of_added = 0 - total_nr = 0 - header_skipped = false - - # First let's query for all custom_fields we're refering to - custom_fields = [] - sname_index = -1 - stype_index = -1 - sgroup_index = -1 - mappings.each.with_index do |(_, v), i| - if v == '-1' - # Fill blank space, so our indices stay the same - custom_fields << nil - sname_index = i - elsif v == '-2' - custom_fields << nil - stype_index = i - elsif v == '-3' - custom_fields << nil - sgroup_index = i - else - cf = CustomField.find_by_id(v) - - # Even if doesn't exist we add nil value in order not to destroy our - # indices - custom_fields << cf - end - end - - rows = SpreadsheetParser.spreadsheet_enumerator(sheet) - - # Now we can iterate through sample data and save stuff into db - rows.each do |row| - # Skip empty rows - next if row.empty? - unless header_skipped - header_skipped = true - next - end - total_nr += 1 - row = SpreadsheetParser.parse_row(row, sheet) - - sample = Sample.new(name: row[sname_index], - team: self, - user: user) - - sample.transaction do - unless sample.valid? - errors = true - raise ActiveRecord::Rollback - end - - row.each.with_index do |value, index| - next unless value.present? - if index == stype_index - stype = SampleType.where(team: self) - .where('name ILIKE ?', value.strip) - .take - - unless stype - stype = SampleType.new(name: value.strip, team: self) - unless stype.save - errors = true - raise ActiveRecord::Rollback - end - end - sample.sample_type = stype - elsif index == sgroup_index - sgroup = SampleGroup.where(team: self) - .where('name ILIKE ?', value.strip) - .take - - unless sgroup - sgroup = SampleGroup.new(name: value.strip, team: self) - unless sgroup.save - errors = true - raise ActiveRecord::Rollback - end - end - sample.sample_group = sgroup - elsif value && custom_fields[index] - # we're working with CustomField - scf = SampleCustomField.new( - sample: sample, - custom_field: custom_fields[index], - value: value - ) - unless scf.valid? - errors = true - raise ActiveRecord::Rollback - end - sample.sample_custom_fields << scf - end - end - if Sample.import([sample], - recursive: true, - validate: false).failed_instances.any? - errors = true - raise ActiveRecord::Rollback - end - nr_of_added += 1 - end - end - - if errors - return { status: :error, nr_of_added: nr_of_added, total_nr: total_nr } - else - return { status: :ok, nr_of_added: nr_of_added, total_nr: total_nr } - end - end - def to_csv(samples, headers) require "csv" @@ -335,14 +217,16 @@ class Team < ApplicationRecord end def self.find_by_object(obj) - case obj.class.name - when 'Protocol' - obj.team_id - when 'MyModule', 'Step' - obj.protocol.team_id - when 'ResultText' - obj.result.my_module.protocol.team_id - end + find( + case obj.class.name + when 'Protocol' + obj.team_id + when 'MyModule', 'Step' + obj.protocol.team_id + when 'ResultText' + obj.result.my_module.protocol.team_id + end + ) end private diff --git a/app/models/team_zip_export.rb b/app/models/team_zip_export.rb index 742ab5294..b741365a7 100644 --- a/app/models/team_zip_export.rb +++ b/app/models/team_zip_export.rb @@ -147,7 +147,7 @@ class TeamZipExport < ZipExport .routes .url_helpers .zip_exports_download_export_all_path(self)}'>" \ - "#{zip_file_file_name}" + "#{zip_file_name}" ) UserNotification.create(notification: notification, user: user) end @@ -197,11 +197,9 @@ class TeamZipExport < ZipExport if type == :step name = "#{directory}/" \ - "#{append_file_suffix(asset.file_file_name, - "_#{i}_Step#{element.step.position_plus_one}")}" + "#{append_file_suffix(asset.file_name, "_#{i}_Step#{element.step.position_plus_one}")}" elsif type == :result - name = "#{directory}/#{append_file_suffix(asset.file_file_name, - "_#{i}")}" + name = "#{directory}/#{append_file_suffix(asset.file_name, "_#{i}")}" end asset.file.copy_to_local_file(:original, name) if asset.file.exists? asset_indexes[asset.id] = name @@ -254,8 +252,7 @@ class TeamZipExport < ZipExport assets = {} asset_counter = 0 handle_name_func = lambda do |asset| - file_name = append_file_suffix(asset.file_file_name, - "_#{asset_counter}").to_s + file_name = append_file_suffix(asset.file_name, "_#{asset_counter}").to_s # Save pair for downloading it later assets[asset] = "#{attach_path}/#{file_name}" diff --git a/app/models/temp_file.rb b/app/models/temp_file.rb index 134dfa830..8ed198c35 100644 --- a/app/models/temp_file.rb +++ b/app/models/temp_file.rb @@ -3,8 +3,7 @@ class TempFile < ApplicationRecord validates :session_id, presence: true - has_attached_file :file - do_not_validate_attachment_file_type :file + has_one_attached :file class << self def destroy_obsolete(temp_file_id) diff --git a/app/models/tiny_mce_asset.rb b/app/models/tiny_mce_asset.rb index b0b9cba17..3e0d86db7 100644 --- a/app/models/tiny_mce_asset.rb +++ b/app/models/tiny_mce_asset.rb @@ -4,7 +4,7 @@ class TinyMceAsset < ApplicationRecord extend ProtocolsExporter attr_accessor :reference before_create :set_reference, optional: true - after_create :update_estimated_size, :self_destruct + after_create :calculate_estimated_size, :self_destruct after_destroy :release_team_space belongs_to :team, inverse_of: :tiny_mce_assets, optional: true @@ -22,20 +22,23 @@ class TinyMceAsset < ApplicationRecord belongs_to :object, polymorphic: true, optional: true, inverse_of: :tiny_mce_assets - has_attached_file :image, - styles: { large: [Constants::LARGE_PIC_FORMAT, :jpg] }, - convert_options: { large: '-quality 100 -strip' } - validates_attachment_content_type :image, - content_type: %r{^image/#{ Regexp.union( - Constants::WHITELISTED_IMAGE_TYPES - ) }} - validates_attachment :image, - presence: true, - size: { - less_than: Rails.configuration.x\ - .file_max_size_mb.megabytes - } + has_one_attached :image + + # has_attached_file :image, + # styles: { large: [Constants::LARGE_PIC_FORMAT, :jpg] }, + # convert_options: { large: '-quality 100 -strip' } + + # validates_attachment_content_type :image, + # content_type: %r{^image/#{ Regexp.union( + # Constants::WHITELISTED_IMAGE_TYPES + # ) }} + # validates_attachment :image, + # presence: true, + # size: { + # less_than: Rails.configuration.x\ + # .file_max_size_mb.megabytes + # } validates :estimated_size, presence: true def source @@ -67,13 +70,10 @@ class TinyMceAsset < ApplicationRecord tm_assets = description.css('img[data-mce-token]') tm_assets.each do |tm_asset| asset_id = tm_asset.attr('data-mce-token') - new_asset_url = find_by_id(Base62.decode(asset_id)) - next unless new_asset_url - - assets_source = new_asset_url.source - if assets_source - tm_asset.set_attribute('data-source-id', assets_source.id) - tm_asset.set_attribute('data-source-type', assets_source.class.name) + new_asset = obj.tiny_mce_assets.find_by_id(Base62.decode(asset_id)) + if new_asset + tm_asset.attributes['src'].value = Rails.application.routes.url_helpers.url_for(new_asset.image) + tm_asset['class'] = 'img-responsive' end tm_asset.attributes['src'].value = new_asset_url.url tm_asset['class'] = 'img-responsive' @@ -81,47 +81,51 @@ class TinyMceAsset < ApplicationRecord description.css('body').inner_html.to_s end - def presigned_url(style = :large, - download: false, - timeout: Constants::URL_LONG_EXPIRE_TIME) - if stored_on_s3? - download_arg = ('attachment; filename=' + CGI.escape(image_file_name) if download) + def file_name + return '' unless image.attached? - signer = Aws::S3::Presigner.new(client: S3_BUCKET.client) - signer.presigned_url(:get_object, - bucket: S3_BUCKET.name, - key: image.path(style)[1..-1], - expires_in: timeout, - response_content_disposition: download_arg) - end + image.blob&.filename&.to_s end - def stored_on_s3? - image.options[:storage].to_sym == :s3 + def file_size + return 0 unless image.attached? + + image.blob&.byte_size end - def url(style = :large, timeout: Constants::URL_LONG_EXPIRE_TIME) - if image.is_stored_on_s3? - presigned_url(style, timeout: timeout) - else - image.url(style) - end + def content_type + image&.blob&.content_type end - def open - if image.is_stored_on_s3? - Kernel.open(presigned_url, 'rb') - else - File.open(image.path, 'rb') - end + def file_size + image&.blob&.byte_size + end + + def preview + image.variant(resize: Constants::LARGE_PIC_FORMAT) end def self.delete_unsaved_image(id) asset = find_by_id(id) asset.destroy if asset && !asset.saved end + + def self.update_estimated_size(id) + asset = find_by_id(id) + return unless asset&.image&.attached? - def self.update_old_tinymce(description, obj = nil, import = false) + size = asset.image.blob.byte_size + return if size.blank? + + e_size = size * Constants::ASSET_ESTIMATED_SIZE_FACTOR + asset.update(estimated_size: e_size) + Rails.logger.info "Asset #{id}: Estimated size successfully calculated" + # update team space taken + asset.team.take_space(e_size) + asset.team.save + end + + def self.update_old_tinymce(description, obj = nil) return description unless description description.scan(/\[~tiny_mce_id:(\w+)\]/).flatten.each do |token| @@ -148,12 +152,9 @@ class TinyMceAsset < ApplicationRecord if exists? order(:id).each do |tiny_mce_asset| asset_guid = get_guid(tiny_mce_asset.id) - asset_file_name = - "rte-#{asset_guid.to_s + - File.extname(tiny_mce_asset.image_file_name).to_s}" + asset_file_name = "rte-#{asset_guid.to_s + File.extname(tiny_mce_asset.image.filename.to_s)}" ostream.put_next_entry("#{dir}/#{asset_file_name}") - input_file = tiny_mce_asset.open - ostream.print(input_file.read) + ostream.print(tiny_mce_asset.image.download) input_file.close end end @@ -176,12 +177,17 @@ class TinyMceAsset < ApplicationRecord return false unless team_id tiny_img_clone = TinyMceAsset.new( - image: image, estimated_size: estimated_size, object: obj, team_id: team_id ) - tiny_img_clone.save! + + tiny_img_clone.transaction do + tiny_img_clone.save! + tiny_img_clone.image.attach(io: image.download, filename: image.filename.to_s) + end + + return false unless tiny_img_clone.persisted? obj.tiny_mce_assets << tiny_img_clone # Prepare array of image to update @@ -202,15 +208,8 @@ class TinyMceAsset < ApplicationRecord TinyMceAsset.delay(queue: :assets, run_at: 1.days.from_now).delete_unsaved_image(id) end - def update_estimated_size - return if image_file_size.blank? - - es = image_file_size * Constants::ASSET_ESTIMATED_SIZE_FACTOR - update(estimated_size: es) - Rails.logger.info "Asset #{id}: Estimated size successfully calculated" - # update team space taken - team.take_space(es) - team.save + def calculate_estimated_size + TinyMceAsset.delay(queue: :assets, run_at: 5.minutes.from_now).update_estimated_size(id) end def release_team_space diff --git a/app/models/user.rb b/app/models/user.rb index 6a46a7f47..15872723d 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -15,14 +15,17 @@ class User < ApplicationRecord :timeoutable, :omniauthable, omniauth_providers: Extends::OMNIAUTH_PROVIDERS, stretches: Constants::PASSWORD_STRETCH_FACTOR - has_attached_file :avatar, - styles: { - medium: Constants::MEDIUM_PIC_FORMAT, - thumb: Constants::THUMB_PIC_FORMAT, - icon: Constants::ICON_PIC_FORMAT, - icon_small: Constants::ICON_SMALL_PIC_FORMAT - }, - default_url: Constants::DEFAULT_AVATAR_URL + + has_one_attached :avatar + + # has_attached_file :avatar, + # styles: { + # medium: Constants::MEDIUM_PIC_FORMAT, + # thumb: Constants::THUMB_PIC_FORMAT, + # icon: Constants::ICON_PIC_FORMAT, + # icon_small: Constants::ICON_SMALL_PIC_FORMAT + # }, + # default_url: Constants::DEFAULT_AVATAR_URL auto_strip_attributes :full_name, :initials, nullify: false validates :full_name, @@ -35,10 +38,10 @@ class User < ApplicationRecord presence: true, length: { maximum: Constants::EMAIL_MAX_LENGTH } - validates_attachment :avatar, - :content_type => { :content_type => ["image/jpeg", "image/png"] }, - size: { less_than: Constants::AVATAR_MAX_SIZE_MB.megabyte, - message: I18n.t('client_api.user.avatar_too_big') } + # validates_attachment :avatar, + # :content_type => { :content_type => ["image/jpeg", "image/png"] }, + # size: { less_than: Constants::AVATAR_MAX_SIZE_MB.megabyte, + # message: I18n.t('client_api.user.avatar_too_big') } validate :time_zone_check store_accessor :settings, :time_zone, :notifications_settings @@ -226,11 +229,6 @@ class User < ApplicationRecord foreign_key: :resource_owner_id, dependent: :delete_all - # If other errors besides parameter "avatar" exist, - # they will propagate to "avatar" also, so remove them - # and put all other (more specific ones) in it - after_validation :filter_paperclip_errors - before_destroy :destroy_notifications def name @@ -249,6 +247,26 @@ class User < ApplicationRecord @avatar_remote_url = url_value end + def avatar_variant(style) + format = case style.to_sym + when :medium + Constants::MEDIUM_PIC_FORMAT + when :thumb + Constants::THUMB_PIC_FORMAT + when :icon + Constants::ICON_PIC_FORMAT + when :icon_small + Constants::ICON_SMALL_PIC_FORMAT + else + Constants::ICON_SMALL_PIC_FORMAT + end + avatar.variant(resize: format) + end + + def avatar_url(style) + Rails.application.routes.url_helpers.url_for(avatar_variant(style)) + end + def date_format settings[:date_format] || Constants::DEFAULT_DATE_FORMAT end @@ -311,20 +329,6 @@ class User < ApplicationRecord self.avatar_file_size = size.to_i end - def filter_paperclip_errors - if errors.key? :avatar - errors.delete(:avatar) - messages = [] - errors.each do |attribute| - errors.full_messages_for(attribute).each do |message| - messages << message.split(' ').drop(1).join(' ') - end - end - errors.clear - errors.add(:avatar, messages.join(',')) - end - end - # Whether user is active (= confirmed) or not def active? confirmed_at.present? diff --git a/app/models/zip_export.rb b/app/models/zip_export.rb index c44590437..6706662a8 100644 --- a/app/models/zip_export.rb +++ b/app/models/zip_export.rb @@ -22,71 +22,51 @@ class ZipExport < ApplicationRecord belongs_to :user, optional: true # Override path only for S3 - if ENV['PAPERCLIP_STORAGE'] == 's3' - s3_path = - if ENV['S3_SUBFOLDER'] - "/#{ENV['S3_SUBFOLDER']}/zip_exports/:attachment/"\ - ":id_partition/:hash/:style/:filename" - else - '/zip_exports/:attachment/:id_partition/:hash/:style/:filename' - end + # if ENV['PAPERCLIP_STORAGE'] == 's3' + # s3_path = + # if ENV['S3_SUBFOLDER'] + # "/#{ENV['S3_SUBFOLDER']}/zip_exports/:attachment/"\ + # ":id_partition/:hash/:style/:filename" + # else + # '/zip_exports/:attachment/:id_partition/:hash/:style/:filename' + # end + # + # has_attached_file :zip_file, path: s3_path + # else + # has_attached_file :zip_file + # end - has_attached_file :zip_file, path: s3_path - else - has_attached_file :zip_file - end + has_one_attached :zip_file - validates_attachment :zip_file, - content_type: { content_type: 'application/zip' } + # validates_attachment :zip_file, + # content_type: { content_type: 'application/zip' } after_create :self_destruct - # When using S3 file upload, we can limit file accessibility with url signing - def presigned_url(style = :original, - download: false, - timeout: Constants::URL_SHORT_EXPIRE_TIME) - if stored_on_s3? - if download - download_arg = 'attachment; filename=' + URI.escape(zip_file_file_name) - else - download_arg = nil - end - - signer = Aws::S3::Presigner.new(client: S3_BUCKET.client) - signer.presigned_url(:get_object, - bucket: S3_BUCKET.name, - key: zip_file.path(style)[1..-1], - expires_in: timeout, - response_content_disposition: download_arg) - end - end - - def stored_on_s3? - zip_file.options[:storage].to_sym == :s3 - end - def self.delete_expired_export(id) export = find_by_id(id) - export.destroy if export + export&.destroy + end + + def zip_file_name + return '' unless file.attached? + + zip_file.blob&.filename&.to_s end def generate_exportable_zip(user, data, type, options = {}) - I18n.backend.date_format = - user.settings[:date_format] || Constants::DEFAULT_DATE_FORMAT - zip_input_dir = FileUtils.mkdir_p( - File.join(Rails.root, "tmp/temp_zip_#{Time.now.to_i}") - ).first - zip_dir = FileUtils.mkdir_p(File.join(Rails.root, 'tmp/zip-ready')).first - zip_file = File.new( - File.join(zip_dir, "export_#{Time.now.strftime('%F %H-%M-%S_UTC')}.zip"), - 'w+' - ) + I18n.backend.date_format = user.settings[:date_format] || Constants::DEFAULT_DATE_FORMAT + zip_input_dir = FileUtils.mkdir_p(File.join(Rails.root, "tmp/temp_zip_#{Time.now.to_i}")).first + tmp_zip_dir = FileUtils.mkdir_p(File.join(Rails.root, 'tmp/zip-ready')).first + tmp_zip_name = "export_#{Time.now.strftime('%F %H-%M-%S_UTC')}.zip" + tmp_zip_file = File.new(File.join(tmp_zip_dir, tmp_zip_name), 'w+') + fill_content(zip_input_dir, data, type, options) - zip!(zip_input_dir, zip_file) - self.zip_file = File.open(zip_file) + zip!(zip_input_dir, tmp_zip_file) + zip_file.attach(io: File.open(tmp_zip_file), filename: tmp_zip_name) generate_notification(user) if save ensure - FileUtils.rm_rf([zip_input_dir, zip_file], secure: true) + FileUtils.rm_rf([zip_input_dir, tmp_zip_file], secure: true) end handle_asynchronously :generate_exportable_zip @@ -122,7 +102,7 @@ class ZipExport < ApplicationRecord .routes .url_helpers .zip_exports_download_path(self)}'>" \ - "#{zip_file_file_name}" + "#{zip_file_name}" ) UserNotification.create(notification: notification, user: user) end diff --git a/app/serializers/api/v1/repository_asset_value_serializer.rb b/app/serializers/api/v1/repository_asset_value_serializer.rb index c6099a3fd..a64c49532 100644 --- a/app/serializers/api/v1/repository_asset_value_serializer.rb +++ b/app/serializers/api/v1/repository_asset_value_serializer.rb @@ -10,20 +10,18 @@ module Api end def file_name - object.asset&.file_file_name + object.asset&.file_name end def file_size - object.asset&.file_file_size + object.asset&.file_size end def url if !object.asset&.file&.exists? nil - elsif object.asset&.file&.is_stored_on_s3? - object.asset.presigned_url(download: true) else - object.asset.file.url + rails_blob_path(object.asset.file, disposition: 'attachment') end end end diff --git a/app/serializers/api/v1/result_asset_serializer.rb b/app/serializers/api/v1/result_asset_serializer.rb index 0e31473a0..d86e999f2 100644 --- a/app/serializers/api/v1/result_asset_serializer.rb +++ b/app/serializers/api/v1/result_asset_serializer.rb @@ -11,20 +11,18 @@ module Api end def file_name - object.asset&.file_file_name + object.asset&.file_name end def file_size - object.asset&.file_file_size + object.asset&.file_size end def url if !object.asset&.file&.exists? nil - elsif object.asset&.file&.is_stored_on_s3? - object.asset.presigned_url(download: true) else - object.asset.file.url + rails_blob_path(object.asset.file, disposition: 'attachment') end end end diff --git a/app/services/experiments/generate_workflow_image_service.rb b/app/services/experiments/generate_workflow_image_service.rb index 725f67be8..39a69324f 100644 --- a/app/services/experiments/generate_workflow_image_service.rb +++ b/app/services/experiments/generate_workflow_image_service.rb @@ -76,9 +76,9 @@ module Experiments def save_file file = Tempfile.open(%w(wimg .png), Rails.root.join('tmp')) @graph.output(png: file.path) - @exp.workflowimg = file + @exp.workflowimg.attach(io: file, filename: File.basename(file.path)) file.close - @exp.save + file.unlink @exp.touch(:workflowimg_updated_at) end end diff --git a/app/services/experiments/move_to_project_service.rb b/app/services/experiments/move_to_project_service.rb index 0603a55ae..6a0837926 100644 --- a/app/services/experiments/move_to_project_service.rb +++ b/app/services/experiments/move_to_project_service.rb @@ -29,8 +29,6 @@ module Experiments end raise ActiveRecord::Rollback unless @exp.save - # To pass the ExperimentsController#updated_img check - @exp.update(workflowimg_updated_at: @exp.updated_at) end @errors.merge!(@exp.errors.to_hash) unless @exp.valid? diff --git a/app/services/repository_actions/duplicate_cell.rb b/app/services/repository_actions/duplicate_cell.rb index 893cc7064..6a2d51bf1 100644 --- a/app/services/repository_actions/duplicate_cell.rb +++ b/app/services/repository_actions/duplicate_cell.rb @@ -43,7 +43,7 @@ module RepositoryActions def duplicate_repository_asset_value old_value = @cell.value - new_asset = create_new_asset(old_value.asset) + new_asset = old_value.asset.duplicate RepositoryAssetValue.create( old_value.attributes.merge( id: nil, asset: new_asset, created_by: @user, last_modified_by: @user, @@ -67,30 +67,5 @@ module RepositoryActions ) ) end - - # reuses the same code we have in copy protocols action - def create_new_asset(old_asset) - new_asset = Asset.new_empty( - old_asset.file_file_name, - old_asset.file_file_size - ) - new_asset.created_by = old_asset.created_by - new_asset.team = @team - new_asset.last_modified_by = @user - new_asset.file_processing = true if old_asset.image? - new_asset.file = old_asset.file - new_asset.save - - return unless new_asset.valid? - - if new_asset.image? - new_asset.file.reprocess!(:large) - new_asset.file.reprocess!(:medium) - end - - new_asset.post_process_file(new_asset.team) - - new_asset - end end end diff --git a/app/services/repository_datatable_service.rb b/app/services/repository_datatable_service.rb index 03a861beb..1871c3ed4 100644 --- a/app/services/repository_datatable_service.rb +++ b/app/services/repository_datatable_service.rb @@ -163,4 +163,51 @@ class RepositoryDatatableService def sort_null_direction(val) val == 'ASC' ? 'LAST' : 'FIRST' end + + def filter_by_asset_value(records, id, dir) + records.joins( + "LEFT OUTER JOIN (SELECT repository_cells.repository_row_id, + active_storage_blobs.filename AS value + FROM repository_cells + INNER JOIN repository_asset_values + ON repository_asset_values.id = repository_cells.value_id + INNER JOIN assets + ON repository_asset_values.asset_id = assets.id + INNER JOIN active_storage_attachments + ON active_storage_attachments.record_id = assets.id + AND active_storage_attachments.record_type = 'Asset' + AND active_storage_attachments.name = 'file' + INNER JOIN active_storage_blobs + ON active_storage_blobs.id = active_storage_attachments.blob_id + WHERE repository_cells.repository_column_id = #{id}) AS values + ON values.repository_row_id = repository_rows.id" + ).order("values.value #{dir}") + end + + def filter_by_text_value(records, id, dir) + records.joins( + "LEFT OUTER JOIN (SELECT repository_cells.repository_row_id, + repository_text_values.data AS value + FROM repository_cells + INNER JOIN repository_text_values + ON repository_text_values.id = repository_cells.value_id + WHERE repository_cells.repository_column_id = #{id}) AS values + ON values.repository_row_id = repository_rows.id" + ).order("values.value #{dir}") + end + + def filter_by_list_value(records, id, dir) + records.joins( + "LEFT OUTER JOIN (SELECT repository_cells.repository_row_id, + repository_list_items.data AS value + FROM repository_cells + INNER JOIN repository_list_values + ON repository_list_values.id = repository_cells.value_id + INNER JOIN repository_list_items + ON repository_list_values.repository_list_item_id = + repository_list_items.id + WHERE repository_cells.repository_column_id = #{id}) AS values + ON values.repository_row_id = repository_rows.id" + ).order("values.value #{dir}") + end end diff --git a/app/services/spreadsheet_parser.rb b/app/services/spreadsheet_parser.rb index ab2c54235..ef36175f4 100644 --- a/app/services/spreadsheet_parser.rb +++ b/app/services/spreadsheet_parser.rb @@ -4,11 +4,6 @@ class SpreadsheetParser filename = file.original_filename file_path = file.path - if file.class == Paperclip::Attachment && file.is_stored_on_s3? - fa = file.fetch - file_path = fa.path - end - case File.extname(filename) when '.csv' Roo::CSV.new(file_path, extension: :csv) diff --git a/app/services/team_importer.rb b/app/services/team_importer.rb index 74ed1dfcb..ecb3c7dc1 100644 --- a/app/services/team_importer.rb +++ b/app/services/team_importer.rb @@ -340,8 +340,8 @@ class TeamImporter tiny_mce_asset.object_id = mappings[tiny_mce_asset.object_id] end tiny_mce_asset.team = team - tiny_mce_asset.image = tiny_mce_file tiny_mce_asset.save! + tiny_mce_asset.image.attach(io: tiny_mce_file, filename: tiny_mce_file.basename) @mce_asset_counter += 1 if tiny_mce_asset.object_id.present? object = tiny_mce_asset.object @@ -794,7 +794,7 @@ class TeamImporter def create_asset(asset_json, team, user_id = nil) asset = Asset.new(asset_json) File.open( - "#{@import_dir}/assets/#{asset.id}/#{asset.file_file_name}" + "#{@import_dir}/assets/#{asset.id}/#{asset.file_name}" ) do |file| orig_asset_id = asset.id asset.id = nil @@ -802,9 +802,9 @@ class TeamImporter asset.last_modified_by_id = user_id || find_user(asset.last_modified_by_id) asset.team = team - asset.file = file asset.in_template = true if @is_template asset.save! + asset.file.attach(io: file, filename: file.basename) asset.post_process_file(team) @asset_mappings[orig_asset_id] = asset.id @asset_counter += 1 diff --git a/app/utilities/protocols_exporter.rb b/app/utilities/protocols_exporter.rb index cb402e2dc..f63d31246 100644 --- a/app/utilities/protocols_exporter.rb +++ b/app/utilities/protocols_exporter.rb @@ -51,12 +51,11 @@ module ProtocolsExporter next unless img img_guid = get_guid(img.id) - asset_file_name = "rte-#{img_guid}" \ - "#{File.extname(img.image_file_name)}" + asset_file_name = "rte-#{img_guid}#{File.extname(img.file_name)}" asset_xml = "\n" - asset_xml << "#{img.image_file_name}\n" - asset_xml << "#{img.image_content_type}\n" + asset_xml << "#{img.file_name}\n" + asset_xml << "#{img.content_type}\n" asset_xml << "\n" tiny_assets_xml << asset_xml end @@ -100,11 +99,11 @@ module ProtocolsExporter step.assets.order(:id).each do |asset| asset_guid = get_guid(asset.id) asset_file_name = "#{asset_guid}" \ - "#{File.extname(asset.file_file_name)}" + "#{File.extname(asset.file_name)}" asset_xml = "\n" - asset_xml << "#{asset.file_file_name}\n" - asset_xml << "#{asset.file_content_type}\n" + asset_xml << "#{asset.file_name}\n" + asset_xml << "#{asset.content_type}\n" asset_xml << "\n" step_xml << asset_xml end diff --git a/app/utilities/protocols_importer.rb b/app/utilities/protocols_importer.rb index ce7b48c03..13d0f1940 100644 --- a/app/utilities/protocols_importer.rb +++ b/app/utilities/protocols_importer.rb @@ -111,9 +111,7 @@ module ProtocolsImporter ) # Decode the file bytes - asset.file = StringIO.new(Base64.decode64(asset_json['bytes'])) - asset.file_file_name = asset_json['fileName'] - asset.file_content_type = asset_json['fileType'] + asset.file.attach(io: StringIO.new(Base64.decode64(asset_json['bytes'])), filename: asset_json['fileName']) asset.save! asset_ids << asset.id @@ -152,12 +150,11 @@ module ProtocolsImporter team_id: team.id, saved: true ) - # Decode the file bytes - tiny_mce_img.image = StringIO.new( - Base64.decode64(tiny_mce_img_json['bytes']) - ) - tiny_mce_img.image_content_type = tiny_mce_img_json['fileType'] tiny_mce_img.save! + + # Decode the file bytes + tiny_mce_img.image.attach(io: StringIO.new(Base64.decode64(tiny_mce_img_json['bytes'])), + filename: tiny_mce_img_json['fileName']) if description.gsub!("data-mce-token=\"#{tiny_mce_img_json['tokenId']}\"", "data-mce-token=\"#{Base62.encode(tiny_mce_img.id)}\"") else diff --git a/app/utilities/wopi_util.rb b/app/utilities/wopi_util.rb index eabeaa693..2ed5ebb9e 100644 --- a/app/utilities/wopi_util.rb +++ b/app/utilities/wopi_util.rb @@ -83,7 +83,7 @@ module WopiUtil default_step_items = { step: @asset.step.id, step_position: { id: @asset.step.id, value_for: 'position_plus_one' }, - asset_name: { id: @asset.id, value_for: 'file_file_name' }, + asset_name: { id: @asset.id, value_for: 'file_name' }, action: action } if @protocol.in_module? project = @protocol.my_module.experiment.project @@ -113,7 +113,7 @@ module WopiUtil project: @my_module.experiment.project, message_items: { result: @asset.result.id, - asset_name: { id: @asset.id, value_for: 'file_file_name' }, + asset_name: { id: @asset.id, value_for: 'file_name' }, action: action }) end diff --git a/app/views/assets/edit.erb b/app/views/assets/edit.erb index 17e71dc25..f4617301f 100644 --- a/app/views/assets/edit.erb +++ b/app/views/assets/edit.erb @@ -6,7 +6,7 @@ - <%= t('assets.head_title.edit', file_name: @asset.file_file_name) %> + <%= t('assets.head_title.edit', file_name: @asset.file_name) %> diff --git a/app/views/assets/view.erb b/app/views/assets/view.erb index e20aa03ff..1bd84e2a2 100644 --- a/app/views/assets/view.erb +++ b/app/views/assets/view.erb @@ -6,7 +6,7 @@ - <%= t('assets.head_title.view', file_name: @asset.file_file_name) %> + <%= t('assets.head_title.view', file_name: @asset.file_name) %> diff --git a/app/views/projects/show/_workflow_img.html.erb b/app/views/projects/show/_workflow_img.html.erb index f75e948ea..55152afa6 100644 --- a/app/views/projects/show/_workflow_img.html.erb +++ b/app/views/projects/show/_workflow_img.html.erb @@ -1,4 +1 @@ -<%= image_tag(@experiment.workflowimg.expiring_url( - Constants::URL_SHORT_EXPIRE_TIME - ), - class: 'img-responsive center-block') %> +<%= image_tag url_for(@experiment.workflowimg), class: 'img-responsive center-block' %> diff --git a/app/views/reports/elements/_my_module_result_asset_element.html.erb b/app/views/reports/elements/_my_module_result_asset_element.html.erb index 3cbb13b6e..e934d8adf 100644 --- a/app/views/reports/elements/_my_module_result_asset_element.html.erb +++ b/app/views/reports/elements/_my_module_result_asset_element.html.erb @@ -24,7 +24,7 @@ <% else %> <%=t "projects.reports.elements.result_asset.file_name", - file: truncate(asset.file_file_name, + file: truncate(asset.file_name, length: Constants::FILENAME_TRUNCATION_LENGTH) %> <% end %>
diff --git a/app/views/reports/elements/_step_asset_element.html.erb b/app/views/reports/elements/_step_asset_element.html.erb index b84d9a61b..ef768927b 100644 --- a/app/views/reports/elements/_step_asset_element.html.erb +++ b/app/views/reports/elements/_step_asset_element.html.erb @@ -2,7 +2,7 @@ <% is_image = asset.image? %> <% timestamp = asset.created_at %> <% icon_class = 'fas ' + (is_image ? 'fa-image' : 'fa-file') %> -
" data-icon-class="<%= icon_class %>"> +
" data-icon-class="<%= icon_class %>">
@@ -16,7 +16,7 @@ <% else %> <%=t 'projects.reports.elements.step_asset.file_name', - file: truncate(asset.file_file_name, + file: truncate(asset.file_name, length: Constants::FILENAME_TRUNCATION_LENGTH) %> <% end %>
diff --git a/app/views/result_assets/_edit.html.erb b/app/views/result_assets/_edit.html.erb index e35b55239..7d1bc574d 100644 --- a/app/views/result_assets/_edit.html.erb +++ b/app/views/result_assets/_edit.html.erb @@ -5,7 +5,7 @@ <%=t "result_assets.edit.uploaded_asset" %>

<%= file_extension_icon(ff.object) %> - <%= ff.object.file_file_name %> + <%= ff.object.file_name %>

<%= ff.file_field :file %> <% end %> diff --git a/app/views/search/results/partials/_asset_text.html.erb b/app/views/search/results/partials/_asset_text.html.erb index f5a483393..04de9d822 100644 --- a/app/views/search/results/partials/_asset_text.html.erb +++ b/app/views/search/results/partials/_asset_text.html.erb @@ -1,6 +1,6 @@ <% query ||= nil %> <% asset_read_allowed = false %> -<% text = query.present? ? highlight(asset.file_file_name, query.strip.split(/\s+/)) : asset.file_file_name %> +<% text = query.present? ? highlight(asset.file_name, query.strip.split(/\s+/)) : asset.file_name %> <% if asset.step %> <% protocol = asset.step.protocol %> diff --git a/app/views/shared/_asset_link.html.erb b/app/views/shared/_asset_link.html.erb index e76ad65c0..237f6a7b1 100644 --- a/app/views/shared/_asset_link.html.erb +++ b/app/views/shared/_asset_link.html.erb @@ -2,19 +2,15 @@ class: 'file-preview-link', id: "modal_link#{asset.id}", data: { no_turbolink: true, id: true, status: 'asset-present', 'preview-url': asset_file_preview_path(asset) } do %> - <% if asset.image? && display_image_tag %> + <% if display_image_tag && asset.previewable? %> <%= image_tag asset.medium_preview %> - <% end %> - <% if display_image_tag %>

- <%= truncate(asset.file_file_name, - length: Constants::FILENAME_TRUNCATION_LENGTH) %> + <%= truncate(asset.file_name, length: Constants::FILENAME_TRUNCATION_LENGTH) %>

- <% else %> - - <%= truncate(asset.file_file_name, - length: Constants::FILENAME_TRUNCATION_LENGTH) %> - + <% else %> + + <%= truncate(asset.file_name, length: Constants::FILENAME_TRUNCATION_LENGTH) %> + <% end %> <% end %> diff --git a/app/views/shared/_file_preview_icon.html.erb b/app/views/shared/_file_preview_icon.html.erb index 45c362022..14b2ef46c 100644 --- a/app/views/shared/_file_preview_icon.html.erb +++ b/app/views/shared/_file_preview_icon.html.erb @@ -1,5 +1,5 @@
- <% if asset.file.previewable_document? %> + <% if asset.previewable? %> <%= image_tag(asset.large_preview) %> <% else %> diff --git a/app/views/steps/attachments/_placeholder.html.erb b/app/views/steps/attachments/_placeholder.html.erb index a07e47d10..56c7acf81 100644 --- a/app/views/steps/attachments/_placeholder.html.erb +++ b/app/views/steps/attachments/_placeholder.html.erb @@ -22,5 +22,5 @@ <% end %> <% end %>
-<% end %> + <% end %>
diff --git a/app/views/users/registrations/edit.html.erb b/app/views/users/registrations/edit.html.erb index ef24e8ce8..9d9cc5464 100644 --- a/app/views/users/registrations/edit.html.erb +++ b/app/views/users/registrations/edit.html.erb @@ -239,4 +239,3 @@ <%= stylesheet_pack_tag 'custom/croppie_styles' %> <%= javascript_include_tag("canvas-to-blob.min") %> <%= javascript_include_tag "users/registrations/edit" %> - diff --git a/config/environments/production.rb b/config/environments/production.rb index 5df50d3fc..676fca7c2 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -48,7 +48,7 @@ Rails.application.configure do # Compress JavaScripts and CSS. - config.assets.js_compressor = :uglifier + config.assets.js_compressor = Uglifier.new(harmony: true) # config.assets.css_compressor = :sass # Do not fallback to assets pipeline if a precompiled asset is missed. @@ -71,6 +71,8 @@ Rails.application.configure do # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies. config.force_ssl = ENV['RAILS_FORCE_SSL'].present? + config.ssl_options = { redirect: { exclude: ->(request) { request.path =~ %r{api\/health} } } } + # Use the lowest log level to ensure availability of diagnostic information # when problems arise. config.log_level = :debug diff --git a/config/initializers/assets.rb b/config/initializers/assets.rb index c88708404..f926d84e1 100644 --- a/config/initializers/assets.rb +++ b/config/initializers/assets.rb @@ -76,7 +76,6 @@ Rails.application.config.assets.precompile += %w(canvas-to-blob.min.js) Rails.application.config.assets.precompile += %w(Sortable.min.js) Rails.application.config.assets.precompile += %w(reports_pdf.css) Rails.application.config.assets.precompile += %w(jszip.min.js) -Rails.application.config.assets.precompile += %w(assets.js) Rails.application.config.assets.precompile += %w(comments.js) Rails.application.config.assets.precompile += %w(projects/show.js) Rails.application.config.assets.precompile += %w(notifications.js) diff --git a/config/initializers/constants.rb b/config/initializers/constants.rb index 47da80f93..0c85255a2 100644 --- a/config/initializers/constants.rb +++ b/config/initializers/constants.rb @@ -93,9 +93,9 @@ class Constants # Picture size formats LARGE_PIC_FORMAT = '800x600>'.freeze MEDIUM_PIC_FORMAT = '300x300>'.freeze - THUMB_PIC_FORMAT = '100x100#'.freeze - ICON_PIC_FORMAT = '40x40#'.freeze - ICON_SMALL_PIC_FORMAT = '30x30#'.freeze + THUMB_PIC_FORMAT = '100x100'.freeze + ICON_PIC_FORMAT = '40x40'.freeze + ICON_SMALL_PIC_FORMAT = '30x30'.freeze # Hands-on-table number of starting columns and rows HANDSONTABLE_INIT_COLS_CNT = 5 diff --git a/config/initializers/extends.rb b/config/initializers/extends.rb index 3a85ede69..b448bbf10 100644 --- a/config/initializers/extends.rb +++ b/config/initializers/extends.rb @@ -55,12 +55,12 @@ class Extends # are only supported REPOSITORY_EXTRA_SEARCH_ATTR = ['repository_text_values.data', 'repository_list_items.data', - 'assets.file_file_name'] + 'active_storage_blobs.filename'] # Array of includes used in search query for repository rows REPOSITORY_SEARCH_INCLUDES = [:repository_text_value, repository_list_value: :repository_list_item, - repository_asset_value: :asset] + repository_asset_value: { asset: { file_attachment: :blob } }] # List of implemented core API versions API_VERSIONS = ['v1'] diff --git a/config/initializers/filter_parameter_logging.rb b/config/initializers/filter_parameter_logging.rb index ec91e2ccf..156ba453d 100644 --- a/config/initializers/filter_parameter_logging.rb +++ b/config/initializers/filter_parameter_logging.rb @@ -1,4 +1,4 @@ # Be sure to restart your server when you modify this file. # Configure sensitive parameters which will be filtered from the log file. -Rails.application.config.filter_parameters += %i(password image) +Rails.application.config.filter_parameters += [:password, :image, /^avatar$/] diff --git a/config/locales/en.yml b/config/locales/en.yml index 6ad5c8f53..fd6ca25a7 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -1265,7 +1265,6 @@ en: scinote_columns_html: "SciNote columns:" file_columns: "Imported columns:" example_value: "Imported file content:" - import_samples: "Import samples" do_not_include_column: "Do not include this column" errors: invalid_file: "The file you provided is invalid. Make sure the file is encoded using %{encoding}." diff --git a/config/routes.rb b/config/routes.rb index 7cb6816e4..5a8ff048e 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -192,8 +192,6 @@ Rails.application.routes.draw do # end member do post 'parse_sheet', defaults: { format: 'json' } - # post 'import_samples' - # post 'export_samples' post 'export_repository', to: 'repositories#export_repository' post 'export_projects' get 'export_projects_modal' @@ -582,7 +580,10 @@ Rails.application.routes.draw do # We cannot use 'resources :assets' because assets is a reserved route # in Rails (assets pipeline) and causes funky behavior +<<<<<<< HEAD get 'files/:id/present', to: 'assets#file_present', as: 'file_present_asset' +======= +>>>>>>> activestorage_migration get 'files/:id/preview', to: 'assets#file_preview', as: 'asset_file_preview' @@ -599,7 +600,6 @@ Rails.application.routes.draw do devise_scope :user do get 'avatar/:id/:style' => 'users/registrations#avatar', as: 'avatar' - post 'avatar_signature' => 'users/registrations#signature' get 'users/auth_token_sign_in' => 'users/sessions#auth_token_create' get 'users/sign_up_provider' => 'users/registrations#new_with_provider' post 'users/complete_sign_up_provider' => diff --git a/db/migrate/20190613134100_convert_to_active_storage.rb b/db/migrate/20190613134100_convert_to_active_storage.rb index 76593aa1b..0d651d7d4 100644 --- a/db/migrate/20190613134100_convert_to_active_storage.rb +++ b/db/migrate/20190613134100_convert_to_active_storage.rb @@ -99,8 +99,8 @@ class ConvertToActiveStorage < ActiveRecord::Migration[5.2] if ENV['PAPERCLIP_STORAGE'] == 's3' interpolate(':class/:attachment/:id_partition/:hash/original/:filename', instance, attachment) else - key = SecureRandom.uuid - File.join('storage', key.first(2), key.first(4).last(2)) + instance.class.generate_unique_secure_token + # File.join('storage', key.first(2), key.first(4).last(2)) end end diff --git a/docker-compose.production.yml b/docker-compose.production.yml index 0877637cb..f8dbfa157 100644 --- a/docker-compose.production.yml +++ b/docker-compose.production.yml @@ -37,7 +37,9 @@ services: - RAILS_ENV=production volumes: - scinote_production_files:/usr/src/app/public/system + - scinote_production_storage:/usr/src/app/storage volumes: scinote_production_postgres: scinote_production_files: + scinote_production_storage: diff --git a/docker-compose.yml b/docker-compose.yml index bfbdcf1a6..a89a1f446 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,6 +29,7 @@ services: - .:/usr/src/app - scinote_development_bundler:/usr/local/bundle/ - scinote_development_files:/usr/src/app/public/system + - scinote_development_storage:/usr/src/app/storage webpack: build: @@ -54,3 +55,4 @@ volumes: scinote_development_postgres: scinote_development_bundler: scinote_development_files: + scinote_development_storage: diff --git a/spec/services/templates_service_spec.rb b/spec/services/templates_service_spec.rb index 8274761c1..d7799b889 100644 --- a/spec/services/templates_service_spec.rb +++ b/spec/services/templates_service_spec.rb @@ -88,7 +88,6 @@ describe TemplatesService do .to eq(tmpl_step.step_comments.size) end end - Asset.all.each(&:paperclip_delete) FileUtils.remove_dir(tmplts_dir) end end