diff --git a/Gemfile b/Gemfile index 9784e8ff2..0aa585d97 100644 --- a/Gemfile +++ b/Gemfile @@ -62,9 +62,6 @@ gem 'bcrypt', '~> 3.1.10' gem 'caracal-rails' # Build docx report gem 'commit_param_routing' # Enables different submit actions in the same form gem 'deface', '~> 1.0' -gem 'delayed_paperclip', - git: 'https://github.com/jrgifford/delayed_paperclip.git', - ref: 'fcf574c' gem 'faker' # Generate fake data gem 'fastimage' # Light gem to get image resolution gem 'httparty', '~> 0.13.1' @@ -93,7 +90,6 @@ gem 'devise-async', git: 'https://github.com/mhfs/devise-async.git', branch: 'devise-4.x' gem 'image_processing', '~> 1.2' -gem 'paperclip', '~> 6.1' # File attachment, image attachment library gem 'rufus-scheduler', '~> 3.5' gem 'discard', '~> 1.0' diff --git a/Gemfile.lock b/Gemfile.lock index 1b8b6a7ea..8bdfc66ba 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -22,15 +22,6 @@ GIT sneaky-save (0.1.3) activerecord (>= 3.2.0) -GIT - remote: https://github.com/jrgifford/delayed_paperclip.git - revision: fcf574c1188213d3c8fce934fd52861a8ba080cb - ref: fcf574c - specs: - delayed_paperclip (3.0.1) - activejob (>= 4.2) - paperclip (>= 3.3) - GIT remote: https://github.com/mhfs/devise-async.git revision: 177f6363a002f7ff28f1d289c8cab7ad8d9cb8c5 @@ -186,7 +177,6 @@ GEM activesupport childprocess (1.0.1) rake (< 13.0) - climate_control (0.2.0) coderay (1.1.2) coffee-rails (5.0.0) coffee-script (>= 2.2.0) @@ -383,12 +373,6 @@ GEM overcommit (0.49.1) childprocess (>= 0.6.3, < 2.0) iniparse (~> 1.4) - paperclip (6.1.0) - activemodel (>= 4.2.0) - activesupport (>= 4.2.0) - mime-types - mimemagic (~> 0.3.0) - terrapin (~> 0.6.0) parallel (1.17.0) parser (2.6.4.0) ast (~> 2.4.0) @@ -556,8 +540,6 @@ GEM activesupport (>= 4.0) sprockets (>= 3.0.0) stream (0.5.2) - terrapin (0.6.0) - climate_control (>= 0.0.3, < 1.0) thor (0.20.3) thread_safe (0.3.6) tilt (2.0.9) @@ -628,7 +610,6 @@ DEPENDENCIES database_cleaner deface (~> 1.0) delayed_job_active_record - delayed_paperclip! devise (~> 4.7.1) devise-async! devise_invitable @@ -662,7 +643,6 @@ DEPENDENCIES omniauth-linkedin-oauth2 omniauth-rails_csrf_protection (~> 0.1) overcommit - paperclip (~> 6.1) pg (~> 1.1) pg_search pry diff --git a/app/models/asset.rb b/app/models/asset.rb index 1335624fb..785e8da99 100644 --- a/app/models/asset.rb +++ b/app/models/asset.rb @@ -15,47 +15,8 @@ class Asset < ApplicationRecord # ActiveStorage configuration has_one_attached :file - # Paperclip validation - # has_attached_file :file, - # styles: lambda { |a| - # if a.previewable_document? - # { - # large: { processors: [:custom_file_preview], - # geometry: Constants::LARGE_PIC_FORMAT, - # format: :jpg }, - # medium: { processors: [:custom_file_preview], - # geometry: Constants::MEDIUM_PIC_FORMAT, - # format: :jpg } - # } - # else - # { - # large: [Constants::LARGE_PIC_FORMAT, :jpg], - # medium: [Constants::MEDIUM_PIC_FORMAT, :jpg] - # } - # end - # }, - # convert_options: { - # medium: '-quality 70 -strip', - # all: '-background "#d2d2d2" -flatten +matte' - # } - - # before_post_process :previewable? # before_post_process :extract_image_quality - # adds image processing in background job - # process_in_background :file, processing_image_url: '/images/:style/processing.gif' - - # validates_attachment :file, - # presence: true, - # size: { - # less_than: Rails.configuration.x.file_max_size_mb.megabytes - # } - # validates :estimated_size, presence: true - # validates :file_present, inclusion: { in: [true, false] } - - # Should be checked for any security leaks - # do_not_validate_attachment_file_type :file - # Asset validation # This could cause some problems if you create empty asset and want to # assign it to result diff --git a/app/models/tiny_mce_asset.rb b/app/models/tiny_mce_asset.rb index e0b985113..1bee2d38a 100644 --- a/app/models/tiny_mce_asset.rb +++ b/app/models/tiny_mce_asset.rb @@ -18,20 +18,6 @@ class TinyMceAsset < ApplicationRecord has_one_attached :image - # has_attached_file :image, - # styles: { large: [Constants::LARGE_PIC_FORMAT, :jpg] }, - # convert_options: { large: '-quality 100 -strip' } - - # validates_attachment_content_type :image, - # content_type: %r{^image/#{ Regexp.union( - # Constants::WHITELISTED_IMAGE_TYPES - # ) }} - # validates_attachment :image, - # presence: true, - # size: { - # less_than: Rails.configuration.x\ - # .file_max_size_mb.megabytes - # } validates :estimated_size, presence: true def self.update_images(object, images, current_user) diff --git a/app/models/zip_export.rb b/app/models/zip_export.rb index 83e0f955f..b32df94d6 100644 --- a/app/models/zip_export.rb +++ b/app/models/zip_export.rb @@ -21,26 +21,8 @@ require 'csv' class ZipExport < ApplicationRecord belongs_to :user, optional: true - # Override path only for S3 - # if ENV['PAPERCLIP_STORAGE'] == 's3' - # s3_path = - # if ENV['S3_SUBFOLDER'] - # "/#{ENV['S3_SUBFOLDER']}/zip_exports/:attachment/"\ - # ":id_partition/:hash/:style/:filename" - # else - # '/zip_exports/:attachment/:id_partition/:hash/:style/:filename' - # end - # - # has_attached_file :zip_file, path: s3_path - # else - # has_attached_file :zip_file - # end - has_one_attached :zip_file - # validates_attachment :zip_file, - # content_type: { content_type: 'application/zip' } - after_create :self_destruct def self.delete_expired_export(id) diff --git a/app/services/protocol_importers/build_protocol_from_client_service.rb b/app/services/protocol_importers/build_protocol_from_client_service.rb index 0d972e3ee..8e441e079 100644 --- a/app/services/protocol_importers/build_protocol_from_client_service.rb +++ b/app/services/protocol_importers/build_protocol_from_client_service.rb @@ -47,7 +47,6 @@ module ProtocolImporters # Serialize steps with nested attributes for Tables and NOT nasted attributes for Assets # We want to avoid creating (downloading) Assets instances on building first time and again on importing/creating, # when both actions are not in a row. - # Also serialization does not work properly with paperclip attrs return nil unless built_protocol built_protocol.steps.map do |step| diff --git a/app/services/user_data_deletion.rb b/app/services/user_data_deletion.rb index c98b2c522..ff0550ec6 100644 --- a/app/services/user_data_deletion.rb +++ b/app/services/user_data_deletion.rb @@ -6,7 +6,8 @@ class UserDataDeletion # Destroy tiny_mce_assets if team.tiny_mce_assets.present? team.tiny_mce_assets.each do |tiny_mce_asset| - paperclip_file_destroy(tiny_mce_asset) if tiny_mce_asset.image.exists? + tiny_mce_asset.image.purge + tiny_mce_asset.destroy! end end team.repositories.each do |repository| @@ -15,8 +16,8 @@ class UserDataDeletion repository_row .repository_cells .where(value_type: 'RepositoryAssetValue').each do |repository_cell| - next unless repository_cell.value.asset.file.exists? - paperclip_file_destroy(repository_cell.value.asset) + repository_cell.value.asset.file.purge + repository_cell.value.asset.destroy! end end repository.destroy @@ -30,8 +31,10 @@ class UserDataDeletion my_module.results.each do |result| result.result_table.delete if result.result_table.present? result.table.delete if result.table.present? - next unless result.asset && result.asset.file.exists? - paperclip_file_destroy(result.asset) + next unless result.asset + + result.asset.file.purge + result.asset.destroy! end my_module.activities.destroy_all my_module.inputs.destroy_all @@ -91,8 +94,8 @@ class UserDataDeletion # Destroy step assets if step.assets.present? step.assets.each do |asset| - next unless asset.file.present? - paperclip_file_destroy(asset) + asset.file.purge + asset.destroy! end end # Destroy step @@ -129,23 +132,4 @@ class UserDataDeletion # Now, simply destroy all user notification relations left user.user_notifications.destroy_all end - - # Workaround for Paperclip preserve_files option, to delete files anyway; - # if you call #clear with a list of styles to delete, - # it'll call #queue_some_for_delete, which doesn't check :preserve_files. - def self.paperclip_file_destroy(asset) - if asset.class.name == 'TinyMceAsset' - all_styles = asset.image.styles.keys.map do |key| - asset.image.styles[key].name - end << :original - asset.image.clear(*all_styles) - else - all_styles = asset.file.styles.keys.map do |key| - asset.file.styles[key].name - end << :original - asset.file.clear(*all_styles) - end - asset.save! - asset.destroy! - end end diff --git a/config/initializers/aws.rb b/config/initializers/aws.rb deleted file mode 100644 index a5db3cb96..000000000 --- a/config/initializers/aws.rb +++ /dev/null @@ -1,5 +0,0 @@ -# frozen_string_literal: true - -if ENV['PAPERCLIP_STORAGE'] == 's3' - S3_BUCKET = Aws::S3::Resource.new.bucket(ENV['S3_BUCKET']) -end diff --git a/config/initializers/paperclip.rb b/config/initializers/paperclip.rb deleted file mode 100644 index f032d32ca..000000000 --- a/config/initializers/paperclip.rb +++ /dev/null @@ -1,278 +0,0 @@ -Paperclip::DataUriAdapter.register - -if ENV['PAPERCLIP_HASH_SECRET'].nil? - puts "WARNING! Environment variable PAPERCLIP_HASH_SECRET must be set." - exit 1 -end - -Paperclip::Attachment.default_options.merge!( - hash_data: ':class/:attachment/:id/:style', - hash_secret: ENV['PAPERCLIP_HASH_SECRET'], - preserve_files: true, - url: '/system/:class/:attachment/:id_partition/:hash/:style/:filename' -) - -Paperclip::UriAdapter.register - -if ENV['PAPERCLIP_STORAGE'] == 's3' - if ENV['S3_BUCKET'].nil? || ENV['AWS_REGION'].nil? - puts 'WARNING! Environment variables S3_BUCKET and AWS_REGION must be set.' - exit 1 - end - - s3_credentials = { bucket: ENV['S3_BUCKET'] } - - if ENV['AWS_ACCESS_KEY_ID'] && ENV['AWS_SECRET_ACCESS_KEY'] - s3_credentials[:access_key_id] = ENV['AWS_ACCESS_KEY_ID'] - s3_credentials[:secret_access_key] = ENV['AWS_SECRET_ACCESS_KEY'] - end - - s3_path = '/:class/:attachment/:id_partition/:hash/:style/:filename' - s3_path.prepend("/#{ENV['S3_SUBFOLDER']}") if ENV['S3_SUBFOLDER'] - - Paperclip::Attachment.default_options.merge!( - url: ':s3_domain_url', - path: s3_path, - storage: :s3, - s3_region: ENV['AWS_REGION'], - s3_host_name: "s3.#{ENV['AWS_REGION']}.amazonaws.com", - s3_protocol: 'https', - s3_credentials: s3_credentials, - s3_permissions: { - original: :private, - medium: :private - }, - s3_storage_class: { - medium: :REDUCED_REDUNDANCY, - thumb: :REDUCED_REDUNDANCY, - icon: :REDUCED_REDUNDANCY, - icon_small: :REDUCED_REDUNDANCY - } - ) -elsif ENV['PAPERCLIP_STORAGE'] == 'filesystem' - Paperclip::Attachment.default_options[:storage] = :filesystem -end - -Paperclip::Attachment.class_eval do - def is_stored_on_s3? - options[:storage].to_sym == :s3 - end - - def fetch - Paperclip.io_adapters.for self - end -end - -module Paperclip - # Checks file for spoofing - class MediaTypeSpoofDetector - def spoofed? - return false if ENV['DISABLE_SPOOF_CHECKING'] - - if has_name? && has_extension? && (media_type_mismatch? || - mapping_override_mismatch?) - Paperclip.log("Content Type Spoof: Filename #{File.basename(@name)} "\ - "(#{supplied_content_type} from Headers, #{content_types_from_name} "\ - "from Extension), content type discovered: "\ - "#{calculated_content_type}. See documentation to allow this "\ - "combination.") - true - else - false - end - end - - private - - # Determine file content type from its name - def content_types_from_name - @content_types_from_name ||= - Paperclip.run('mimetype', '-b -- :file_name', file_name: @name).chomp - end - - # Determine file media type from its name - def media_types_from_name - @media_types_from_name ||= extract_media_type content_types_from_name - end - - # Determine file content type from mimetype command - def type_from_mimetype_command - @type_from_mimetype_command ||= - Paperclip.run('mimetype', '-b -- :file', file: @file.path).chomp - end - - # Determine file media type from mimetype command - def media_type_from_mimetype_command - @media_type_from_mimetype_command ||= - extract_media_type type_from_mimetype_command - end - - # Determine file content type from it's content (file and mimetype command) - def type_from_file_command - unless defined? @type_from_file_command - @type_from_file_command = - Paperclip.run('file', '-b --mime -- :file', file: @file.path) - .split(/[:;]\s+/).first - - if allowed_spoof_exception?(@type_from_file_command, - media_type_from_file_command) || - (@type_from_file_command.in?(%w(text/plain text/html)) && - media_type_from_mimetype_command.in?(%w(text application))) - # File content type is generalized, so rely on file extension for - # correct/more specific content type - @type_from_file_command = type_from_mimetype_command - end - end - @type_from_file_command - rescue StandardError => e - puts e - end - - # Determine file media type from it's content (file and mimetype command) - def media_type_from_file_command - @media_type_from_file_command ||= - extract_media_type type_from_file_command - end - - def extract_media_type(content_type) - if content_type.empty? - '' - else - content_type.split('/').first - end - end - - def media_type_mismatch? - calculated_type_mismatch? - end - - # Checks file media type mismatch between file's name and header - # NOTE: Can't rely on headers, as different OS can have different file type - # MIME mappings - def supplied_type_mismatch? - !allowed_spoof_exception?(supplied_content_type, supplied_media_type) && - media_types_from_name != supplied_media_type - end - - # Checks file media type mismatch between file's name and content - def calculated_type_mismatch? - !allowed_spoof_exception?(calculated_content_type, - calculated_media_type) && - media_types_from_name != calculated_media_type - end - - # Checks file content type mismatch between file's name and content - def mapping_override_mismatch? - !allowed_spoof_exception?(calculated_content_type, - calculated_media_type) && - content_types_from_name != calculated_content_type - end - - # Check if we have a file spoof exception which is allowed/safe - def allowed_spoof_exception?(content_type, media_type) - content_type == 'application/octet-stream' || - (content_type == 'inode/x-empty' && @file.size.zero?) || - (content_type == 'text/x-c' && - content_types_from_name == 'text/x-java') || - (media_type.in?(%w(image audio video)) && - media_type == media_types_from_name) || - (content_types_from_name.in? %W(#{} - text/plain - application/octet-stream)) || - # Types taken from: http://filext.com/faq/office_mime_types.php and - # https://www.openoffice.org/framework/documentation/mimetypes/mimetypes.html - # - # Generic application - (Set[content_type, content_types_from_name].subset? Set.new %w( - text/rtf - application/rtf - )) || - # Word processor application - (Set[content_type, content_types_from_name].subset? Set.new %w( - application/zip - application/vnd.ms-office - application/msword - application/msword-template - application/vnd.openxmlformats-officedocument.wordprocessingml.document - application/vnd.openxmlformats-officedocument.wordprocessingml.template - application/vnd.ms-word.document.macroEnabled.12 - application/vnd.ms-word.template.macroEnabled.12 - application/vnd.oasis.opendocument.text - application/vnd.oasis.opendocument.text-template - application/vnd.oasis.opendocument.text-web - application/vnd.oasis.opendocument.text-master - application/vnd.sun.xml.writer - application/vnd.sun.xml.writer.template - application/vnd.sun.xml.writer.global - application/vnd.stardivision.writer - application/vnd.stardivision.writer-global - application/x-starwriter - )) || - # Spreadsheet application - (Set[content_type, content_types_from_name].subset? Set.new %w( - application/zip - application/vnd.ms-office - application/vnd.ms-excel - application/vnd.openxmlformats-officedocument.spreadsheetml.sheet - application/vnd.openxmlformats-officedocument.spreadsheetml.template - application/vnd.ms-excel.sheet.macroEnabled.12 - application/vnd.ms-excel.template.macroEnabled.12 - application/vnd.ms-excel.addin.macroEnabled.12 - application/vnd.ms-excel.sheet.binary.macroEnabled.12 - application/vnd.oasis.opendocument.spreadsheet - application/vnd.oasis.opendocument.spreadsheet-template - application/vnd.sun.xml.calc - application/vnd.sun.xml.calc.template - application/vnd.stardivision.calc - application/x-starcalc - application/CDFV2-encrypted - )) || - # Presentation application - (Set[content_type, content_types_from_name].subset? Set.new %w( - application/zip - application/vnd.ms-office - application/vnd.ms-powerpoint - application/vnd.openxmlformats-officedocument.presentationml.presentation - application/vnd.openxmlformats-officedocument.presentationml.template - application/vnd.openxmlformats-officedocument.presentationml.slideshow - application/vnd.ms-powerpoint.addin.macroEnabled.12 - application/vnd.ms-powerpoint.presentation.macroEnabled.12 - application/vnd.ms-powerpoint.template.macroEnabled.12 - application/vnd.ms-powerpoint.slideshow.macroEnabled.12 - application/vnd.oasis.opendocument.presentation - application/vnd.oasis.opendocument.presentation-template - application/vnd.sun.xml.impress - application/vnd.sun.xml.impress.template - application/vnd.stardivision.impress - application/vnd.stardivision.impress-packed - application/x-starimpress - text/x-gettext-translation-template - )) || - # Graphics application - (Set[content_type, content_types_from_name].subset? Set.new %w( - application/vnd.ms-office - application/vnd.oasis.opendocument.graphics - application/vnd.oasis.opendocument.graphics-template - application/vnd.sun.xml.draw - application/vnd.sun.xml.draw.template - application/vnd.stardivision.draw - application/x-stardraw - )) || - # Formula application - (Set[content_type, content_types_from_name].subset? Set.new %w( - application/vnd.ms-office - application/vnd.oasis.opendocument.formula - application/vnd.sun.xml.math - application/vnd.stardivision.math - application/x-starmath - )) || - # Chart application - (Set[content_type, content_types_from_name].subset? Set.new %w( - application/vnd.ms-office - application/vnd.oasis.opendocument.chart - application/vnd.stardivision.chart - application/x-starchart - )) - end - end -end diff --git a/lib/paperclip_processors/custom_file_preview.rb b/lib/paperclip_processors/custom_file_preview.rb deleted file mode 100644 index 01e2b57cc..000000000 --- a/lib/paperclip_processors/custom_file_preview.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -module Paperclip - class CustomFilePreview < Processor - def make - pdftoppm_path = ENV['PDFTOPPM_PATH'] || 'pdftoppm' - libreoffice_path = ENV['LIBREOFFICE_PATH'] || 'soffice' - directory = File.dirname(@file.path) - basename = File.basename(@file.path, '.*') - original_preview_file = File.join(directory, "#{basename}.png") - dst = TempfileFactory.new.generate("#{basename}.#{options[:format]}") - - begin - if @file.content_type == 'application/pdf' - Paperclip.run( - pdftoppm_path, - "-singlefile -r 72 -png #{@file.path} #{File.join(directory, basename)}" - ) - else - Paperclip.run( - libreoffice_path, - "--headless --invisible --convert-to png --outdir #{directory} #{@file.path}" - ) - end - - convert( - ":source -resize '#{options[:geometry]}' -format #{options[:format]} #{options[:convert_options]} :dest", - source: File.expand_path(original_preview_file), - dest: File.expand_path(dst.path) - ) - ensure - File.delete(original_preview_file) if File.file?(original_preview_file) - end - - dst - rescue StandardError => e - raise Paperclip::Error, "There was an error generating document preview - #{e}" - end - end -end diff --git a/lib/tasks/active_storage.rake b/lib/tasks/active_storage.rake index afac1c0ed..c0142e81e 100644 --- a/lib/tasks/active_storage.rake +++ b/lib/tasks/active_storage.rake @@ -24,6 +24,8 @@ namespace :active_storage do end puts 'Finished' elsif ENV['PAPERCLIP_STORAGE'] == 's3' || ENV['ACTIVESTORAGE_SERVICE'] == 'amazon' + S3_BUCKET = Aws::S3::Resource.new.bucket(ENV['S3_BUCKET']) + ActiveStorage::Blob.find_each do |blob| next unless blob.key.match?(%r{assets|experiments|temp_files|tiny_mce_assets|users|zip_exports\/}) diff --git a/lib/tasks/paperclip.rake b/lib/tasks/paperclip.rake deleted file mode 100644 index 21c79a6e3..000000000 --- a/lib/tasks/paperclip.rake +++ /dev/null @@ -1,53 +0,0 @@ -namespace :paperclip do - - def trim_url(url) - url.split("?").first - end - - def print_model_files(model, file_attr) - - model.all.each do |a| - file = a.send file_attr - styles = file.options[:styles] - url = file.url - - puts trim_url(url) - - if styles - styles.each do |style, option| - url = file.url style - puts trim_url(url) - end - end - end - end - - desc 'List all paperclip files' - task files: :environment do - print_model_files Asset, :file - print_model_files User, :avatar - end - - desc 'Reprocess the Assets attachents styles' - task :reprocess, [:before] => :environment do |_, args| - error = false - assets = Asset - if args.present? && args[:before].present? - assets = assets.where('updated_at < ?', eval(args[:before])) - end - assets.find_each(batch_size: 100) do |asset| - next unless asset.image? - begin - asset.file.reprocess! :medium, :large - rescue - error = true - $stderr.puts "exception while processing #{asset.file_file_name} " \ - "ID: #{asset.id}:" - end - end - unless error - $stderr.puts 'All gone well! ' \ - 'You can grab a beer now and enjoy the evening.' - end - end -end