Remove Paperclip from codebase [SCI-3908]

Gemfile

@@ -62,9 +62,6 @@ gem 'bcrypt', '~> 3.1.10'
 gem 'caracal-rails' # Build docx report
 gem 'commit_param_routing' # Enables different submit actions in the same form
 gem 'deface', '~> 1.0'
-gem 'delayed_paperclip',
-    git: 'https://github.com/jrgifford/delayed_paperclip.git',
-    ref: 'fcf574c'
 gem 'faker' # Generate fake data
 gem 'fastimage' # Light gem to get image resolution
 gem 'httparty', '~> 0.13.1'
@@ -93,7 +90,6 @@ gem 'devise-async',
     git: 'https://github.com/mhfs/devise-async.git',
     branch: 'devise-4.x'
 gem 'image_processing', '~> 1.2'
-gem 'paperclip', '~> 6.1' # File attachment, image attachment library
 gem 'rufus-scheduler', '~> 3.5'
 
 gem 'discard', '~> 1.0'

Gemfile.lock

@@ -22,15 +22,6 @@ GIT
     sneaky-save (0.1.3)
       activerecord (>= 3.2.0)
 
-GIT
-  remote: https://github.com/jrgifford/delayed_paperclip.git
-  revision: fcf574c1188213d3c8fce934fd52861a8ba080cb
-  ref: fcf574c
-  specs:
-    delayed_paperclip (3.0.1)
-      activejob (>= 4.2)
-      paperclip (>= 3.3)
-
 GIT
   remote: https://github.com/mhfs/devise-async.git
   revision: 177f6363a002f7ff28f1d289c8cab7ad8d9cb8c5
@@ -186,7 +177,6 @@ GEM
       activesupport
     childprocess (1.0.1)
       rake (< 13.0)
-    climate_control (0.2.0)
     coderay (1.1.2)
     coffee-rails (5.0.0)
       coffee-script (>= 2.2.0)
@@ -383,12 +373,6 @@ GEM
     overcommit (0.49.1)
       childprocess (>= 0.6.3, < 2.0)
       iniparse (~> 1.4)
-    paperclip (6.1.0)
-      activemodel (>= 4.2.0)
-      activesupport (>= 4.2.0)
-      mime-types
-      mimemagic (~> 0.3.0)
-      terrapin (~> 0.6.0)
     parallel (1.17.0)
     parser (2.6.4.0)
       ast (~> 2.4.0)
@@ -556,8 +540,6 @@ GEM
       activesupport (>= 4.0)
       sprockets (>= 3.0.0)
     stream (0.5.2)
-    terrapin (0.6.0)
-      climate_control (>= 0.0.3, < 1.0)
     thor (0.20.3)
     thread_safe (0.3.6)
     tilt (2.0.9)
@@ -628,7 +610,6 @@ DEPENDENCIES
   database_cleaner
   deface (~> 1.0)
   delayed_job_active_record
-  delayed_paperclip!
   devise (~> 4.7.1)
   devise-async!
   devise_invitable
@@ -662,7 +643,6 @@ DEPENDENCIES
   omniauth-linkedin-oauth2
   omniauth-rails_csrf_protection (~> 0.1)
   overcommit
-  paperclip (~> 6.1)
   pg (~> 1.1)
   pg_search
   pry

app/models/asset.rb

@@ -15,47 +15,8 @@ class Asset < ApplicationRecord
   # ActiveStorage configuration
   has_one_attached :file
 
-  # Paperclip validation
-  # has_attached_file :file,
-  #                   styles: lambda { |a|
-  #                     if a.previewable_document?
-  #                       {
-  #                         large: { processors: [:custom_file_preview],
-  #                                  geometry: Constants::LARGE_PIC_FORMAT,
-  #                                  format: :jpg },
-  #                         medium: { processors: [:custom_file_preview],
-  #                                   geometry: Constants::MEDIUM_PIC_FORMAT,
-  #                                   format: :jpg }
-  #                       }
-  #                     else
-  #                       {
-  #                         large: [Constants::LARGE_PIC_FORMAT, :jpg],
-  #                         medium: [Constants::MEDIUM_PIC_FORMAT, :jpg]
-  #                       }
-  #                     end
-  #                   },
-  #                   convert_options: {
-  #                     medium: '-quality 70 -strip',
-  #                     all: '-background "#d2d2d2" -flatten +matte'
-  #                   }
-
-  # before_post_process :previewable?
   # before_post_process :extract_image_quality
 
-  # adds image processing in background job
-  # process_in_background :file, processing_image_url: '/images/:style/processing.gif'
-
-  # validates_attachment :file,
-  #                      presence: true,
-  #                      size: {
-  #                        less_than: Rails.configuration.x.file_max_size_mb.megabytes
-  #                      }
-  # validates :estimated_size, presence: true
-  # validates :file_present, inclusion: { in: [true, false] }
-
-  # Should be checked for any security leaks
-  # do_not_validate_attachment_file_type :file
-
   # Asset validation
   # This could cause some problems if you create empty asset and want to
   # assign it to result

app/models/tiny_mce_asset.rb

@@ -18,20 +18,6 @@ class TinyMceAsset < ApplicationRecord
 
   has_one_attached :image
 
-  # has_attached_file :image,
-  #                   styles: { large: [Constants::LARGE_PIC_FORMAT, :jpg] },
-  #                   convert_options: { large: '-quality 100 -strip' }
-
-  # validates_attachment_content_type :image,
-  #                                   content_type: %r{^image/#{ Regexp.union(
-  #                                     Constants::WHITELISTED_IMAGE_TYPES
-  #                                   ) }}
-  # validates_attachment :image,
-  #                      presence: true,
-  #                      size: {
-  #                        less_than: Rails.configuration.x\
-  #                                        .file_max_size_mb.megabytes
-  #                      }
   validates :estimated_size, presence: true
 
   def self.update_images(object, images, current_user)

app/models/zip_export.rb

@@ -21,26 +21,8 @@ require 'csv'
 class ZipExport < ApplicationRecord
   belongs_to :user, optional: true
 
-  # Override path only for S3
-  # if ENV['PAPERCLIP_STORAGE'] == 's3'
-  #   s3_path =
-  #     if ENV['S3_SUBFOLDER']
-  #       "/#{ENV['S3_SUBFOLDER']}/zip_exports/:attachment/"\
-  #       ":id_partition/:hash/:style/:filename"
-  #     else
-  #       '/zip_exports/:attachment/:id_partition/:hash/:style/:filename'
-  #     end
-  #
-  #   has_attached_file :zip_file, path: s3_path
-  # else
-  #   has_attached_file :zip_file
-  # end
-
   has_one_attached :zip_file
 
-  # validates_attachment :zip_file,
-  #                      content_type: { content_type: 'application/zip' }
-
   after_create :self_destruct
 
   def self.delete_expired_export(id)

app/services/protocol_importers/build_protocol_from_client_service.rb

@@ -47,7 +47,6 @@ module ProtocolImporters
       # Serialize steps with nested attributes for Tables and NOT nasted attributes for Assets
       # We want to avoid creating (downloading) Assets instances on building first time and again on importing/creating,
       # when both actions are not in a row.
-      # Also serialization does not work properly with paperclip attrs
       return nil unless built_protocol
 
       built_protocol.steps.map do |step|

app/services/user_data_deletion.rb

@@ -6,7 +6,8 @@ class UserDataDeletion
       # Destroy tiny_mce_assets
       if team.tiny_mce_assets.present?
         team.tiny_mce_assets.each do |tiny_mce_asset|
-          paperclip_file_destroy(tiny_mce_asset) if tiny_mce_asset.image.exists?
+          tiny_mce_asset.image.purge
+          tiny_mce_asset.destroy!
         end
       end
       team.repositories.each do |repository|
@@ -15,8 +16,8 @@ class UserDataDeletion
           repository_row
             .repository_cells
             .where(value_type: 'RepositoryAssetValue').each do |repository_cell|
-            next unless repository_cell.value.asset.file.exists?
+            repository_cell.value.asset.file.purge
-            paperclip_file_destroy(repository_cell.value.asset)
+            repository_cell.value.asset.destroy!
           end
         end
         repository.destroy
@@ -30,8 +31,10 @@ class UserDataDeletion
             my_module.results.each do |result|
               result.result_table.delete if result.result_table.present?
               result.table.delete if result.table.present?
-              next unless result.asset && result.asset.file.exists?
+              next unless result.asset
-              paperclip_file_destroy(result.asset)
+
+              result.asset.file.purge
+              result.asset.destroy!
             end
             my_module.activities.destroy_all
             my_module.inputs.destroy_all
@@ -91,8 +94,8 @@ class UserDataDeletion
       # Destroy step assets
       if step.assets.present?
         step.assets.each do |asset|
-          next unless asset.file.present?
+          asset.file.purge
-          paperclip_file_destroy(asset)
+          asset.destroy!
         end
       end
       # Destroy step
@@ -129,23 +132,4 @@ class UserDataDeletion
     # Now, simply destroy all user notification relations left
     user.user_notifications.destroy_all
   end
-
-  # Workaround for Paperclip preserve_files option, to delete files anyway;
-  # if you call #clear with a list of styles to delete,
-  # it'll call #queue_some_for_delete, which doesn't check :preserve_files.
-  def self.paperclip_file_destroy(asset)
-    if asset.class.name == 'TinyMceAsset'
-      all_styles = asset.image.styles.keys.map do |key|
-        asset.image.styles[key].name
-      end << :original
-      asset.image.clear(*all_styles)
-    else
-      all_styles = asset.file.styles.keys.map do |key|
-        asset.file.styles[key].name
-      end << :original
-      asset.file.clear(*all_styles)
-    end
-    asset.save!
-    asset.destroy!
-  end
 end

config/initializers/aws.rb

@@ -1,5 +0,0 @@
-# frozen_string_literal: true
-
-if ENV['PAPERCLIP_STORAGE'] == 's3'
-  S3_BUCKET = Aws::S3::Resource.new.bucket(ENV['S3_BUCKET'])
-end

config/initializers/paperclip.rb

@@ -1,278 +0,0 @@
-Paperclip::DataUriAdapter.register
-
-if ENV['PAPERCLIP_HASH_SECRET'].nil?
-  puts "WARNING! Environment variable PAPERCLIP_HASH_SECRET must be set."
-  exit 1
-end
-
-Paperclip::Attachment.default_options.merge!(
-  hash_data: ':class/:attachment/:id/:style',
-  hash_secret: ENV['PAPERCLIP_HASH_SECRET'],
-  preserve_files: true,
-  url: '/system/:class/:attachment/:id_partition/:hash/:style/:filename'
-)
-
-Paperclip::UriAdapter.register
-
-if ENV['PAPERCLIP_STORAGE'] == 's3'
-  if ENV['S3_BUCKET'].nil? || ENV['AWS_REGION'].nil?
-    puts 'WARNING! Environment variables S3_BUCKET and AWS_REGION must be set.'
-    exit 1
-  end
-
-  s3_credentials = { bucket: ENV['S3_BUCKET'] }
-
-  if ENV['AWS_ACCESS_KEY_ID'] && ENV['AWS_SECRET_ACCESS_KEY']
-    s3_credentials[:access_key_id] = ENV['AWS_ACCESS_KEY_ID']
-    s3_credentials[:secret_access_key] = ENV['AWS_SECRET_ACCESS_KEY']
-  end
-
-  s3_path = '/:class/:attachment/:id_partition/:hash/:style/:filename'
-  s3_path.prepend("/#{ENV['S3_SUBFOLDER']}") if ENV['S3_SUBFOLDER']
-
-  Paperclip::Attachment.default_options.merge!(
-    url: ':s3_domain_url',
-    path: s3_path,
-    storage: :s3,
-    s3_region: ENV['AWS_REGION'],
-    s3_host_name: "s3.#{ENV['AWS_REGION']}.amazonaws.com",
-    s3_protocol: 'https',
-    s3_credentials: s3_credentials,
-    s3_permissions: {
-      original: :private,
-      medium: :private
-    },
-    s3_storage_class: {
-      medium: :REDUCED_REDUNDANCY,
-      thumb: :REDUCED_REDUNDANCY,
-      icon: :REDUCED_REDUNDANCY,
-      icon_small: :REDUCED_REDUNDANCY
-    }
-  )
-elsif ENV['PAPERCLIP_STORAGE'] == 'filesystem'
-  Paperclip::Attachment.default_options[:storage] = :filesystem
-end
-
-Paperclip::Attachment.class_eval do
-  def is_stored_on_s3?
-    options[:storage].to_sym == :s3
-  end
-
-  def fetch
-    Paperclip.io_adapters.for self
-  end
-end
-
-module Paperclip
-  # Checks file for spoofing
-  class MediaTypeSpoofDetector
-    def spoofed?
-      return false if ENV['DISABLE_SPOOF_CHECKING']
-
-      if has_name? && has_extension? && (media_type_mismatch? ||
-        mapping_override_mismatch?)
-        Paperclip.log("Content Type Spoof: Filename #{File.basename(@name)} "\
-          "(#{supplied_content_type} from Headers, #{content_types_from_name} "\
-          "from Extension), content type discovered: "\
-          "#{calculated_content_type}. See documentation to allow this "\
-          "combination.")
-        true
-      else
-        false
-      end
-    end
-
-    private
-
-    # Determine file content type from its name
-    def content_types_from_name
-      @content_types_from_name ||=
-        Paperclip.run('mimetype', '-b -- :file_name', file_name: @name).chomp
-    end
-
-    # Determine file media type from its name
-    def media_types_from_name
-      @media_types_from_name ||= extract_media_type content_types_from_name
-    end
-
-    # Determine file content type from mimetype command
-    def type_from_mimetype_command
-      @type_from_mimetype_command ||=
-        Paperclip.run('mimetype', '-b -- :file', file: @file.path).chomp
-    end
-
-    # Determine file media type from mimetype command
-    def media_type_from_mimetype_command
-      @media_type_from_mimetype_command ||=
-        extract_media_type type_from_mimetype_command
-    end
-
-    # Determine file content type from it's content (file and mimetype command)
-    def type_from_file_command
-      unless defined? @type_from_file_command
-        @type_from_file_command =
-          Paperclip.run('file', '-b --mime -- :file', file: @file.path)
-                   .split(/[:;]\s+/).first
-
-        if allowed_spoof_exception?(@type_from_file_command,
-                                    media_type_from_file_command) ||
-           (@type_from_file_command.in?(%w(text/plain text/html)) &&
-            media_type_from_mimetype_command.in?(%w(text application)))
-          # File content type is generalized, so rely on file extension for
-          # correct/more specific content type
-          @type_from_file_command = type_from_mimetype_command
-        end
-      end
-      @type_from_file_command
-    rescue StandardError => e
-      puts e
-    end
-
-    # Determine file media type from it's content (file and mimetype command)
-    def media_type_from_file_command
-      @media_type_from_file_command ||=
-        extract_media_type type_from_file_command
-    end
-
-    def extract_media_type(content_type)
-      if content_type.empty?
-        ''
-      else
-        content_type.split('/').first
-      end
-    end
-
-    def media_type_mismatch?
-      calculated_type_mismatch?
-    end
-
-    # Checks file media type mismatch between file's name and header
-    # NOTE: Can't rely on headers, as different OS can have different file type
-    # MIME mappings
-    def supplied_type_mismatch?
-      !allowed_spoof_exception?(supplied_content_type, supplied_media_type) &&
-        media_types_from_name != supplied_media_type
-    end
-
-    # Checks file media type mismatch between file's name and content
-    def calculated_type_mismatch?
-      !allowed_spoof_exception?(calculated_content_type,
-                                calculated_media_type) &&
-        media_types_from_name != calculated_media_type
-    end
-
-    # Checks file content type mismatch between file's name and content
-    def mapping_override_mismatch?
-      !allowed_spoof_exception?(calculated_content_type,
-                                calculated_media_type) &&
-        content_types_from_name != calculated_content_type
-    end
-
-    # Check if we have a file spoof exception which is allowed/safe
-    def allowed_spoof_exception?(content_type, media_type)
-      content_type == 'application/octet-stream' ||
-        (content_type == 'inode/x-empty' && @file.size.zero?) ||
-        (content_type == 'text/x-c' &&
-         content_types_from_name == 'text/x-java') ||
-        (media_type.in?(%w(image audio video)) &&
-         media_type == media_types_from_name) ||
-        (content_types_from_name.in? %W(#{}
-                                        text/plain
-                                        application/octet-stream)) ||
-        # Types taken from: http://filext.com/faq/office_mime_types.php and
-        # https://www.openoffice.org/framework/documentation/mimetypes/mimetypes.html
-        #
-        # Generic application
-        (Set[content_type, content_types_from_name].subset? Set.new %w(
-          text/rtf
-          application/rtf
-        )) ||
-        # Word processor application
-        (Set[content_type, content_types_from_name].subset? Set.new %w(
-          application/zip
-          application/vnd.ms-office
-          application/msword
-          application/msword-template
-          application/vnd.openxmlformats-officedocument.wordprocessingml.document
-          application/vnd.openxmlformats-officedocument.wordprocessingml.template
-          application/vnd.ms-word.document.macroEnabled.12
-          application/vnd.ms-word.template.macroEnabled.12
-          application/vnd.oasis.opendocument.text
-          application/vnd.oasis.opendocument.text-template
-          application/vnd.oasis.opendocument.text-web
-          application/vnd.oasis.opendocument.text-master
-          application/vnd.sun.xml.writer
-          application/vnd.sun.xml.writer.template
-          application/vnd.sun.xml.writer.global
-          application/vnd.stardivision.writer
-          application/vnd.stardivision.writer-global
-          application/x-starwriter
-        )) ||
-        # Spreadsheet application
-        (Set[content_type, content_types_from_name].subset? Set.new %w(
-          application/zip
-          application/vnd.ms-office
-          application/vnd.ms-excel
-          application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
-          application/vnd.openxmlformats-officedocument.spreadsheetml.template
-          application/vnd.ms-excel.sheet.macroEnabled.12
-          application/vnd.ms-excel.template.macroEnabled.12
-          application/vnd.ms-excel.addin.macroEnabled.12
-          application/vnd.ms-excel.sheet.binary.macroEnabled.12
-          application/vnd.oasis.opendocument.spreadsheet
-          application/vnd.oasis.opendocument.spreadsheet-template
-          application/vnd.sun.xml.calc
-          application/vnd.sun.xml.calc.template
-          application/vnd.stardivision.calc
-          application/x-starcalc
-          application/CDFV2-encrypted
-        )) ||
-        # Presentation application
-        (Set[content_type, content_types_from_name].subset? Set.new %w(
-          application/zip
-          application/vnd.ms-office
-          application/vnd.ms-powerpoint
-          application/vnd.openxmlformats-officedocument.presentationml.presentation
-          application/vnd.openxmlformats-officedocument.presentationml.template
-          application/vnd.openxmlformats-officedocument.presentationml.slideshow
-          application/vnd.ms-powerpoint.addin.macroEnabled.12
-          application/vnd.ms-powerpoint.presentation.macroEnabled.12
-          application/vnd.ms-powerpoint.template.macroEnabled.12
-          application/vnd.ms-powerpoint.slideshow.macroEnabled.12
-          application/vnd.oasis.opendocument.presentation
-          application/vnd.oasis.opendocument.presentation-template
-          application/vnd.sun.xml.impress
-          application/vnd.sun.xml.impress.template
-          application/vnd.stardivision.impress
-          application/vnd.stardivision.impress-packed
-          application/x-starimpress
-          text/x-gettext-translation-template
-        )) ||
-        # Graphics application
-        (Set[content_type, content_types_from_name].subset? Set.new %w(
-          application/vnd.ms-office
-          application/vnd.oasis.opendocument.graphics
-          application/vnd.oasis.opendocument.graphics-template
-          application/vnd.sun.xml.draw
-          application/vnd.sun.xml.draw.template
-          application/vnd.stardivision.draw
-          application/x-stardraw
-        )) ||
-        # Formula application
-        (Set[content_type, content_types_from_name].subset? Set.new %w(
-          application/vnd.ms-office
-          application/vnd.oasis.opendocument.formula
-          application/vnd.sun.xml.math
-          application/vnd.stardivision.math
-          application/x-starmath
-        )) ||
-        # Chart application
-        (Set[content_type, content_types_from_name].subset? Set.new %w(
-          application/vnd.ms-office
-          application/vnd.oasis.opendocument.chart
-          application/vnd.stardivision.chart
-          application/x-starchart
-        ))
-    end
-  end
-end

lib/paperclip_processors/custom_file_preview.rb

@@ -1,40 +0,0 @@
-# frozen_string_literal: true
-
-module Paperclip
-  class CustomFilePreview < Processor
-    def make
-      pdftoppm_path = ENV['PDFTOPPM_PATH'] || 'pdftoppm'
-      libreoffice_path = ENV['LIBREOFFICE_PATH'] || 'soffice'
-      directory = File.dirname(@file.path)
-      basename  = File.basename(@file.path, '.*')
-      original_preview_file = File.join(directory, "#{basename}.png")
-      dst = TempfileFactory.new.generate("#{basename}.#{options[:format]}")
-
-      begin
-        if @file.content_type == 'application/pdf'
-          Paperclip.run(
-            pdftoppm_path,
-            "-singlefile -r 72 -png #{@file.path} #{File.join(directory, basename)}"
-          )
-        else
-          Paperclip.run(
-            libreoffice_path,
-            "--headless --invisible --convert-to png --outdir #{directory} #{@file.path}"
-          )
-        end
-
-        convert(
-          ":source -resize '#{options[:geometry]}' -format #{options[:format]} #{options[:convert_options]} :dest",
-          source: File.expand_path(original_preview_file),
-          dest: File.expand_path(dst.path)
-        )
-      ensure
-        File.delete(original_preview_file) if File.file?(original_preview_file)
-      end
-
-      dst
-    rescue StandardError => e
-      raise Paperclip::Error, "There was an error generating document preview - #{e}"
-    end
-  end
-end

lib/tasks/active_storage.rake

@@ -24,6 +24,8 @@ namespace :active_storage do
       end
       puts 'Finished'
     elsif ENV['PAPERCLIP_STORAGE'] == 's3' || ENV['ACTIVESTORAGE_SERVICE'] == 'amazon'
+      S3_BUCKET = Aws::S3::Resource.new.bucket(ENV['S3_BUCKET'])
+
       ActiveStorage::Blob.find_each do |blob|
         next unless blob.key.match?(%r{assets|experiments|temp_files|tiny_mce_assets|users|zip_exports\/})
 

lib/tasks/paperclip.rake

@@ -1,53 +0,0 @@
-namespace :paperclip do
-
-  def trim_url(url)
-    url.split("?").first
-  end
-
-  def print_model_files(model, file_attr)
-
-    model.all.each do |a|
-      file = a.send file_attr
-      styles = file.options[:styles]
-      url = file.url
-
-      puts trim_url(url)
-
-      if styles
-        styles.each do |style, option|
-          url = file.url style
-          puts trim_url(url)
-        end
-      end
-    end
-  end
-
-  desc 'List all paperclip files'
-  task files: :environment do
-    print_model_files Asset, :file
-    print_model_files User, :avatar
-  end
-
-  desc 'Reprocess the Assets attachents styles'
-  task :reprocess, [:before] => :environment do |_, args|
-    error = false
-    assets = Asset
-    if args.present? && args[:before].present?
-      assets = assets.where('updated_at < ?', eval(args[:before]))
-    end
-    assets.find_each(batch_size: 100) do |asset|
-      next unless asset.image?
-      begin
-        asset.file.reprocess! :medium, :large
-      rescue
-        error = true
-        $stderr.puts "exception while processing #{asset.file_file_name} " \
-                     "ID: #{asset.id}:"
-      end
-    end
-    unless error
-      $stderr.puts 'All gone well! ' \
-                   'You can grab a beer now and enjoy the evening.'
-    end
-  end
-end