Remove Paperclip DB columns and fix experiment templates [SCI-3988]

Gemfile

@@ -90,7 +90,6 @@ gem 'devise-async',
     git: 'https://github.com/mhfs/devise-async.git',
     branch: 'devise-4.x'
 gem 'image_processing', '~> 1.2'
-gem 'paperclip', '~> 6.1'
 gem 'rufus-scheduler', '~> 3.5'
 
 gem 'discard', '~> 1.0'

Gemfile.lock

@@ -177,7 +177,6 @@ GEM
       activesupport
     childprocess (1.0.1)
       rake (< 13.0)
-    climate_control (0.2.0)
     coderay (1.1.2)
     coffee-rails (5.0.0)
       coffee-script (>= 2.2.0)
@@ -374,12 +373,6 @@ GEM
     overcommit (0.49.1)
       childprocess (>= 0.6.3, < 2.0)
       iniparse (~> 1.4)
-    paperclip (6.1.0)
-      activemodel (>= 4.2.0)
-      activesupport (>= 4.2.0)
-      mime-types
-      mimemagic (~> 0.3.0)
-      terrapin (~> 0.6.0)
     parallel (1.17.0)
     parser (2.6.4.0)
       ast (~> 2.4.0)
@@ -542,8 +535,6 @@ GEM
       activesupport (>= 4.0)
       sprockets (>= 3.0.0)
     stream (0.5.2)
-    terrapin (0.6.0)
-      climate_control (>= 0.0.3, < 1.0)
     thor (0.20.3)
     thread_safe (0.3.6)
     tilt (2.0.9)
@@ -648,7 +639,6 @@ DEPENDENCIES
   omniauth-linkedin-oauth2
   omniauth-rails_csrf_protection (~> 0.1)
   overcommit
-  paperclip (~> 6.1)
   pg (~> 1.1)
   pg_search
   pry

app/models/asset.rb

@@ -217,9 +217,6 @@ class Asset < ApplicationRecord
   end
 
   def post_process_file(team = nil)
-    # Update self.empty
-    update(file_present: true)
-
     # Extract asset text if it's of correct type
     if text?
       Rails.logger.info "Asset #{id}: Creating extract text job"

app/models/user.rb

@@ -243,8 +243,8 @@ class User < ApplicationRecord
   def avatar_remote_url=(url_value)
     self.avatar = URI.parse(url_value)
     # Assuming url_value is http://example.com/photos/face.png
-    # avatar_file_name == "face.png"
-    # avatar_content_type == "image/png"
+    # avatar.filename == "face.png"
+    # avatar.content_type == "image/png"
     @avatar_remote_url = url_value
   end
 
@@ -325,13 +325,6 @@ class User < ApplicationRecord
       .distinct
   end
 
-  def empty_avatar(name, size)
-    file_ext = name.split(".").last
-    self.avatar_file_name = name
-    self.avatar_content_type = Rack::Mime.mime_type(".#{file_ext}")
-    self.avatar_file_size = size.to_i
-  end
-
   # Whether user is active (= confirmed) or not
   def active?
     confirmed_at.present?

app/services/team_importer.rb

@@ -798,17 +798,10 @@ class TeamImporter
 
   # returns asset object
   def create_asset(asset_json, team, user_id = nil)
-    ### Fix for support templates
-    asset_info = asset_json['asset'] || asset_json
-
-    asset = Asset.new(asset_info)
+    asset = Asset.new(asset_json['asset'])
     asset_blob = asset_json['asset_blob']
     ### Fix for support templates
-    asset_file_name = if asset_blob
-                        asset_blob['filename']
-                      else
-                        asset_json['file_file_name']
-                      end
+    asset_file_name = asset_blob['filename']
     file = File.open("#{@import_dir}/assets/#{asset.id}/#{asset_file_name}")
     orig_asset_id = asset.id
     asset.id = nil

db/migrate/20191009146101_convert_experiments_to_active_storage.rb

@@ -1,105 +0,0 @@
-# frozen_string_literal: true
-
-class ConvertExperimentsToActiveStorage < ActiveRecord::Migration[5.2]
-  require 'open-uri'
-
-  ID_PARTITION_LIMIT = 1_000_000_000
-  DIGEST = OpenSSL::Digest.const_get('SHA1').new
-  MODELS = [Experiment].freeze
-
-  def up
-    ActiveRecord::Base.connection.raw_connection.prepare('active_storage_exp_blob_statement', <<-SQL)
-      INSERT INTO active_storage_blobs (
-        key, filename, content_type, metadata, byte_size, checksum, created_at
-      ) VALUES ($1, $2, $3, '{}', $4, $5, $6)
-      RETURNING id;
-    SQL
-
-    ActiveRecord::Base.connection.raw_connection.prepare('active_storage_exp_attachment_statement', <<-SQL)
-      INSERT INTO active_storage_attachments (
-        name, record_type, record_id, blob_id, created_at
-      ) VALUES ($1, $2, $3, $4, $5)
-    SQL
-
-    transaction do
-      MODELS.each do |model|
-        next unless ActiveRecord::Base.connection.table_exists?(model.table_name)
-
-        attachments = model.column_names.map do |c|
-          $1 if c =~ /(.+)_file_name$/
-        end.compact
-
-        next if attachments.blank?
-        
-        model.left_outer_joins(:workflowimg_attachment)
-            .where('active_storage_attachments.id IS NULL')
-            .find_each.each do |instance|
-          attachments.each do |attachment|
-            next if instance.__send__("#{attachment}_file_name").blank?
-
-            res = ActiveRecord::Base.connection.raw_connection.exec_prepared(
-              'active_storage_exp_blob_statement', [
-                key(instance, attachment),
-                instance.__send__("#{attachment}_file_name"),
-                instance.__send__("#{attachment}_content_type"),
-                instance.__send__("#{attachment}_file_size") || 0,
-                checksum(attachment),
-                instance.updated_at.iso8601
-              ]
-            )
-
-            ActiveRecord::Base.connection.raw_connection.exec_prepared(
-              'active_storage_exp_attachment_statement', [
-                attachment,
-                model.name,
-                instance.id,
-                res[0]['id'],
-                instance.updated_at.iso8601
-              ]
-            )
-          end
-        end
-      end
-    end
-  end
-
-  def down
-    raise ActiveRecord::IrreversibleMigration
-  end
-
-  private
-
-  ID_PARTITION_LIMIT = 1_000_000_000
-  DIGEST = OpenSSL::Digest.const_get('SHA1').new
-
-  def id_partition(id)
-    if id < ID_PARTITION_LIMIT
-      format('%09d', id).scan(/\d{3}/).join('/')
-    else
-      format('%012d', id).scan(/\d{3}/).join('/')
-    end
-  end
-
-  def hash_data(instance, attachment)
-    "#{instance.class.to_s.underscore.pluralize}/#{attachment.pluralize}/#{instance.id}/original"
-  end
-
-  def interpolate(pattern, instance, attachment)
-    path = pattern
-    path = path.gsub(':class', instance.class.to_s.underscore.pluralize)
-    path = path.gsub(':attachment', attachment.pluralize)
-    path = path.gsub(':id_partition', id_partition(instance.id))
-    path = path.gsub(':hash', OpenSSL::HMAC.hexdigest(DIGEST,
-                                                      ENV['PAPERCLIP_HASH_SECRET'],
-                                                      hash_data(instance, attachment)))
-    path.gsub(':filename', instance.__send__("#{attachment}_file_name"))
-  end
-
-  def key(instance, attachment)
-    interpolate(':class/:attachment/:id_partition/:hash/original/:filename', instance, attachment)
-  end
-
-  def checksum(_attachment)
-    'dummy'
-  end
-end

db/migrate/20191023162335_remove_paperclip_columns.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+class RemovePaperclipColumns < ActiveRecord::Migration[6.0]
+  def up
+    remove_columns :assets,
+                   :file_file_name,
+                   :file_content_type,
+                   :file_file_size,
+                   :file_updated_at,
+                   :file_present
+    remove_columns :experiments,
+                   :workflowimg_file_name,
+                   :workflowimg_content_type,
+                   :workflowimg_file_size,
+                   :workflowimg_updated_at
+    remove_columns :temp_files,
+                   :file_file_name,
+                   :file_content_type,
+                   :file_file_size,
+                   :file_updated_at
+    remove_columns :tiny_mce_assets,
+                   :image_file_name,
+                   :image_content_type,
+                   :image_file_size,
+                   :image_updated_at
+    remove_columns :users,
+                   :avatar_file_name,
+                   :avatar_content_type,
+                   :avatar_file_size,
+                   :avatar_updated_at
+    remove_columns :zip_exports,
+                   :zip_file_file_name,
+                   :zip_file_content_type,
+                   :zip_file_file_size,
+                   :zip_file_updated_at
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end

db/structure.sql

@@ -214,14 +214,9 @@ CREATE TABLE public.assets (
     id bigint NOT NULL,
     created_at timestamp without time zone NOT NULL,
     updated_at timestamp without time zone NOT NULL,
-    file_file_name character varying,
-    file_content_type character varying,
-    file_file_size bigint,
-    file_updated_at timestamp without time zone,
     created_by_id bigint,
     last_modified_by_id bigint,
     estimated_size integer DEFAULT 0 NOT NULL,
-    file_present boolean DEFAULT false NOT NULL,
     lock character varying(1024),
     lock_ttl integer,
     version integer DEFAULT 1,
@@ -528,10 +523,6 @@ CREATE TABLE public.experiments (
     restored_on timestamp without time zone,
     created_at timestamp without time zone NOT NULL,
     updated_at timestamp without time zone NOT NULL,
-    workflowimg_file_name character varying,
-    workflowimg_content_type character varying,
-    workflowimg_file_size bigint,
-    workflowimg_updated_at timestamp without time zone,
     uuid uuid
 );
 
@@ -2050,11 +2041,7 @@ CREATE TABLE public.temp_files (
     id bigint NOT NULL,
     session_id character varying NOT NULL,
     created_at timestamp without time zone NOT NULL,
-    updated_at timestamp without time zone NOT NULL,
-    file_file_name character varying,
-    file_content_type character varying,
-    file_file_size bigint,
-    file_updated_at timestamp without time zone
+    updated_at timestamp without time zone NOT NULL
 );
 
 
@@ -2083,10 +2070,6 @@ ALTER SEQUENCE public.temp_files_id_seq OWNED BY public.temp_files.id;
 
 CREATE TABLE public.tiny_mce_assets (
     id bigint NOT NULL,
-    image_file_name character varying,
-    image_content_type character varying,
-    image_file_size bigint,
-    image_updated_at timestamp without time zone,
     estimated_size integer DEFAULT 0 NOT NULL,
     team_id integer,
     created_at timestamp without time zone NOT NULL,
@@ -2353,10 +2336,6 @@ CREATE TABLE public.users (
     last_sign_in_ip character varying,
     created_at timestamp without time zone NOT NULL,
     updated_at timestamp without time zone NOT NULL,
-    avatar_file_name character varying,
-    avatar_content_type character varying,
-    avatar_file_size bigint,
-    avatar_updated_at timestamp without time zone,
     confirmation_token character varying,
     confirmed_at timestamp without time zone,
     confirmation_sent_at timestamp without time zone,
@@ -2534,10 +2513,6 @@ CREATE TABLE public.zip_exports (
     user_id bigint,
     created_at timestamp without time zone NOT NULL,
     updated_at timestamp without time zone NOT NULL,
-    zip_file_file_name character varying,
-    zip_file_content_type character varying,
-    zip_file_file_size bigint,
-    zip_file_updated_at timestamp without time zone,
     type character varying
 );
 
@@ -3746,13 +3721,6 @@ CREATE INDEX index_assets_on_created_at ON public.assets USING btree (created_at
 CREATE INDEX index_assets_on_created_by_id ON public.assets USING btree (created_by_id);
 
 
---
--- Name: index_assets_on_file_file_name; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE INDEX index_assets_on_file_file_name ON public.assets USING gin (public.trim_html_tags((file_file_name)::text) public.gin_trgm_ops);
-
-
 --
 -- Name: index_assets_on_last_modified_by_id; Type: INDEX; Schema: public; Owner: -
 --
@@ -6521,6 +6489,7 @@ INSERT INTO "schema_migrations" (version) VALUES
 ('20190830141257'),
 ('20190910125740'),
 ('20191001133557'),
-('20191009146101');
+('20191009146101'),
+('20191023162335');
 
 

spec/models/asset_spec.rb

@@ -20,7 +20,6 @@ describe Asset, type: :model do
     it { should have_db_column :created_by_id }
     it { should have_db_column :last_modified_by_id }
     it { should have_db_column :estimated_size }
-    it { should have_db_column :file_present }
     it { should have_db_column :lock }
     it { should have_db_column :lock_ttl }
     it { should have_db_column :version }

spec/services/model_importers/test_experiment_data/experiment.json

@@ -303,7 +303,6 @@
                                         "created_at": "2019-01-21T13:09:35.615Z",
                                         "created_by_id": 1,
                                         "estimated_size": 17799,
-                                        "file_present": true,
                                         "file_processing": null,
                                         "id": 21,
                                         "last_modified_by_id": null,
@@ -600,7 +599,6 @@
                             "created_at": "2019-01-21T12:45:29.138Z",
                             "created_by_id": 1,
                             "estimated_size": 232,
-                            "file_present": true,
                             "file_processing": null,
                             "id": 1,
                             "last_modified_by_id": 1,
@@ -1075,7 +1073,6 @@
                                         "created_at": "2019-01-21T12:45:32.296Z",
                                         "created_by_id": 1,
                                         "estimated_size": 1129370,
-                                        "file_present": true,
                                         "file_processing": null,
                                         "id": 8,
                                         "last_modified_by_id": 1,
@@ -1127,7 +1124,6 @@
                             "created_at": "2019-01-21T12:45:32.847Z",
                             "created_by_id": 1,
                             "estimated_size": 46131,
-                            "file_present": true,
                             "file_processing": null,
                             "id": 9,
                             "last_modified_by_id": 1,