From 109793c7fb5b8323f18a4cc079d34950e70e4877 Mon Sep 17 00:00:00 2001 From: Oleksii Kriuchykhin Date: Thu, 13 Jun 2019 12:10:44 +0200 Subject: [PATCH 1/3] Upgrade to Rails 5.2.3 and add ActiveStorage [SCI-3539] --- .rubocop.yml | 2 +- Gemfile | 3 +- Gemfile.lock | 208 +++++++++--------- bin/bundle | 2 +- bin/setup | 4 +- bin/update | 6 +- bin/yarn | 6 +- config/boot.rb | 1 + config/cable.yml | 2 +- config/environments/development.rb | 22 +- config/environments/production.rb | 56 ++--- config/environments/test.rb | 5 + .../application_controller_renderer.rb | 10 +- .../initializers/content_security_policy.rb | 25 +++ .../new_framework_defaults_5_2.rb | 38 ++++ config/storage.yml | 34 +++ ...te_active_storage_tables.active_storage.rb | 27 +++ db/schema.rb | 40 +++- docker-compose.yml | 1 + spec/factories/activities.rb | 4 +- spec/factories/asset_text_datums.rb | 2 +- spec/factories/assets.rb | 12 +- spec/factories/custom_fields.rb | 2 +- spec/factories/notifications.rb | 8 +- spec/factories/repository_columns.rb | 10 +- spec/factories/sample_groups.rb | 2 +- spec/factories/teams.rb | 4 +- spec/factories/tinymce_assets.rb | 6 +- spec/factories/user_identities.rb | 4 +- spec/factories/user_notifications.rb | 2 +- spec/factories/user_teams.rb | 6 +- spec/factories/users.rb | 10 +- spec/factories/wopi_apps.rb | 2 +- 33 files changed, 361 insertions(+), 205 deletions(-) create mode 100644 config/initializers/content_security_policy.rb create mode 100644 config/initializers/new_framework_defaults_5_2.rb create mode 100644 config/storage.yml create mode 100644 db/migrate/20190613094834_create_active_storage_tables.active_storage.rb diff --git a/.rubocop.yml b/.rubocop.yml index 8fcf1f300..b94e63629 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -91,7 +91,7 @@ Style/FormatString: EnforcedStyle: format Style/FrozenStringLiteralComment: - EnforcedStyle: when_needed + EnforcedStyle: always Style/GuardClause: Enabled: false diff --git a/Gemfile b/Gemfile index 48053ffac..4b7c719bd 100644 --- a/Gemfile +++ b/Gemfile @@ -5,6 +5,7 @@ source 'http://rubygems.org' ruby '2.5.5' +gem 'bootsnap', require: false gem 'webpacker', '~> 3.5' gem 'bootstrap-sass', '~> 3.3.7' gem 'bootstrap_form', '~> 2.7.0' @@ -13,7 +14,7 @@ gem 'devise_invitable' gem 'figaro' gem 'pg', '~> 0.18' gem 'pg_search' # PostgreSQL full text search -gem 'rails', '~> 5.1.7' +gem 'rails', '~> 5.2.3' gem 'recaptcha', require: 'recaptcha/rails' gem 'sanitize', '~> 4.4' gem 'sassc-rails' diff --git a/Gemfile.lock b/Gemfile.lock index bd6711394..ac5cda6f2 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -42,25 +42,25 @@ GIT GEM remote: http://rubygems.org/ specs: - actioncable (5.1.7) - actionpack (= 5.1.7) + actioncable (5.2.3) + actionpack (= 5.2.3) nio4r (~> 2.0) - websocket-driver (~> 0.6.1) - actionmailer (5.1.7) - actionpack (= 5.1.7) - actionview (= 5.1.7) - activejob (= 5.1.7) + websocket-driver (>= 0.6.1) + actionmailer (5.2.3) + actionpack (= 5.2.3) + actionview (= 5.2.3) + activejob (= 5.2.3) mail (~> 2.5, >= 2.5.4) rails-dom-testing (~> 2.0) - actionpack (5.1.7) - actionview (= 5.1.7) - activesupport (= 5.1.7) + actionpack (5.2.3) + actionview (= 5.2.3) + activesupport (= 5.2.3) rack (~> 2.0) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.0.2) - actionview (5.1.7) - activesupport (= 5.1.7) + actionview (5.2.3) + activesupport (= 5.2.3) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) @@ -70,18 +70,22 @@ GEM activemodel (>= 4.1, < 6) case_transform (>= 0.2) jsonapi-renderer (>= 0.1.1.beta1, < 0.3) - activejob (5.1.7) - activesupport (= 5.1.7) + activejob (5.2.3) + activesupport (= 5.2.3) globalid (>= 0.3.6) - activemodel (5.1.7) - activesupport (= 5.1.7) - activerecord (5.1.7) - activemodel (= 5.1.7) - activesupport (= 5.1.7) - arel (~> 8.0) - activerecord-import (1.0.1) + activemodel (5.2.3) + activesupport (= 5.2.3) + activerecord (5.2.3) + activemodel (= 5.2.3) + activesupport (= 5.2.3) + arel (>= 9.0) + activerecord-import (1.0.2) activerecord (>= 3.2) - activesupport (5.1.7) + activestorage (5.2.3) + actionpack (= 5.2.3) + activerecord (= 5.2.3) + marcel (~> 0.3.1) + activesupport (5.2.3) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 0.7, < 2) minitest (~> 5.1) @@ -91,18 +95,18 @@ GEM aes_key_wrap (1.0.1) ajax-datatables-rails (0.3.1) railties (>= 3.1) - arel (8.0.0) + arel (9.0.0) aspector (0.14.0) ast (2.4.0) auto_strip_attributes (2.5.0) activerecord (>= 4.0) - autoprefixer-rails (9.5.1.1) + autoprefixer-rails (9.6.0) execjs autosize-rails (1.18.17) rails (>= 3.1) awesome_print (1.8.0) aws-eventstream (1.0.3) - aws-partitions (1.172.0) + aws-partitions (1.174.0) aws-sdk-core (3.54.2) aws-eventstream (~> 1.0, >= 1.0.2) aws-partitions (~> 1.0) @@ -117,9 +121,9 @@ GEM aws-sigv4 (~> 1.1) aws-sigv4 (1.1.0) aws-eventstream (~> 1.0, >= 1.0.2) - backports (3.14.0) + backports (3.15.0) base62 (1.0.0) - bcrypt (3.1.12) + bcrypt (3.1.13) better_errors (2.5.1) coderay (>= 1.0.0) erubi (>= 1.0.0) @@ -127,6 +131,8 @@ GEM bindata (2.4.4) binding_of_caller (0.8.0) debug_inspector (>= 0.0.1) + bootsnap (1.4.4) + msgpack (~> 1.0) bootstrap-sass (3.3.7) autoprefixer-rails (>= 5.2.1) sass (>= 3.3.4) @@ -139,13 +145,13 @@ GEM activesupport (>= 3.0.0) uniform_notifier (~> 1.11) byebug (11.0.1) - capybara (3.18.0) + capybara (3.23.0) addressable mini_mime (>= 0.1.3) nokogiri (~> 1.8) rack (>= 1.6.0) rack-test (>= 0.6.3) - regexp_parser (~> 1.2) + regexp_parser (~> 1.5) xpath (~> 3.2) capybara-email (3.0.1) capybara (>= 2.4, < 4.0) @@ -157,9 +163,9 @@ GEM climate_control (0.2.0) cliver (0.3.2) coderay (1.1.2) - coffee-rails (4.2.2) + coffee-rails (5.0.0) coffee-script (>= 2.2.0) - railties (>= 4.0.0) + railties (>= 5.2.0) coffee-script (2.4.1) coffee-script-source execjs @@ -193,7 +199,7 @@ GEM cucumber-wire (0.0.1) database_cleaner (1.7.0) debug_inspector (0.0.3) - deface (1.3.2) + deface (1.4.0) nokogiri (>= 1.6) polyglot rails (>= 4.1) @@ -213,48 +219,47 @@ GEM actionmailer (>= 5.0) devise (>= 4.6) diff-lcs (1.3) - discard (1.0.0) - activerecord (>= 4.2, < 6) + discard (1.1.0) + activerecord (>= 4.2, < 7) docile (1.3.1) doorkeeper (5.1.0) railties (>= 5) erubi (1.8.0) - et-orbi (1.2.0) + et-orbi (1.2.1) tzinfo execjs (2.7.0) - factory_bot (4.8.2) - activesupport (>= 3.0.0) - factory_bot_rails (4.8.2) - factory_bot (~> 4.8.2) - railties (>= 3.0.0) - faker (1.8.7) + factory_bot (5.0.2) + activesupport (>= 4.2.0) + factory_bot_rails (5.0.2) + factory_bot (~> 5.0.2) + railties (>= 4.2.0) + faker (1.9.3) i18n (>= 0.7) faraday (0.15.4) multipart-post (>= 1.2, < 3) - ffi (1.10.0) + ffi (1.11.1) figaro (1.1.1) thor (~> 0.14) - fugit (1.2.0) + fugit (1.2.1) et-orbi (~> 1.1, >= 1.1.8) raabro (~> 1.1) gherkin (5.1.0) globalid (0.4.2) activesupport (>= 4.2.0) hammerjs-rails (2.0.8) - hashdiff (0.3.9) + hashdiff (0.4.0) hashie (3.6.0) httparty (0.13.7) json (~> 1.8) multi_xml (>= 0.5.2) i18n (1.6.0) concurrent-ruby (~> 1.0) - i18n-js (3.2.1) + i18n-js (3.3.0) i18n (>= 0.6.6) iniparse (1.4.4) jaro_winkler (1.5.2) - jbuilder (2.8.0) + jbuilder (2.9.1) activesupport (>= 4.2.0) - multi_json (>= 1.2) jmespath (1.4.0) jquery-rails (4.3.3) rails-dom-testing (>= 1, < 3) @@ -265,14 +270,14 @@ GEM js_cookie_rails (2.2.0) railties (>= 3.1) json (1.8.6) - json-jwt (1.10.0) + json-jwt (1.10.1) activesupport (>= 4.2) aes_key_wrap bindata json_matchers (0.11.0) json_schema - json_schema (0.20.4) - jsonapi-renderer (0.2.0) + json_schema (0.20.6) + jsonapi-renderer (0.2.1) jwt (1.5.6) kaminari (1.1.1) activesupport (>= 4.1.0) @@ -300,6 +305,8 @@ GEM nokogiri (>= 1.5.9) mail (2.7.1) mini_mime (>= 0.1.1) + marcel (0.3.3) + mimemagic (~> 0.3.2) method_source (0.9.2) mime-types (1.25.1) mimemagic (0.3.3) @@ -308,15 +315,16 @@ GEM minitest (5.11.3) momentjs-rails (2.17.1) railties (>= 3.1) + msgpack (1.2.10) multi_json (1.13.1) multi_test (0.1.2) multi_xml (0.6.0) - multipart-post (2.0.0) + multipart-post (2.1.1) nested_form_fields (0.8.2) coffee-rails (>= 3.2.1) jquery-rails rails (>= 3.2.0) - newrelic_rpm (6.2.0.354) + newrelic_rpm (6.4.0.356) nio4r (2.3.1) nokogiri (1.8.5) mini_portile2 (~> 2.3.0) @@ -337,7 +345,7 @@ GEM oauth2 (~> 1.1) omniauth (~> 1.9) orm_adapter (0.5.0) - overcommit (0.47.0) + overcommit (0.48.1) childprocess (~> 0.6, >= 0.6.3) iniparse (~> 1.4) paperclip (6.1.0) @@ -350,7 +358,7 @@ GEM parser (2.6.3.0) ast (~> 2.4.0) pg (0.21.0) - pg_search (2.1.7) + pg_search (2.2.0) activerecord (>= 4.2) activesupport (>= 4.2) phantomjs (2.1.1.0) @@ -367,7 +375,7 @@ GEM pry (~> 0.10) pry-rails (0.3.9) pry (>= 0.10.4) - public_suffix (3.0.3) + public_suffix (3.1.0) puma (3.12.1) raabro (1.1.6) rack (2.0.7) @@ -377,17 +385,18 @@ GEM rack rack-test (1.1.0) rack (>= 1.0, < 3) - rails (5.1.7) - actioncable (= 5.1.7) - actionmailer (= 5.1.7) - actionpack (= 5.1.7) - actionview (= 5.1.7) - activejob (= 5.1.7) - activemodel (= 5.1.7) - activerecord (= 5.1.7) - activesupport (= 5.1.7) + rails (5.2.3) + actioncable (= 5.2.3) + actionmailer (= 5.2.3) + actionpack (= 5.2.3) + actionview (= 5.2.3) + activejob (= 5.2.3) + activemodel (= 5.2.3) + activerecord (= 5.2.3) + activestorage (= 5.2.3) + activesupport (= 5.2.3) bundler (>= 1.3.0) - railties (= 5.1.7) + railties (= 5.2.3) sprockets-rails (>= 2.0.0) rails-controller-testing (1.0.4) actionpack (>= 5.0.1.x) @@ -405,21 +414,21 @@ GEM rails (> 3.1) rails_serve_static_assets (0.0.5) rails_stdout_logging (0.0.5) - railties (5.1.7) - actionpack (= 5.1.7) - activesupport (= 5.1.7) + railties (5.2.3) + actionpack (= 5.2.3) + activesupport (= 5.2.3) method_source rake (>= 0.8.7) - thor (>= 0.18.1, < 2.0) + thor (>= 0.19.0, < 2.0) rainbow (3.0.0) rake (12.3.2) rb-fsevent (0.10.3) rb-inotify (0.10.0) ffi (~> 1.0) rdoc (6.1.1) - recaptcha (4.14.0) + recaptcha (5.0.0) json - regexp_parser (1.4.0) + regexp_parser (1.5.1) responders (2.4.1) actionpack (>= 4.2.0, < 6.0) railties (>= 4.2.0, < 6.0) @@ -429,36 +438,36 @@ GEM roo (2.8.2) nokogiri (~> 1) rubyzip (>= 1.2.1, < 2.0.0) - rspec-core (3.7.1) - rspec-support (~> 3.7.0) - rspec-expectations (3.7.0) + rspec-core (3.8.0) + rspec-support (~> 3.8.0) + rspec-expectations (3.8.4) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.7.0) - rspec-mocks (3.7.0) + rspec-support (~> 3.8.0) + rspec-mocks (3.8.0) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.7.0) - rspec-rails (3.7.2) + rspec-support (~> 3.8.0) + rspec-rails (3.8.2) actionpack (>= 3.0) activesupport (>= 3.0) railties (>= 3.0) - rspec-core (~> 3.7.0) - rspec-expectations (~> 3.7.0) - rspec-mocks (~> 3.7.0) - rspec-support (~> 3.7.0) - rspec-support (3.7.0) - rubocop (0.68.0) + rspec-core (~> 3.8.0) + rspec-expectations (~> 3.8.0) + rspec-mocks (~> 3.8.0) + rspec-support (~> 3.8.0) + rspec-support (3.8.2) + rubocop (0.71.0) jaro_winkler (~> 1.5.1) parallel (~> 1.10) - parser (>= 2.5, != 2.5.1.1) + parser (>= 2.6) rainbow (>= 2.2.2, < 4.0) ruby-progressbar (~> 1.7) - unicode-display_width (>= 1.4.0, < 1.6) - rubocop-performance (1.2.0) + unicode-display_width (>= 1.4.0, < 1.7) + rubocop-performance (1.3.0) rubocop (>= 0.68.0) ruby-graphviz (1.2.4) - ruby-progressbar (1.10.0) + ruby-progressbar (1.10.1) ruby_dep (1.5.0) - rubyzip (1.2.2) + rubyzip (1.2.3) rufus-scheduler (3.6.0) fugit (~> 1.1, >= 1.1.6) safe_yaml (1.0.5) @@ -485,8 +494,8 @@ GEM railties (>= 4.0.0) sdoc (1.0.0) rdoc (>= 5.0) - shoulda-matchers (3.1.2) - activesupport (>= 4.0.0) + shoulda-matchers (4.1.0) + activesupport (>= 4.2.0) silencer (1.0.1) simple_token_authentication (1.15.1) actionmailer (>= 3.2.6, < 6) @@ -513,7 +522,7 @@ GEM thread_safe (0.3.6) tilt (2.0.9) timecop (0.9.1) - tinymce-rails (4.9.3) + tinymce-rails (4.9.4) railties (>= 3.1.1) turbolinks (5.1.1) turbolinks-source (~> 5.1) @@ -523,24 +532,24 @@ GEM uglifier (4.1.20) execjs (>= 0.3.0, < 3) underscore-rails (1.8.3) - unicode-display_width (1.5.0) + unicode-display_width (1.6.0) uniform_notifier (1.12.1) warden (1.2.8) rack (>= 2.0.6) - webmock (3.5.1) + webmock (3.6.0) addressable (>= 2.3.6) crack (>= 0.3.2) - hashdiff + hashdiff (>= 0.4.0, < 2.0.0) webpacker (3.6.0) activesupport (>= 4.2) rack-proxy (>= 0.6.1) railties (>= 4.2) - websocket-driver (0.6.5) + websocket-driver (0.7.1) websocket-extensions (>= 0.1.0) - websocket-extensions (0.1.3) + websocket-extensions (0.1.4) whacamole (1.2.0) wicked_pdf (1.1.0) - wkhtmltopdf-heroku (2.12.4.0) + wkhtmltopdf-heroku (2.12.5.0) xpath (3.2.0) nokogiri (~> 1.8) yomu (0.2.4) @@ -563,6 +572,7 @@ DEPENDENCIES bcrypt (~> 3.1.10) better_errors binding_of_caller + bootsnap bootstrap-sass (~> 3.3.7) bootstrap-select-rails (~> 1.12.4) bootstrap3-datetimepicker-rails (~> 4.15.35) @@ -617,7 +627,7 @@ DEPENDENCIES pry-rails puma rack-attack - rails (~> 5.1.7) + rails (~> 5.2.3) rails-controller-testing rails_12factor rails_autolink (~> 1.1, >= 1.1.6) diff --git a/bin/bundle b/bin/bundle index 66e9889e8..f19acf5b5 100755 --- a/bin/bundle +++ b/bin/bundle @@ -1,3 +1,3 @@ #!/usr/bin/env ruby -ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__) +ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__) load Gem.bin_path('bundler', 'bundle') diff --git a/bin/setup b/bin/setup index 78c4e861d..94fd4d797 100755 --- a/bin/setup +++ b/bin/setup @@ -1,10 +1,9 @@ #!/usr/bin/env ruby -require 'pathname' require 'fileutils' include FileUtils # path to your application root. -APP_ROOT = Pathname.new File.expand_path('../../', __FILE__) +APP_ROOT = File.expand_path('..', __dir__) def system!(*args) system(*args) || abort("\n== Command #{args} failed ==") @@ -21,7 +20,6 @@ chdir APP_ROOT do # Install JavaScript dependencies if using Yarn # system('bin/yarn') - # puts "\n== Copying sample files ==" # unless File.exist?('config/database.yml') # cp 'config/database.yml.sample', 'config/database.yml' diff --git a/bin/update b/bin/update index a8e4462f2..58bfaed51 100755 --- a/bin/update +++ b/bin/update @@ -1,10 +1,9 @@ #!/usr/bin/env ruby -require 'pathname' require 'fileutils' include FileUtils # path to your application root. -APP_ROOT = Pathname.new File.expand_path('../../', __FILE__) +APP_ROOT = File.expand_path('..', __dir__) def system!(*args) system(*args) || abort("\n== Command #{args} failed ==") @@ -18,6 +17,9 @@ chdir APP_ROOT do system! 'gem install bundler --conservative' system('bundle check') || system!('bundle install') + # Install JavaScript dependencies if using Yarn + # system('bin/yarn') + puts "\n== Updating database ==" system! 'bin/rails db:migrate' diff --git a/bin/yarn b/bin/yarn index c2bacef83..460dd565b 100755 --- a/bin/yarn +++ b/bin/yarn @@ -1,8 +1,8 @@ #!/usr/bin/env ruby -VENDOR_PATH = File.expand_path('..', __dir__) -Dir.chdir(VENDOR_PATH) do +APP_ROOT = File.expand_path('..', __dir__) +Dir.chdir(APP_ROOT) do begin - exec "yarnpkg #{ARGV.join(" ")}" + exec "yarnpkg", *ARGV rescue Errno::ENOENT $stderr.puts "Yarn executable was not detected in the system." $stderr.puts "Download Yarn at https://yarnpkg.com/en/docs/install" diff --git a/config/boot.rb b/config/boot.rb index 30f5120df..b9e460cef 100644 --- a/config/boot.rb +++ b/config/boot.rb @@ -1,3 +1,4 @@ ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__) require 'bundler/setup' # Set up gems listed in the Gemfile. +require 'bootsnap/setup' # Speed up boot time by caching expensive operations. diff --git a/config/cable.yml b/config/cable.yml index e93e8dfed..e81931f2c 100644 --- a/config/cable.yml +++ b/config/cable.yml @@ -6,5 +6,5 @@ test: production: adapter: redis - url: redis://localhost:6379/1 + url: <%= ENV.fetch("REDIS_URL") { "redis://localhost:6379/1" } %> channel_prefix: scinote_production diff --git a/config/environments/development.rb b/config/environments/development.rb index 22df30016..345984ca4 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -27,8 +27,6 @@ Rails.application.configure do from: Rails.application.secrets.mailer_from, reply_to: Rails.application.secrets.mailer_reply_to } - config.action_mailer.raise_delivery_errors = false - config.action_mailer.perform_caching = false if ENV['CUCUMBER'] == 'cucumber' config.action_mailer.delivery_method = :test @@ -57,6 +55,14 @@ Rails.application.configure do password: Rails.application.secrets.mailer_password } + # Store uploaded files on the local file system (see config/storage.yml for options) + config.active_storage.service = ENV['ACTIVESTORAGE_SERVICE'] || :local + + # Don't care if the mailer can't send. + config.action_mailer.raise_delivery_errors = false + + config.action_mailer.perform_caching = false + # Print deprecation notices to the Rails logger. config.active_support.deprecation = :log @@ -65,6 +71,9 @@ Rails.application.configure do # Raise an error on page load if there are pending migrations. config.active_record.migration_error = :page_load + # Highlight code that triggered database queries in logs. + config.active_record.verbose_query_logs = true + # Debug mode disables concatenation and preprocessing of assets. # This option may cause significant delays in view rendering with a large # number of complex assets. @@ -86,6 +95,9 @@ Rails.application.configure do # one you use and put it into application.yml! BetterErrors::Middleware.allow_ip! ENV['TRUSTED_IP'] if ENV['TRUSTED_IP'] + # Suppress logger output for asset requests. + config.assets.quiet = false + # Raises error for missing translations config.action_view.raise_on_missing_translations = true @@ -108,11 +120,7 @@ Rails.application.configure do # Use an evented file watcher to asynchronously detect changes in source code, # routes, locales, etc. This feature depends on the listen gem. - #config.file_watcher = ActiveSupport::EventedFileUpdateChecker - - # Suppress logger output for asset requests. By default logger output is - # enabled. - # config.assets.quiet = true + config.file_watcher = ActiveSupport::EventedFileUpdateChecker # Enable/disable caching. By default caching is disabled. if Rails.root.join('tmp/caching-dev.txt').exist? diff --git a/config/environments/production.rb b/config/environments/production.rb index 600a005b6..5df50d3fc 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -42,44 +42,44 @@ Rails.application.configure do password: Rails.application.secrets.mailer_password } #config.action_mailer.perform_deliveries = false + # Ensures that a master key has been made available in either ENV["RAILS_MASTER_KEY"] + # or in config/master.key. This key is used to decrypt credentials (and other encrypted files). + # config.require_master_key = true - # Enable Rack::Cache to put a simple HTTP cache in front of your application - # Add `rack-cache` to your Gemfile before enabling this. - # For large-scale production use, consider using a caching reverse proxy like - # NGINX, varnish or squid. - # config.action_dispatch.rack_cache = true # Compress JavaScripts and CSS. - config.assets.js_compressor = Uglifier.new(harmony: true) + config.assets.js_compressor = :uglifier # config.assets.css_compressor = :sass # Do not fallback to assets pipeline if a precompiled asset is missed. config.assets.compile = false - # Asset digests allow you to set far-future HTTP expiration dates on all assets, - # yet still be able to expire them through the digest params. - config.assets.digest = true - # `config.assets.precompile` and `config.assets.version` have moved to config/initializers/assets.rb # Specifies the header that your server uses for sending files. # config.action_dispatch.x_sendfile_header = 'X-Sendfile' # for Apache # config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for NGINX + # Store uploaded files on the local file system (see config/storage.yml for options) + config.active_storage.service = ENV['ACTIVESTORAGE_SERVICE'] || :local + + # Mount Action Cable outside main process or domain + # config.action_cable.mount_path = nil + # config.action_cable.url = 'wss://example.com/cable' + # config.action_cable.allowed_request_origins = [ 'http://example.com', /http:\/\/example.*/ ] + # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies. config.force_ssl = ENV['RAILS_FORCE_SSL'].present? - # Display info and higher on production. - config.log_level = :info + # Use the lowest log level to ensure availability of diagnostic information + # when problems arise. + config.log_level = :debug # Prepend all log lines with the following tags. - # config.log_tags = [ :subdomain, :uuid ] + config.log_tags = [ :request_id ] + - # Use a different logger for distributed setups. - # config.logger = ActiveSupport::TaggedLogging.new(SyslogLogger.new) - # Enable serving of images, stylesheets, and JavaScripts from an asset server. - # config.action_controller.asset_host = 'http://assets.example.com' # Ignore bad email addresses and do not raise email delivery errors. # Set this to true and configure the email server for immediate delivery to raise delivery errors. @@ -87,7 +87,7 @@ Rails.application.configure do # Enable locale fallbacks for I18n (makes lookups for any locale fall back to # the I18n.default_locale when a translation cannot be found). - config.i18n.fallbacks = [I18n.default_locale] + config.i18n.fallbacks = true # Send deprecation notices to registered listeners. config.active_support.deprecation = :notify @@ -119,7 +119,7 @@ Rails.application.configure do # require 'syslog/logger' # config.logger = ActiveSupport::TaggedLogging.new(Syslog::Logger.new 'app-name') - if ENV['RAILS_LOG_TO_STDOUT'].present? + if ENV["RAILS_LOG_TO_STDOUT"].present? logger = ActiveSupport::Logger.new(STDOUT) logger.formatter = config.log_formatter config.logger = ActiveSupport::TaggedLogging.new(logger) @@ -132,30 +132,12 @@ Rails.application.configure do config.active_job.queue_adapter = :delayed_job config.active_job.queue_name_prefix = "scinote_#{Rails.env}" config.action_mailer.perform_caching = false - - # `config.assets.precompile` and `config.assets.version` have moved to config/initializers/assets.rb - # Enable serving of images, stylesheets, and JavaScripts from an asset server. # config.action_controller.asset_host = 'http://assets.example.com' - # Specifies the header that your server uses for sending files. - # config.action_dispatch.x_sendfile_header = 'X-Sendfile' # for Apache - # config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for NGINX - - # Mount Action Cable outside main process or domain - # config.action_cable.mount_path = nil - # config.action_cable.url = 'wss://example.com/cable' - # config.action_cable.allowed_request_origins = [ 'http://example.com', /http:\/\/example.*/ ] - # Disable serving static files from the `/public` folder by default since # Apache or NGINX already handles this. config.public_file_server.enabled = ENV['RAILS_SERVE_STATIC_FILES'].present? - - # Attempt to read encrypted secrets from `config/secrets.yml.enc`. - # Requires an encryption key in `ENV["RAILS_MASTER_KEY"]` or - # `config/secrets.yml.key`. - config.read_encrypted_secrets = true - # Enable new team on sign up new_team_on_signup = ENV['NEW_TEAM_ON_SIGNUP'] || 'true' if new_team_on_signup == 'true' diff --git a/config/environments/test.rb b/config/environments/test.rb index e88351ccb..bedcd5046 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -43,6 +43,11 @@ Rails.application.configure do # Disable request forgery protection in test environment. config.action_controller.allow_forgery_protection = false + # Store uploaded files on the local file system in a temporary directory + config.active_storage.service = :test + + config.action_mailer.perform_caching = false + # Tell Action Mailer not to deliver emails to the real world. # The :test delivery method accumulates sent emails in the # ActionMailer::Base.deliveries array. diff --git a/config/initializers/application_controller_renderer.rb b/config/initializers/application_controller_renderer.rb index 51639b67a..89d2efab2 100644 --- a/config/initializers/application_controller_renderer.rb +++ b/config/initializers/application_controller_renderer.rb @@ -1,6 +1,8 @@ # Be sure to restart your server when you modify this file. -# ApplicationController.renderer.defaults.merge!( -# http_host: 'example.org', -# https: false -# ) +# ActiveSupport::Reloader.to_prepare do +# ApplicationController.renderer.defaults.merge!( +# http_host: 'example.org', +# https: false +# ) +# end diff --git a/config/initializers/content_security_policy.rb b/config/initializers/content_security_policy.rb new file mode 100644 index 000000000..d3bcaa5ec --- /dev/null +++ b/config/initializers/content_security_policy.rb @@ -0,0 +1,25 @@ +# Be sure to restart your server when you modify this file. + +# Define an application-wide content security policy +# For further information see the following documentation +# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy + +# Rails.application.config.content_security_policy do |policy| +# policy.default_src :self, :https +# policy.font_src :self, :https, :data +# policy.img_src :self, :https, :data +# policy.object_src :none +# policy.script_src :self, :https +# policy.style_src :self, :https + +# # Specify URI for violation reports +# # policy.report_uri "/csp-violation-report-endpoint" +# end + +# If you are using UJS then enable automatic nonce generation +# Rails.application.config.content_security_policy_nonce_generator = -> request { SecureRandom.base64(16) } + +# Report CSP violations to a specified URI +# For further information see the following documentation: +# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy-Report-Only +# Rails.application.config.content_security_policy_report_only = true diff --git a/config/initializers/new_framework_defaults_5_2.rb b/config/initializers/new_framework_defaults_5_2.rb new file mode 100644 index 000000000..c383d072b --- /dev/null +++ b/config/initializers/new_framework_defaults_5_2.rb @@ -0,0 +1,38 @@ +# Be sure to restart your server when you modify this file. +# +# This file contains migration options to ease your Rails 5.2 upgrade. +# +# Once upgraded flip defaults one by one to migrate to the new default. +# +# Read the Guide for Upgrading Ruby on Rails for more info on each option. + +# Make Active Record use stable #cache_key alongside new #cache_version method. +# This is needed for recyclable cache keys. +# Rails.application.config.active_record.cache_versioning = true + +# Use AES-256-GCM authenticated encryption for encrypted cookies. +# Also, embed cookie expiry in signed or encrypted cookies for increased security. +# +# This option is not backwards compatible with earlier Rails versions. +# It's best enabled when your entire app is migrated and stable on 5.2. +# +# Existing cookies will be converted on read then written with the new scheme. +# Rails.application.config.action_dispatch.use_authenticated_cookie_encryption = true + +# Use AES-256-GCM authenticated encryption as default cipher for encrypting messages +# instead of AES-256-CBC, when use_authenticated_message_encryption is set to true. +# Rails.application.config.active_support.use_authenticated_message_encryption = true + +# Add default protection from forgery to ActionController::Base instead of in +# ApplicationController. +# Rails.application.config.action_controller.default_protect_from_forgery = true + +# Store boolean values are in sqlite3 databases as 1 and 0 instead of 't' and +# 'f' after migrating old data. +# Rails.application.config.active_record.sqlite3.represent_boolean_as_integer = true + +# Use SHA-1 instead of MD5 to generate non-sensitive digests, such as the ETag header. +# Rails.application.config.active_support.use_sha1_digests = true + +# Make `form_with` generate id attributes for any generated HTML tags. +# Rails.application.config.action_view.form_with_generates_ids = true diff --git a/config/storage.yml b/config/storage.yml new file mode 100644 index 000000000..d32f76e8f --- /dev/null +++ b/config/storage.yml @@ -0,0 +1,34 @@ +test: + service: Disk + root: <%= Rails.root.join("tmp/storage") %> + +local: + service: Disk + root: <%= Rails.root.join("storage") %> + +# Use rails credentials:edit to set the AWS secrets (as aws:access_key_id|secret_access_key) +# amazon: +# service: S3 +# access_key_id: <%= Rails.application.credentials.dig(:aws, :access_key_id) %> +# secret_access_key: <%= Rails.application.credentials.dig(:aws, :secret_access_key) %> +# region: us-east-1 +# bucket: your_own_bucket + +# Remember not to checkin your GCS keyfile to a repository +# google: +# service: GCS +# project: your_project +# credentials: <%= Rails.root.join("path/to/gcs.keyfile") %> +# bucket: your_own_bucket + +# Use rails credentials:edit to set the Azure Storage secret (as azure_storage:storage_access_key) +# microsoft: +# service: AzureStorage +# storage_account_name: your_account_name +# storage_access_key: <%= Rails.application.credentials.dig(:azure_storage, :storage_access_key) %> +# container: your_container_name + +# mirror: +# service: Mirror +# primary: local +# mirrors: [ amazon, google, microsoft ] diff --git a/db/migrate/20190613094834_create_active_storage_tables.active_storage.rb b/db/migrate/20190613094834_create_active_storage_tables.active_storage.rb new file mode 100644 index 000000000..0b2ce257c --- /dev/null +++ b/db/migrate/20190613094834_create_active_storage_tables.active_storage.rb @@ -0,0 +1,27 @@ +# This migration comes from active_storage (originally 20170806125915) +class CreateActiveStorageTables < ActiveRecord::Migration[5.2] + def change + create_table :active_storage_blobs do |t| + t.string :key, null: false + t.string :filename, null: false + t.string :content_type + t.text :metadata + t.bigint :byte_size, null: false + t.string :checksum, null: false + t.datetime :created_at, null: false + + t.index [ :key ], unique: true + end + + create_table :active_storage_attachments do |t| + t.string :name, null: false + t.references :record, null: false, polymorphic: true, index: false + t.references :blob, null: false + + t.datetime :created_at, null: false + + t.index [ :record_type, :record_id, :name, :blob_id ], name: "index_active_storage_attachments_uniqueness", unique: true + t.foreign_key :active_storage_blobs, column: :blob_id + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 42cf7ba6d..d2216eae0 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,12 +10,33 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20190520135317) do +ActiveRecord::Schema.define(version: 2019_06_13_094834) do # These are extensions that must be enabled in order to support this database - enable_extension "plpgsql" - enable_extension "pg_trgm" enable_extension "btree_gist" + enable_extension "pg_trgm" + enable_extension "plpgsql" + + create_table "active_storage_attachments", force: :cascade do |t| + t.string "name", null: false + t.string "record_type", null: false + t.bigint "record_id", null: false + t.bigint "blob_id", null: false + t.datetime "created_at", null: false + t.index ["blob_id"], name: "index_active_storage_attachments_on_blob_id" + t.index ["record_type", "record_id", "name", "blob_id"], name: "index_active_storage_attachments_uniqueness", unique: true + end + + create_table "active_storage_blobs", force: :cascade do |t| + t.string "key", null: false + t.string "filename", null: false + t.string "content_type" + t.text "metadata" + t.bigint "byte_size", null: false + t.string "checksum", null: false + t.datetime "created_at", null: false + t.index ["key"], name: "index_active_storage_blobs_on_key", unique: true + end create_table "activities", force: :cascade do |t| t.bigint "my_module_id" @@ -56,7 +77,7 @@ ActiveRecord::Schema.define(version: 20190520135317) do t.datetime "updated_at", null: false t.string "file_file_name" t.string "file_content_type" - t.integer "file_file_size" + t.bigint "file_file_size" t.datetime "file_updated_at" t.bigint "created_by_id" t.bigint "last_modified_by_id" @@ -169,7 +190,7 @@ ActiveRecord::Schema.define(version: 20190520135317) do t.datetime "updated_at", null: false t.string "workflowimg_file_name" t.string "workflowimg_content_type" - t.integer "workflowimg_file_size" + t.bigint "workflowimg_file_size" t.datetime "workflowimg_updated_at" t.uuid "uuid" t.index ["archived_by_id"], name: "index_experiments_on_archived_by_id" @@ -737,14 +758,14 @@ ActiveRecord::Schema.define(version: 20190520135317) do t.datetime "updated_at", null: false t.string "file_file_name" t.string "file_content_type" - t.integer "file_file_size" + t.bigint "file_file_size" t.datetime "file_updated_at" end create_table "tiny_mce_assets", force: :cascade do |t| t.string "image_file_name" t.string "image_content_type" - t.integer "image_file_size" + t.bigint "image_file_size" t.datetime "image_updated_at" t.integer "estimated_size", default: 0, null: false t.integer "step_id" @@ -857,7 +878,7 @@ ActiveRecord::Schema.define(version: 20190520135317) do t.datetime "updated_at", null: false t.string "avatar_file_name" t.string "avatar_content_type" - t.integer "avatar_file_size" + t.bigint "avatar_file_size" t.datetime "avatar_updated_at" t.string "confirmation_token" t.datetime "confirmed_at" @@ -924,12 +945,13 @@ ActiveRecord::Schema.define(version: 20190520135317) do t.datetime "updated_at", null: false t.string "zip_file_file_name" t.string "zip_file_content_type" - t.integer "zip_file_file_size" + t.bigint "zip_file_file_size" t.datetime "zip_file_updated_at" t.string "type" t.index ["user_id"], name: "index_zip_exports_on_user_id" end + add_foreign_key "active_storage_attachments", "active_storage_blobs", column: "blob_id" add_foreign_key "activities", "experiments" add_foreign_key "activities", "my_modules" add_foreign_key "activities", "projects" diff --git a/docker-compose.yml b/docker-compose.yml index ab06fd129..bfbdcf1a6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -44,6 +44,7 @@ services: bash -c "./bin/webpack-dev-server" environment: - RAILS_ENV=development + - WORKER=true volumes: - .:/usr/src/app - scinote_development_bundler:/usr/local/bundle/ diff --git a/spec/factories/activities.rb b/spec/factories/activities.rb index 75e833b81..690a3347f 100644 --- a/spec/factories/activities.rb +++ b/spec/factories/activities.rb @@ -2,8 +2,8 @@ FactoryBot.define do factory :activity do - type_of :create_project - message Faker::Lorem.sentence(10) + type_of { :create_project } + message { Faker::Lorem.sentence(10) } subject { create :project } owner { create :user } team diff --git a/spec/factories/asset_text_datums.rb b/spec/factories/asset_text_datums.rb index 749bfe673..34c364647 100644 --- a/spec/factories/asset_text_datums.rb +++ b/spec/factories/asset_text_datums.rb @@ -2,7 +2,7 @@ FactoryBot.define do factory :asset_text_datum do - data "Sample name\tSample type\n" + "sample6\tsample\n" + "\n" + data { "Sample name\tSample type\n" + "sample6\tsample\n" + "\n" } asset end end diff --git a/spec/factories/assets.rb b/spec/factories/assets.rb index 3ea91dfa6..ae9088e9b 100644 --- a/spec/factories/assets.rb +++ b/spec/factories/assets.rb @@ -2,11 +2,11 @@ FactoryBot.define do factory :asset do - file_file_name 'sample_file.txt' - file_content_type 'text/plain' - file_file_size 69 - version 1 - estimated_size 232 - file_processing false + file_file_name { 'sample_file.txt' } + file_content_type { 'text/plain' } + file_file_size { 69 } + version { 1 } + estimated_size { 232 } + file_processing { false } end end diff --git a/spec/factories/custom_fields.rb b/spec/factories/custom_fields.rb index 20f9a0acc..5b6656675 100644 --- a/spec/factories/custom_fields.rb +++ b/spec/factories/custom_fields.rb @@ -2,7 +2,7 @@ FactoryBot.define do factory :custom_field do - name 'My custom field' + name { 'My custom field' } user team end diff --git a/spec/factories/notifications.rb b/spec/factories/notifications.rb index bfefcfcf8..5ca75eeec 100644 --- a/spec/factories/notifications.rb +++ b/spec/factories/notifications.rb @@ -2,9 +2,9 @@ FactoryBot.define do factory :notification do - title 'Admin was added as Owner to project ' \ - 'Demo project - qPCR by User.' - message 'Project: Demo project - qPCR' - type_of 'assignment' + title { 'Admin was added as Owner to project ' \ + 'Demo project - qPCR by User.' } + message { 'Project: Demo project - qPCR' } + type_of { 'assignment' } end end diff --git a/spec/factories/repository_columns.rb b/spec/factories/repository_columns.rb index b120b555e..7c47ed993 100644 --- a/spec/factories/repository_columns.rb +++ b/spec/factories/repository_columns.rb @@ -5,22 +5,22 @@ FactoryBot.define do sequence(:name) { |n| "My column-#{n}" } created_by { create :user } repository - data_type :RepositoryTextValue + data_type { :RepositoryTextValue } trait :text_type do - data_type :RepositoryTextValue + data_type { :RepositoryTextValue } end trait :date_type do - data_type :RepositoryDateValue + data_type { :RepositoryDateValue } end trait :list_type do - data_type :RepositoryListValue + data_type { :RepositoryListValue } end trait :asset_type do - data_type :RepositoryAssetValue + data_type { :RepositoryAssetValue } end end end diff --git a/spec/factories/sample_groups.rb b/spec/factories/sample_groups.rb index 884e48e29..8f125787b 100644 --- a/spec/factories/sample_groups.rb +++ b/spec/factories/sample_groups.rb @@ -3,7 +3,7 @@ FactoryBot.define do factory :sample_group do name { Faker::Name.unique.name } - color Faker::Color.hex_color + color { Faker::Color.hex_color } team end end diff --git a/spec/factories/teams.rb b/spec/factories/teams.rb index ea7e7aca3..1010b7eab 100644 --- a/spec/factories/teams.rb +++ b/spec/factories/teams.rb @@ -6,8 +6,8 @@ FactoryBot.define do sequence(:name) { |n| "My team-#{n}" } description { Faker::Lorem.sentence } space_taken { 1048576 } - without_templates true - without_intro_demo true + without_templates { true } + without_intro_demo { true } trait :with_members do users { create_list :user, 3 } end diff --git a/spec/factories/tinymce_assets.rb b/spec/factories/tinymce_assets.rb index fd4ea64de..dbd07ffc2 100644 --- a/spec/factories/tinymce_assets.rb +++ b/spec/factories/tinymce_assets.rb @@ -3,8 +3,8 @@ FactoryBot.define do factory :tiny_mce_asset do association :team, factory: :team - image_file_name 'sample_file.jpg' - image_content_type 'image/jpeg' - image_file_size 69 + image_file_name { 'sample_file.jpg' } + image_content_type { 'image/jpeg' } + image_file_size { 69 } end end diff --git a/spec/factories/user_identities.rb b/spec/factories/user_identities.rb index c33e17570..a3bac238c 100644 --- a/spec/factories/user_identities.rb +++ b/spec/factories/user_identities.rb @@ -3,7 +3,7 @@ FactoryBot.define do factory :user_identity do user - uid Faker::Crypto.unique.sha1 - provider Faker::App.unique.name + uid { Faker::Crypto.unique.sha1 } + provider { Faker::App.unique.name } end end diff --git a/spec/factories/user_notifications.rb b/spec/factories/user_notifications.rb index 579e8cdf8..4e2f98e09 100644 --- a/spec/factories/user_notifications.rb +++ b/spec/factories/user_notifications.rb @@ -2,6 +2,6 @@ FactoryBot.define do factory :user_notification do - checked false + checked { false } end end diff --git a/spec/factories/user_teams.rb b/spec/factories/user_teams.rb index c13c75db5..bb5b9c3b6 100644 --- a/spec/factories/user_teams.rb +++ b/spec/factories/user_teams.rb @@ -5,13 +5,13 @@ FactoryBot.define do user team trait :admin do - role 'admin' + role { 'admin' } end trait :guest do - role 'guest' + role { 'guest' } end trait :normal_user do # default enum by DB - role 'normal_user' + role { 'normal_user' } end end end diff --git a/spec/factories/users.rb b/spec/factories/users.rb index efff4c751..0a4bde69a 100644 --- a/spec/factories/users.rb +++ b/spec/factories/users.rb @@ -2,11 +2,11 @@ FactoryBot.define do factory :user do - full_name 'admin' - initials 'AD' + full_name { 'admin' } + initials { 'AD' } sequence(:email) { |n| "user-#{n}@example.com" } - password 'asdf1243' - password_confirmation 'asdf1243' - current_sign_in_at DateTime.now + password { 'asdf1243' } + password_confirmation { 'asdf1243' } + current_sign_in_at { DateTime.now } end end diff --git a/spec/factories/wopi_apps.rb b/spec/factories/wopi_apps.rb index 067f3c94b..a4cc87137 100644 --- a/spec/factories/wopi_apps.rb +++ b/spec/factories/wopi_apps.rb @@ -3,7 +3,7 @@ FactoryBot.define do factory :wopi_app do sequence(:name) { |n| "WopiApp-#{n}" } - icon 'app-icon' + icon { 'app-icon' } wopi_discovery end end From ba2daede0b916d6b5a34b66d424f97940463065a Mon Sep 17 00:00:00 2001 From: Oleksii Kriuchykhin Date: Mon, 24 Jun 2019 15:17:42 +0200 Subject: [PATCH 2/3] Add migration scripts [SCI-3539] --- .gitignore | 3 + Dockerfile | 2 +- Dockerfile.production | 2 +- Gemfile | 4 +- Gemfile.lock | 11 +- app/models/asset.rb | 83 +++++---- config/storage.yml | 14 +- ...0190613134100_convert_to_active_storage.rb | 117 ++++++++++++ db/schema.rb | 2 +- .../service/custom_s3_service.rb | 172 ++++++++++++++++++ lib/tasks/active_storage.rake | 69 +++++++ 11 files changed, 433 insertions(+), 46 deletions(-) create mode 100644 db/migrate/20190613134100_convert_to_active_storage.rb create mode 100644 lib/active_storage/service/custom_s3_service.rb create mode 100644 lib/tasks/active_storage.rake diff --git a/.gitignore b/.gitignore index c75412598..c0f9f219b 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,9 @@ ehthumbs.db # Ignore temporary files public/system/* +# Ignore ActiveStorage Disc service storage directory +storage/ + # Ignore robots.txt public/robots.txt diff --git a/Dockerfile b/Dockerfile index a653a486e..4b13d676d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ruby:2.5.5 +FROM ruby:2.6.3 MAINTAINER BioSistemika # Get version of Debian (lsb_release substitute) and save it to /tmp/lsb_release for further commands diff --git a/Dockerfile.production b/Dockerfile.production index ca922f441..533c1e64f 100644 --- a/Dockerfile.production +++ b/Dockerfile.production @@ -1,4 +1,4 @@ -FROM ruby:2.5.5 +FROM ruby:2.6.3 MAINTAINER BioSistemika RUN echo deb "http://http.debian.net/debian stretch-backports main" >> /etc/apt/sources.list diff --git a/Gemfile b/Gemfile index 4b7c719bd..4946767cd 100644 --- a/Gemfile +++ b/Gemfile @@ -2,7 +2,7 @@ source 'http://rubygems.org' -ruby '2.5.5' +ruby '2.6.3' gem 'bootsnap', require: false @@ -80,7 +80,9 @@ gem 'underscore-rails' gem 'wicked_pdf', '~> 1.1.0' gem 'wkhtmltopdf-heroku' +gem 'aws-sdk-rails' gem 'aws-sdk-s3' +gem 'mini_magick' gem 'paperclip', '~> 6.1' # File attachment, image attachment library gem 'delayed_job_active_record' gem 'devise-async', diff --git a/Gemfile.lock b/Gemfile.lock index ac5cda6f2..e2e5e6687 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -115,10 +115,16 @@ GEM aws-sdk-kms (1.21.0) aws-sdk-core (~> 3, >= 3.53.0) aws-sigv4 (~> 1.1) + aws-sdk-rails (2.1.0) + aws-sdk-ses (~> 1) + railties (>= 3) aws-sdk-s3 (1.42.0) aws-sdk-core (~> 3, >= 3.53.0) aws-sdk-kms (~> 1) aws-sigv4 (~> 1.1) + aws-sdk-ses (1.22.0) + aws-sdk-core (~> 3, >= 3.53.0) + aws-sigv4 (~> 1.1) aws-sigv4 (1.1.0) aws-eventstream (~> 1.0, >= 1.0.2) backports (3.15.0) @@ -310,6 +316,7 @@ GEM method_source (0.9.2) mime-types (1.25.1) mimemagic (0.3.3) + mini_magick (4.9.3) mini_mime (1.0.1) mini_portile2 (2.3.0) minitest (5.11.3) @@ -567,6 +574,7 @@ DEPENDENCIES auto_strip_attributes (~> 2.1) autosize-rails awesome_print + aws-sdk-rails aws-sdk-s3 base62 bcrypt (~> 3.1.10) @@ -610,6 +618,7 @@ DEPENDENCIES kaminari listen (~> 3.0) logging (~> 2.0.0) + mini_magick momentjs-rails (~> 2.17.1) nested_form_fields newrelic_rpm @@ -664,7 +673,7 @@ DEPENDENCIES yomu RUBY VERSION - ruby 2.5.5p157 + ruby 2.6.3p62 BUNDLED WITH 1.17.3 diff --git a/app/models/asset.rb b/app/models/asset.rb index 6d7b332e6..ff78b324e 100644 --- a/app/models/asset.rb +++ b/app/models/asset.rb @@ -6,48 +6,51 @@ class Asset < ApplicationRecord require 'tempfile' # Lock duration set to 30 minutes - LOCK_DURATION = 60*30 + LOCK_DURATION = 60 * 30 + + # ActiveStorage configuration + has_one_attached :file # Paperclip validation - has_attached_file :file, - styles: lambda { |a| - if a.previewable_document? - { - large: { processors: [:custom_file_preview], - geometry: Constants::LARGE_PIC_FORMAT, - format: :jpg }, - medium: { processors: [:custom_file_preview], - geometry: Constants::MEDIUM_PIC_FORMAT, - format: :jpg } - } - else - { - large: [Constants::LARGE_PIC_FORMAT, :jpg], - medium: [Constants::MEDIUM_PIC_FORMAT, :jpg] - } - end - }, - convert_options: { - medium: '-quality 70 -strip', - all: '-background "#d2d2d2" -flatten +matte' - } + # has_attached_file :file, + # styles: lambda { |a| + # if a.previewable_document? + # { + # large: { processors: [:custom_file_preview], + # geometry: Constants::LARGE_PIC_FORMAT, + # format: :jpg }, + # medium: { processors: [:custom_file_preview], + # geometry: Constants::MEDIUM_PIC_FORMAT, + # format: :jpg } + # } + # else + # { + # large: [Constants::LARGE_PIC_FORMAT, :jpg], + # medium: [Constants::MEDIUM_PIC_FORMAT, :jpg] + # } + # end + # }, + # convert_options: { + # medium: '-quality 70 -strip', + # all: '-background "#d2d2d2" -flatten +matte' + # } - before_post_process :previewable? - before_post_process :extract_image_quality + # before_post_process :previewable? + # before_post_process :extract_image_quality # adds image processing in background job - process_in_background :file, processing_image_url: '/images/:style/processing.gif' + # process_in_background :file, processing_image_url: '/images/:style/processing.gif' - validates_attachment :file, - presence: true, - size: { - less_than: Rails.configuration.x.file_max_size_mb.megabytes - } - validates :estimated_size, presence: true - validates :file_present, inclusion: { in: [true, false] } + # validates_attachment :file, + # presence: true, + # size: { + # less_than: Rails.configuration.x.file_max_size_mb.megabytes + # } + # validates :estimated_size, presence: true + # validates :file_present, inclusion: { in: [true, false] } # Should be checked for any security leaks - do_not_validate_attachment_file_type :file + # do_not_validate_attachment_file_type :file # Asset validation # This could cause some problems if you create empty asset and want to @@ -200,6 +203,18 @@ class Asset < ApplicationRecord end end + def medium_preview + file.variant(resize: Constants::MEDIUM_PIC_FORMAT) + end + + def large_preview + file.variant(resize: Constants::LARGE_PIC_FORMAT) + end + + def file_size + file.blob.byte_size + end + def extract_image_quality return unless ['image/jpeg', 'image/pjpeg'].include? file_content_type diff --git a/config/storage.yml b/config/storage.yml index d32f76e8f..fa3150876 100644 --- a/config/storage.yml +++ b/config/storage.yml @@ -6,13 +6,13 @@ local: service: Disk root: <%= Rails.root.join("storage") %> -# Use rails credentials:edit to set the AWS secrets (as aws:access_key_id|secret_access_key) -# amazon: -# service: S3 -# access_key_id: <%= Rails.application.credentials.dig(:aws, :access_key_id) %> -# secret_access_key: <%= Rails.application.credentials.dig(:aws, :secret_access_key) %> -# region: us-east-1 -# bucket: your_own_bucket +amazon: + service: CustomS3 + access_key_id: <%= ENV["AWS_ACCESS_KEY_ID"] %> + secret_access_key: <%= ENV["AWS_SECRET_ACCESS_KEY"] %> + region: <%= ENV["AWS_REGION"] %> + bucket: <%= ENV["S3_BUCKET"] %> + subfolder: <%= ENV["S3_SUBFOLDER"] %> # Remember not to checkin your GCS keyfile to a repository # google: diff --git a/db/migrate/20190613134100_convert_to_active_storage.rb b/db/migrate/20190613134100_convert_to_active_storage.rb new file mode 100644 index 000000000..ea9de35b3 --- /dev/null +++ b/db/migrate/20190613134100_convert_to_active_storage.rb @@ -0,0 +1,117 @@ +# frozen_string_literal: true + +class ConvertToActiveStorage < ActiveRecord::Migration[5.2] + require 'open-uri' + + ID_PARTITION_LIMIT = 1_000_000_000 + DIGEST = OpenSSL::Digest.const_get('SHA1').new + + def up + ActiveRecord::Base.connection.raw_connection.prepare('active_storage_blob_statement', <<-SQL) + INSERT INTO active_storage_blobs ( + key, filename, content_type, metadata, byte_size, checksum, created_at + ) VALUES ($1, $2, $3, '{}', $4, $5, $6) + RETURNING id; + SQL + + ActiveRecord::Base.connection.raw_connection.prepare('active_storage_attachment_statement', <<-SQL) + INSERT INTO active_storage_attachments ( + name, record_type, record_id, blob_id, created_at + ) VALUES ($1, $2, $3, $4, $5) + SQL + + Rails.application.eager_load! + models = ApplicationRecord.descendants.reject(&:abstract_class?) + + transaction do + models.each do |model| + attachments = model.column_names.map do |c| + $1 if c =~ /(.+)_file_name$/ + end.compact + + next if attachments.blank? + + model.find_each.each do |instance| + attachments.each do |attachment| + next if instance.__send__("#{attachment}_file_name").blank? + + res = ActiveRecord::Base.connection.raw_connection.exec_prepared( + 'active_storage_blob_statement', [ + key(instance, attachment), + instance.__send__("#{attachment}_file_name"), + instance.__send__("#{attachment}_content_type"), + instance.__send__("#{attachment}_file_size") || 0, + checksum(instance.__send__(attachment)), + instance.updated_at.iso8601 + ] + ) + + ActiveRecord::Base.connection.raw_connection.exec_prepared( + 'active_storage_attachment_statement', [ + attachment, + model.name, + instance.id, + res[0]['id'], + instance.updated_at.iso8601 + ] + ) + end + end + end + end + end + + def down + # raise ActiveRecord::IrreversibleMigration + end + + private + + ID_PARTITION_LIMIT = 1_000_000_000 + DIGEST = OpenSSL::Digest.const_get('SHA1').new + + def id_partition(id) + if id < ID_PARTITION_LIMIT + format('%09d', id).scan(/\d{3}/).join('/') + else + format('%012d', id).scan(/\d{3}/).join('/') + end + end + + def hash_data(instance, attachment) + "#{instance.class.to_s.underscore.pluralize}/#{attachment.pluralize}/#{instance.id}/original" + end + + def interpolate(pattern, instance, attachment) + path = pattern + path = path.gsub(':class', instance.class.to_s.underscore.pluralize) + path = path.gsub(':attachment', attachment.pluralize) + path = path.gsub(':id_partition', id_partition(instance.id)) + path = path.gsub(':hash', OpenSSL::HMAC.hexdigest(DIGEST, + ENV['PAPERCLIP_HASH_SECRET'], + hash_data(instance, attachment))) + path.gsub(':filename', instance.__send__("#{attachment}_file_name")) + end + + def key(instance, attachment) + # SecureRandom.uuid + # Alternatively: + pattern = if ENV['PAPERCLIP_STORAGE'] == 's3' + ':class/:attachment/:id_partition/:hash/original/:filename' + else + "#{Rails.root}/public/system/:class/:attachment/:id_partition/:hash/original/:filename" + end + interpolate(pattern, instance, attachment) + end + + def checksum(_attachment) + 'dummy' + # local files stored on disk: + # url = attachment.path + # Digest::MD5.base64digest(File.read(url)) + + # remote files stored on another person's computer: + # url = attachment.url + # Digest::MD5.base64digest(Net::HTTP.get(URI(url))) + end +end diff --git a/db/schema.rb b/db/schema.rb index d2216eae0..c69194b9d 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2019_06_13_094834) do +ActiveRecord::Schema.define(version: 2019_06_13_134100) do # These are extensions that must be enabled in order to support this database enable_extension "btree_gist" diff --git a/lib/active_storage/service/custom_s3_service.rb b/lib/active_storage/service/custom_s3_service.rb new file mode 100644 index 000000000..7847284b7 --- /dev/null +++ b/lib/active_storage/service/custom_s3_service.rb @@ -0,0 +1,172 @@ +# frozen_string_literal: true + +# Copyright (c) 2017-2019 David Heinemeier Hansson, Basecamp +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +gem 'aws-sdk-s3', '~> 1.14' + +require 'aws-sdk-s3' +require 'active_support/core_ext/numeric/bytes' + +module ActiveStorage + # Wraps the Amazon Simple Storage Service (S3) as an Active Storage service. + # See ActiveStorage::Service for the generic API documentation that applies to all services. + class Service::CustomS3Service < Service + attr_reader :client, :bucket + attr_reader :multipart_upload_threshold, :upload_options + attr_reader :subfolder + + def initialize(bucket:, upload: {}, **options) + @subfolder = options.delete(:subfolder) + + @client = Aws::S3::Resource.new(**options) + @bucket = @client.bucket(bucket) + + @multipart_upload_threshold = upload.fetch(:multipart_threshold, 100.megabytes) + @upload_options = upload + end + + def upload(key, io, checksum: nil, content_type: nil, **) + instrument :upload, key: key, checksum: checksum do + if io.size < multipart_upload_threshold + upload_with_single_part key, io, checksum: checksum, content_type: content_type + else + upload_with_multipart key, io, content_type: content_type + end + end + end + + def download(key, &block) + if block_given? + instrument :streaming_download, key: key do + stream(key, &block) + end + else + instrument :download, key: key do + object_for(key).get.body.string.force_encoding(Encoding::BINARY) + rescue Aws::S3::Errors::NoSuchKey + raise ActiveStorage::FileNotFoundError + end + end + end + + def download_chunk(key, range) + instrument :download_chunk, key: key, range: range do + object_for(key).get(range: "bytes=#{range.begin}-#{range.exclude_end? ? range.end - 1 : range.end}") + .body + .string + .force_encoding(Encoding::BINARY) + rescue Aws::S3::Errors::NoSuchKey + raise ActiveStorage::FileNotFoundError + end + end + + def delete(key) + instrument :delete, key: key do + object_for(key).delete + end + end + + def delete_prefixed(prefix) + instrument :delete_prefixed, prefix: prefix do + prefix = subfolder.present? ? File.join(subfolder, prefix) : prefix + bucket.objects(prefix: prefix).batch_delete! + end + end + + def exist?(key) + instrument :exist, key: key do |payload| + answer = object_for(key).exists? + payload[:exist] = answer + answer + end + end + + def url(key, expires_in:, filename:, disposition:, content_type:) + instrument :url, key: key do |payload| + generated_url = object_for(key).presigned_url :get, expires_in: expires_in.to_i, + response_content_disposition: content_disposition_with(type: disposition, filename: filename), + response_content_type: content_type + + payload[:url] = generated_url + + generated_url + end + end + + def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:) + instrument :url, key: key do |payload| + generated_url = object_for(key).presigned_url :put, expires_in: expires_in.to_i, + content_type: content_type, content_length: content_length, content_md5: checksum + + payload[:url] = generated_url + + generated_url + end + end + + def headers_for_direct_upload(_key, content_type:, checksum:, **) + { 'Content-Type' => content_type, 'Content-MD5' => checksum } + end + + private + + MAXIMUM_UPLOAD_PARTS_COUNT = 10000 + MINIMUM_UPLOAD_PART_SIZE = 5.megabytes + + def upload_with_single_part(key, io, checksum: nil, content_type: nil) + object_for(key).put(body: io, content_md5: checksum, content_type: content_type, **upload_options) + rescue Aws::S3::Errors::BadDigest + raise ActiveStorage::IntegrityError + end + + def upload_with_multipart(key, io, content_type: nil) + part_size = [io.size.fdiv(MAXIMUM_UPLOAD_PARTS_COUNT).ceil, MINIMUM_UPLOAD_PART_SIZE].max + + object_for(key).upload_stream(content_type: content_type, part_size: part_size, **upload_options) do |out| + IO.copy_stream(io, out) + end + end + + def object_for(key) + key = subfolder.present? ? File.join(subfolder, key) : key + bucket.object(key) + end + + # Reads the object for the given key in chunks, yielding each to the block. + def stream(key) + object = object_for(key) + + chunk_size = 5.megabytes + offset = 0 + + raise ActiveStorage::FileNotFoundError unless object.exists? + + while offset < object.content_length + yield object.get(range: "bytes=#{offset}-#{offset + chunk_size - 1}") + .body + .string + .force_encoding(Encoding::BINARY) + offset += chunk_size + end + end + end +end diff --git a/lib/tasks/active_storage.rake b/lib/tasks/active_storage.rake new file mode 100644 index 000000000..4c7292076 --- /dev/null +++ b/lib/tasks/active_storage.rake @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +namespace :active_storage do + ID_PARTITION_LIMIT = 1_000_000_000 + DIGEST = OpenSSL::Digest.const_get('SHA1').new + + def id_partition(id) + if id < ID_PARTITION_LIMIT + format('%09d', id).scan(/\d{3}/).join('/') + else + format('%012d', id).scan(/\d{3}/).join('/') + end + end + + def hash_data(attachment) + "#{attachment.record_type.underscore.pluralize}/#{attachment.name.pluralize}/#{attachment.record.id}/original" + end + + def interpolate(pattern, attachment) + path = pattern + path = path.gsub(':class', attachment.record_type.underscore.pluralize) + path = path.gsub(':attachment', attachment.name.pluralize) + path = path.gsub(':id_partition', id_partition(attachment.record.id)) + path = path.gsub(':hash', OpenSSL::HMAC.hexdigest(DIGEST, ENV['PAPERCLIP_HASH_SECRET'], hash_data(attachment))) + path.gsub(':filename', attachment.blob.filename.to_s) + end + + desc 'Copy all files from Paperclip to ActiveStorage' + task :migrate_files, [:before] => :environment do |_, _args| + if ENV['PAPERCLIP_STORAGE'] == 'filesystem' + local_path = "#{Rails.root}/public/system/:class/:attachment/:id_partition/:hash/original/:filename" + + ActiveStorage::Attachment.find_each do |attachment| + src = interpolate(local_path, attachment) + dst_dir = File.join( + 'storage', + attachment.blob.key.first(2), + attachment.blob.key.first(4).last(2) + ) + dst = File.join(dst_dir, attachment.blob.key) + + FileUtils.mkdir_p(dst_dir) + puts "Copying #{src} to #{dst}" + FileUtils.cp(src, dst) + end + elsif ENV['PAPERCLIP_STORAGE'] == 's3' + + s3_path = ':class/:attachment/:id_partition/:hash/original/:filename' + s3_path = "#{ENV['S3_SUBFOLDER']}/" + s3_path if ENV['S3_SUBFOLDER'] + + ActiveStorage::Attachment.find_each do |attachment| + src_path = interpolate(s3_path, attachment) + + next unless S3_BUCKET.object(src_path).exists? + + dst_path = ENV['S3_SUBFOLDER'] ? File.join(ENV['S3_SUBFOLDER'], attachment.blob.key) : attachment.blob.key + + puts "Copying #{src_path} to #{dst_path}" + + s3.copy_object(bucket: S3_BUCKET.name, + copy_source: S3_BUCKET.name + src_path, + key: dst_path) + rescue StandardError => e + puts 'Caught exception copying object ' + src_path + ':' + puts e.message + end + end + end +end From 5acb16d43e03d3e4e2f57e0cb91ea5adf8f9136c Mon Sep 17 00:00:00 2001 From: Oleksii Kriuchykhin Date: Fri, 28 Jun 2019 08:17:09 +0200 Subject: [PATCH 3/3] Update Asset helper methods [SCI-3539] --- .rubocop.yml | 2 +- app/controllers/assets_controller.rb | 6 +-- app/models/asset.rb | 52 +++++++++++++++---- app/models/protocol.rb | 4 +- .../repository_actions/duplicate_cell.rb | 4 +- app/views/my_modules/archive/_result.html.erb | 2 +- .../_my_module_result_asset_element.html.erb | 2 +- .../elements/_step_asset_element.html.erb | 2 +- app/views/search/results/_assets.html.erb | 2 +- app/views/search/results/_results.html.erb | 2 +- app/views/shared/_asset_link.html.erb | 52 ++++++------------- app/views/shared/_file_preview_icon.html.erb | 2 +- app/views/steps/attachments/_item.html.erb | 10 ---- .../steps/attachments/_placeholder.html.erb | 19 ++++--- config/initializers/paperclip.rb | 21 -------- ...0190613134100_convert_to_active_storage.rb | 14 ++--- lib/tasks/paperclip.rake | 2 +- 17 files changed, 90 insertions(+), 108 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index b94e63629..af6fc7333 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -5,7 +5,7 @@ AllCops: - "vendor/**/*" - "db/schema.rb" UseCache: false - TargetRubyVersion: 2.5 + TargetRubyVersion: 2.6 ##################### Style #################################### diff --git a/app/controllers/assets_controller.rb b/app/controllers/assets_controller.rb index 00c5e918a..60caa59a6 100644 --- a/app/controllers/assets_controller.rb +++ b/app/controllers/assets_controller.rb @@ -63,7 +63,7 @@ class AssetsController < ApplicationController def file_preview response_json = { 'id' => @asset.id, - 'type' => (@asset.is_image? ? 'image' : 'file'), + 'type' => (@asset.image? ? 'image' : 'file'), 'filename' => truncate(escape_input(@asset.file_file_name), length: Constants::FILENAME_TRUNCATION_LENGTH), @@ -78,7 +78,7 @@ class AssetsController < ApplicationController can_manage_repository_rows?(@repository.team) end - if @asset.is_image? + if @asset.image? if ['image/jpeg', 'image/pjpeg'].include? @asset.file.content_type response_json['quality'] = @asset.file_image_quality || 90 end @@ -322,7 +322,7 @@ class AssetsController < ApplicationController def asset_data_type(asset) return 'wopi' if wopi_file?(asset) - return 'image' if asset.is_image? + return 'image' if asset.image? 'file' end diff --git a/app/models/asset.rb b/app/models/asset.rb index ff78b324e..77470189a 100644 --- a/app/models/asset.rb +++ b/app/models/asset.rb @@ -203,16 +203,30 @@ class Asset < ApplicationRecord end end + def previewable? + previewable_document? || previewable_image? + end + def medium_preview - file.variant(resize: Constants::MEDIUM_PIC_FORMAT) + return file.variant(resize: Constants::MEDIUM_PIC_FORMAT) if previewable_image? + + 'medium/processing.gif' + # file.preview(resize: Constants::MEDIUM_PIC_FORMAT) end def large_preview - file.variant(resize: Constants::LARGE_PIC_FORMAT) + return file.variant(resize: Constants::LARGE_PIC_FORMAT) if previewable_image? + + 'large/processing.gif' + # file.preview(resize: Constants::LARGE_PIC_FORMAT) + end + + def file_name + file.blob&.filename&.to_s end def file_size - file.blob.byte_size + file.blob&.byte_size end def extract_image_quality @@ -227,13 +241,8 @@ class Asset < ApplicationRecord Rails.logger.info "There was an error extracting image quality - #{e}" end - def previewable? - file.previewable_image? || file.previewable_document? - end - - def is_image? - %r{^image/#{Regexp.union(Constants::WHITELISTED_IMAGE_TYPES)}} === - file.content_type + def image? + file.blob.content_type == %r{^image/#{Regexp.union(Constants::WHITELISTED_IMAGE_TYPES)}} end def text? @@ -520,6 +529,29 @@ class Asset < ApplicationRecord private + def previewable_document? + previewable = Constants::PREVIEWABLE_FILE_TYPES.include?(file.blob&.content_type) + + filename = file.blob&.filename + content_type = file.blob&.content_type + + extensions = %w(.xlsx .docx .pptx .xls .doc .ppt) + # Mimetype sometimes recognizes Office files as zip files + # In this case we also check the extension of the given file + # Otherwise the conversion should fail if the file is being something else + previewable ||= (content_type == 'application/zip' && extensions.include?(File.extname(filename))) + + # Mimetype also sometimes recognizes '.xls' and '.ppt' files as + # application/x-ole-storage (https://github.com/minad/mimemagic/issues/50) + previewable ||= (content_type == 'application/x-ole-storage' && %w(.xls .ppt).include?(File.extname(filename))) + + previewable + end + + def previewable_image? + file.blob&.content_type =~ %r{^image/#{Regexp.union(Constants::WHITELISTED_IMAGE_TYPES)}} + end + def filter_paperclip_errors if errors.size > 1 temp_errors = errors[:file] diff --git a/app/models/protocol.rb b/app/models/protocol.rb index 2e857e14a..add6f26d4 100644 --- a/app/models/protocol.rb +++ b/app/models/protocol.rb @@ -236,7 +236,7 @@ class Protocol < ApplicationRecord dest.file = src.file dest.save! - if dest.is_image? + if dest.image? dest.file.reprocess!(:large) dest.file.reprocess!(:medium) end @@ -325,7 +325,7 @@ class Protocol < ApplicationRecord asset2.created_by = current_user asset2.team = dest.team asset2.last_modified_by = current_user - asset2.file_processing = true if asset.is_image? + asset2.file_processing = true if asset.image? asset2.save! step2.assets << asset2 diff --git a/app/services/repository_actions/duplicate_cell.rb b/app/services/repository_actions/duplicate_cell.rb index c6a8478dc..554638226 100644 --- a/app/services/repository_actions/duplicate_cell.rb +++ b/app/services/repository_actions/duplicate_cell.rb @@ -77,13 +77,13 @@ module RepositoryActions new_asset.created_by = old_asset.created_by new_asset.team = @team new_asset.last_modified_by = @user - new_asset.file_processing = true if old_asset.is_image? + new_asset.file_processing = true if old_asset.image? new_asset.file = old_asset.file new_asset.save return unless new_asset.valid? - if new_asset.is_image? + if new_asset.image? new_asset.file.reprocess!(:large) new_asset.file.reprocess!(:medium) end diff --git a/app/views/my_modules/archive/_result.html.erb b/app/views/my_modules/archive/_result.html.erb index 51b70b1da..64cc1d1aa 100644 --- a/app/views/my_modules/archive/_result.html.erb +++ b/app/views/my_modules/archive/_result.html.erb @@ -36,7 +36,7 @@

<% if result.is_asset %> - <% if result.asset.is_image? %> + <% if result.asset.image? %> <% else %> diff --git a/app/views/reports/elements/_my_module_result_asset_element.html.erb b/app/views/reports/elements/_my_module_result_asset_element.html.erb index ca24d4f9a..3cbb13b6e 100644 --- a/app/views/reports/elements/_my_module_result_asset_element.html.erb +++ b/app/views/reports/elements/_my_module_result_asset_element.html.erb @@ -1,6 +1,6 @@ <% result ||= @result %> <% asset = result.asset %> -<% is_image = result.asset.is_image? %> +<% is_image = result.asset.image? %> <% comments = result.result_comments %> <% timestamp = asset.created_at %> <% icon_class = 'fas ' + (is_image ? 'fa-image' : 'fa-file') %> diff --git a/app/views/reports/elements/_step_asset_element.html.erb b/app/views/reports/elements/_step_asset_element.html.erb index f9bbcde53..b84d9a61b 100644 --- a/app/views/reports/elements/_step_asset_element.html.erb +++ b/app/views/reports/elements/_step_asset_element.html.erb @@ -1,5 +1,5 @@ <% asset ||= @asset %> -<% is_image = asset.is_image? %> +<% is_image = asset.image? %> <% timestamp = asset.created_at %> <% icon_class = 'fas ' + (is_image ? 'fa-image' : 'fa-file') %>
" data-icon-class="<%= icon_class %>"> diff --git a/app/views/search/results/_assets.html.erb b/app/views/search/results/_assets.html.erb index 73a884c4e..68b42a216 100644 --- a/app/views/search/results/_assets.html.erb +++ b/app/views/search/results/_assets.html.erb @@ -1,6 +1,6 @@ <% @asset_results.each do |asset| %>
- <% if asset.is_image? %> + <% if asset.image? %> <% else %> <% if wopi_file?(asset) %> diff --git a/app/views/search/results/_results.html.erb b/app/views/search/results/_results.html.erb index 4f2849096..a062ba182 100644 --- a/app/views/search/results/_results.html.erb +++ b/app/views/search/results/_results.html.erb @@ -5,7 +5,7 @@ <% elsif result.is_table %> <% else %> - <% if result.asset.is_image? %> + <% if result.asset.image? %> <% else %> diff --git a/app/views/shared/_asset_link.html.erb b/app/views/shared/_asset_link.html.erb index 7dbd2482f..f214ed94f 100644 --- a/app/views/shared/_asset_link.html.erb +++ b/app/views/shared/_asset_link.html.erb @@ -1,37 +1,19 @@ -<% if asset.file_present %> - <% if asset.file.processing? && display_image_tag && asset.is_image? %> - - <%= image_tag 'medium/processing.gif' %> - <%= link_to download_asset_path(asset), - class: 'file-preview-link', - id: "modal_link#{asset.id}", - data: { no_turbolink: true, id: true, status: 'asset-present', 'preview-url': asset_file_preview_path(asset) } do %> - <%= truncate(asset.file_file_name, - length: Constants::FILENAME_TRUNCATION_LENGTH) %> - <% end %> - - <% else %> - <%= link_to download_asset_path(asset), - class: 'file-preview-link', - id: "modal_link#{asset.id}", - data: { no_turbolink: true, id: true, status: 'asset-present', 'preview-url': asset_file_preview_path(asset) } do %> - <% if asset.is_image? && display_image_tag %> - <%= image_tag asset.url(:medium) %> - <% end %> - <% if display_image_tag %> -

- <%= truncate(asset.file_file_name, - length: Constants::FILENAME_TRUNCATION_LENGTH) %> -

- <% else %> - - <%= truncate(asset.file_file_name, - length: Constants::FILENAME_TRUNCATION_LENGTH) %> - - <% end %> - <% end %> +<%= link_to download_asset_path(asset), + class: 'file-preview-link', + id: "modal_link#{asset.id}", + data: { no_turbolink: true, id: true, status: 'asset-present', 'preview-url': asset_file_preview_path(asset) } do %> + <% if asset.image? && display_image_tag %> + <%= image_tag asset.medium_preview %> + <% end %> + <% if display_image_tag %> +

+ <%= truncate(asset.file_file_name, + length: Constants::FILENAME_TRUNCATION_LENGTH) %> +

+ <% else %> + + <%= truncate(asset.file_file_name, + length: Constants::FILENAME_TRUNCATION_LENGTH) %> + <% end %> -<% else %> - <%= image_tag 'medium/processing.gif' %> <% end %> diff --git a/app/views/shared/_file_preview_icon.html.erb b/app/views/shared/_file_preview_icon.html.erb index a37b5e60e..45c362022 100644 --- a/app/views/shared/_file_preview_icon.html.erb +++ b/app/views/shared/_file_preview_icon.html.erb @@ -1,6 +1,6 @@
<% if asset.file.previewable_document? %> - <%= image_tag(asset.url(:large)) %> + <%= image_tag(asset.large_preview) %> <% else %> <% end %> diff --git a/app/views/steps/attachments/_item.html.erb b/app/views/steps/attachments/_item.html.erb index 08c1be4b3..679f4dafa 100644 --- a/app/views/steps/attachments/_item.html.erb +++ b/app/views/steps/attachments/_item.html.erb @@ -1,19 +1,9 @@ -<% if asset.file.processing? && (asset.is_image? || asset.file.previewable_document?) %> - <% asset_status = 'asset-loading' %> - <% present_url = step_file_present_asset_path(asset.id) %> -<% else %> - <% asset_status = 'asset-present' %> - <% present_url = '' %> -<% end %> -
<%= link_to download_asset_path(asset), class: 'file-preview-link', id: "modal_link#{asset.id}", data: { no_turbolink: true, id: true, - status: asset_status, - 'present-url': present_url, 'preview-url': asset_file_preview_path(asset), 'order-atoz': az_ordered_assets_index(step, asset.id), 'order-ztoa': assets_count - az_ordered_assets_index(step, asset.id), diff --git a/app/views/steps/attachments/_placeholder.html.erb b/app/views/steps/attachments/_placeholder.html.erb index c5c079c52..a07e47d10 100644 --- a/app/views/steps/attachments/_placeholder.html.erb +++ b/app/views/steps/attachments/_placeholder.html.erb @@ -1,23 +1,22 @@ -<% image_preview = (asset.is_image? || asset.file.previewable_document?) ? asset.url(:medium) : nil %> -
-
- <% if image_preview %> - <%= image_tag image_preview %> +
+ <% if asset.previewable? %> + <%= image_tag asset.medium_preview %> <% else %> - + <% end %>
-
<%= truncate(asset.file_file_name, - length: Constants::FILENAME_TRUNCATION_LENGTH) %>
+
+ <%= truncate(asset.file_name, length: Constants::FILENAME_TRUNCATION_LENGTH) %> +
<%= t('protocols.steps.attachments.modified_label') %> <%= l(asset.updated_at, format: :full_date) if asset.updated_at %>
- <%= number_to_human_size(asset.file.size) %> + <%= number_to_human_size(asset.file_size) %>
<% if edit_page %>
- <% unless ff.object.file.exists? && ff.object.locked? %> + <% unless ff.object.file.attached? && ff.object.locked? %> <%= ff.remove_nested_fields_link do %> <% end %> diff --git a/config/initializers/paperclip.rb b/config/initializers/paperclip.rb index 7ac23bbe9..f032d32ca 100644 --- a/config/initializers/paperclip.rb +++ b/config/initializers/paperclip.rb @@ -61,27 +61,6 @@ Paperclip::Attachment.class_eval do def fetch Paperclip.io_adapters.for self end - - def previewable_document? - previewable = Constants::PREVIEWABLE_FILE_TYPES.include?(content_type) - - extensions = %w(.xlsx .docx .pptx .xls .doc .ppt) - # Mimetype sometimes recognizes Office files as zip files - # In this case we also check the extension of the given file - # Otherwise the conversion should fail if the file is being something else - previewable ||= (content_type == 'application/zip' && extensions.include?(File.extname(original_filename))) - - # Mimetype also sometimes recognizes '.xls' and '.ppt' files as - # application/x-ole-storage (https://github.com/minad/mimemagic/issues/50) - previewable ||= - (content_type == 'application/x-ole-storage' && %w(.xls .ppt).include?(File.extname(original_filename))) - - previewable - end - - def previewable_image? - content_type =~ %r{^image/#{Regexp.union(Constants::WHITELISTED_IMAGE_TYPES)}} - end end module Paperclip diff --git a/db/migrate/20190613134100_convert_to_active_storage.rb b/db/migrate/20190613134100_convert_to_active_storage.rb index ea9de35b3..76593aa1b 100644 --- a/db/migrate/20190613134100_convert_to_active_storage.rb +++ b/db/migrate/20190613134100_convert_to_active_storage.rb @@ -41,7 +41,7 @@ class ConvertToActiveStorage < ActiveRecord::Migration[5.2] instance.__send__("#{attachment}_file_name"), instance.__send__("#{attachment}_content_type"), instance.__send__("#{attachment}_file_size") || 0, - checksum(instance.__send__(attachment)), + checksum(attachment), instance.updated_at.iso8601 ] ) @@ -96,12 +96,12 @@ class ConvertToActiveStorage < ActiveRecord::Migration[5.2] def key(instance, attachment) # SecureRandom.uuid # Alternatively: - pattern = if ENV['PAPERCLIP_STORAGE'] == 's3' - ':class/:attachment/:id_partition/:hash/original/:filename' - else - "#{Rails.root}/public/system/:class/:attachment/:id_partition/:hash/original/:filename" - end - interpolate(pattern, instance, attachment) + if ENV['PAPERCLIP_STORAGE'] == 's3' + interpolate(':class/:attachment/:id_partition/:hash/original/:filename', instance, attachment) + else + key = SecureRandom.uuid + File.join('storage', key.first(2), key.first(4).last(2)) + end end def checksum(_attachment) diff --git a/lib/tasks/paperclip.rake b/lib/tasks/paperclip.rake index eea765620..21c79a6e3 100644 --- a/lib/tasks/paperclip.rake +++ b/lib/tasks/paperclip.rake @@ -36,7 +36,7 @@ namespace :paperclip do assets = assets.where('updated_at < ?', eval(args[:before])) end assets.find_each(batch_size: 100) do |asset| - next unless asset.is_image? + next unless asset.image? begin asset.file.reprocess! :medium, :large rescue