scinote-eln/scinote-web
1
1
Fork 0
mirror of https://github.com/scinote-eln/scinote-web.git synced 2024-09-20 14:45:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge latest features/export-all [SCI-2733]

Browse source
This commit is contained in:
Oleksii Kriuchykhin 2018-10-16 17:20:19 +02:00
parent 1cafbd8c50 00139beb89
commit 02f536cd08
35 changed files with 949 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

58
app/assets/javascripts/projects/index.js
View file

@ -27,6 +27,11 @@
var projectActionsModalBody = null;
var projectActionsModalFooter = null;
var exportProjectsModal = null;
var exportProjectsModalHeader = null;
var exportProjectsModalBody = null;
var exportProjectsBtn = null;
var projectsViewMode = 'cards';
var projectsViewFilter = $('.projects-view-filter.active').data('filter');
var projectsViewFilterChanged = false;
@ -210,6 +215,42 @@
});
}
/**
* Initialize the JS for export projects modal to work.
*/
function initExportProjectsModal() {
exportProjectsBtn.click(function() {
// Load HTML to refresh users list
$.ajax({
url: exportProjectsBtn.data('export-projects-url'),
type: 'POST',
dataType: 'json',
data: {
project_ids: selectedProjects
},
success: function(data) {
// Update modal title
exportProjectsModalHeader.html(data.title);
// Set modal body
exportProjectsModalBody.html(data.html);
// Show the modal
exportProjectsModal.modal('show');
},
error: function() {
// TODO
}
});
});
// Remove modal content when modal window is closed.
exportProjectsModal.on('hidden.bs.modal', function() {
exportProjectsModalHeader.html('');
exportProjectsModalBody.html('');
});
}
// Initialize reloading manage user modal content after posting new
// user.
@ -295,10 +336,17 @@
projectActionsModalBody = projectActionsModal.find('.modal-body');
projectActionsModalFooter = projectActionsModal.find('.modal-footer');
exportProjectsModal = $('#export-projects-modal');
exportProjectsModalHeader = exportProjectsModal.find('.modal-title');
exportProjectsModalBody = exportProjectsModal.find('.modal-body');
exportProjectsBtn = $('#export-projects-button');
exportProjectsBtn.addClass('disabled');
updateSelectedCards();
initNewProjectModal();
initEditProjectModal();
initManageUsersModal();
initExportProjectsModal();
Comments.initCommentOptions('ul.content-comments', true);
Comments.initEditComments('.panel-project .tab-content');
Comments.initDeleteComments('.panel-project .tab-content');
@ -315,10 +363,15 @@
if (this.checked && index === -1) {
$(this).closest('.panel-project').addClass('selected');
selectedProjects.push(projectId);
exportProjectsBtn.removeClass('disabled');
// Otherwise, if checkbox is not checked and ID is in list of selected IDs
} else if (!this.checked && index !== -1) {
$(this).closest('.panel-project').removeClass('selected');
selectedProjects.splice(index, 1);
if (selectedProjects.length === 0) {
exportProjectsBtn.addClass('disabled');
}
}
});
@ -484,9 +537,14 @@
// If checkbox is checked and row ID is not in list of selected project IDs
if (this.checked && index === -1) {
selectedProjects.push(rowId);
exportProjectsBtn.removeClass('disabled');
// Otherwise, if checkbox is not checked and ID is in list of selected IDs
} else if (!this.checked && index !== -1) {
selectedProjects.splice(index, 1);
if (selectedProjects.length === 0) {
exportProjectsBtn.addClass('disabled');
}
}
updateDataTableSelectAllCtrl();

18
app/assets/stylesheets/reports.scss
View file

@ -68,7 +68,7 @@ label {
}
.ht_clone_top,.ht_clone_left,.ht_clone_corner {
display: none !important;
display: none !important;
}
}
@ -323,8 +323,14 @@ label {
// Result table element style
.report-result-table-element {
.report-element-header {
.table-name {
margin-left: 15px;
}
}
.report-element-body {
padding-top: 30px;
padding-top: 15px;
}
}
@ -493,6 +499,14 @@ label {
.repository-name {
margin-left: 5px;
}
.table-name {
margin-left: 15px;
}
}
.report-element-body {
padding-top: 15px;
}
&:hover > .report-element-header {

68
app/controllers/teams_controller.rb
View file

@ -1,9 +1,11 @@
class TeamsController < ApplicationController
before_action :load_vars, only: [:parse_sheet, :import_samples, :export_samples]
before_action :load_vars, only: %i(parse_sheet import_samples
export_samples export_projects)
before_action :check_create_samples_permissions, only: %i(parse_sheet
import_samples)
before_action :check_view_samples_permission, only: [:export_samples]
before_action :check_export_projects_permissions, only: :export_projects
def parse_sheet
session[:return_to] ||= request.referer
@ -223,6 +225,39 @@ class TeamsController < ApplicationController
redirect_back(fallback_location: root_path)
end
def export_projects
if export_projects_params[:project_ids]
# Check if user has enough requests for the day
limit = (ENV['EXPORT_ALL_LIMIT_24_HOURS'] || 3).to_i
if limit.zero? \
|| current_user.export_vars['num_of_export_all_last_24_hours'] >= limit
render json: {
html: render_to_string(
partial: 'projects/export/error.html.erb',
locals: { limit: limit }
),
title: t('projects.export_projects.modal_title_error')
}
else
current_user.export_vars['num_of_export_all_last_24_hours'] += 1
current_user.save
ids = generate_export_projects_zip
curr_num = current_user.export_vars['num_of_export_all_last_24_hours']
render json: {
html: render_to_string(
partial: 'projects/export/success.html.erb',
locals: { num_projects: ids.length,
limit: limit,
num_of_requests_left: limit - curr_num }
),
title: t('projects.export_projects.modal_title_success')
}
end
end
end
def routing_error(error = 'Routing error', status = :not_found, exception=nil)
redirect_to root_path
end
@ -259,6 +294,10 @@ class TeamsController < ApplicationController
params.permit(sample_ids: [], header_ids: []).to_h
end
def export_projects_params
params.permit(:id, project_ids: []).to_h
end
def check_create_samples_permissions
render_403 unless can_create_samples?(@team)
end
@ -269,6 +308,17 @@ class TeamsController < ApplicationController
end
end
def check_export_projects_permissions
render_403 unless can_read_team?(@team)
if export_projects_params[:project_ids]
projects = Project.where(id: export_projects_params[:project_ids])
projects.each do |project|
render_403 unless can_read_project?(current_user, project)
end
end
end
def generate_samples_zip
zip = ZipExport.create(user: current_user)
zip.generate_exportable_zip(
@ -280,4 +330,20 @@ class TeamsController < ApplicationController
:samples
)
end
def generate_export_projects_zip
ids = Project.where(id: export_projects_params[:project_ids],
team_id: @team)
.index_by(&:id)
options = { team: @team }
zip = TeamZipExport.create(user: current_user)
zip.generate_exportable_zip(
current_user,
ids,
:teams,
options
)
ids
end
end

10
app/controllers/zip_exports_controller.rb
View file

@ -1,5 +1,6 @@
class ZipExportsController < ApplicationController
before_action :load_var, only: :download
before_action :load_var_export_all, only: :download_export_all_zip
before_action :check_edit_permissions, only: :download
def download
@ -12,6 +13,10 @@ class ZipExportsController < ApplicationController
end
end
def download_export_all_zip
download
end
def file_expired; end
private
@ -21,6 +26,11 @@ class ZipExportsController < ApplicationController
redirect_to(file_expired_url, status: 301) and return unless @zip_export
end
def load_var_export_all
@zip_export = TeamZipExport.find_by_id(params[:id])
redirect_to(file_expired_url, status: 301) and return unless @zip_export
end
def check_edit_permissions
render_403 unless @zip_export.user == current_user
end

81
app/helpers/reports_helper.rb
View file

@ -1,12 +1,25 @@
module ReportsHelper
include StringUtility
def render_new_element(hide)
render partial: 'reports/elements/new_element.html.erb',
locals: { hide: hide }
end
def render_report_element(element, provided_locals = nil)
children_html = ''.html_safe
# Determine partial
file_name = element.type_of
if element.type_of.in? ReportExtends::MY_MODULE_CHILDREN_ELEMENTS
file_name = "my_module_#{element.type_of.singularize}"
end
view = "reports/elements/#{file_name}_element.html.erb"
# Set locals
locals = provided_locals.nil? ? {} : provided_locals.clone
children_html = ''.html_safe
# First, recursively render element's children
if element.comments? || element.project_header?
# Render no children
@ -31,25 +44,54 @@ module ReportsHelper
end
children_html.safe_concat render_new_element(false)
end
file_name = element.type_of
if element.type_of.in? ReportExtends::MY_MODULE_CHILDREN_ELEMENTS
file_name = "my_module_#{element.type_of.singularize}"
end
view = "reports/elements/#{file_name}_element.html.erb"
locals = provided_locals.nil? ? {} : provided_locals.clone
locals[:children] = children_html
# ReportExtends is located in config/initializers/extends/report_extends.rb
if provided_locals[:export_all]
# Set path and filename locals for files and tables in export all ZIP
if element['type_of'] == 'my_module_repository'
obj_id = element[:repository_id]
elsif element['type_of'].in? %w(step_asset step_table result_asset
result_table)
parent_el = ReportElement.find(element['parent_id'])
parent_type = parent_el[:type_of]
parent = parent_type.singularize.classify.constantize
.find(parent_el["#{parent_type}_id"])
if parent.class == Step
obj_id = if element['type_of'] == 'step_asset'
element[:asset_id]
elsif element['type_of'] == 'step_table'
element[:table_id]
end
elsif parent.class == MyModule
result = Result.find(element[:result_id])
obj_id = if element['type_of'] == 'result_asset'
result.asset.id
elsif element['type_of'] == 'result_table'
result.table.id
end
end
end
if obj_id
locals[:path] =
provided_locals[:obj_filenames][element['type_of'].to_sym][obj_id]
.sub(%r{/usr/src/app/tmp/temp-zip-\d+/}, '')
locals[:filename] = locals[:path].split('/').last
end
end
# ReportExtends is located in config/initializers/extends/report_extends.rb
ReportElement.type_ofs.keys.each do |type|
next unless element.public_send("#{type}?")
element.element_references.each do |el_ref|
locals[el_ref.class.name.underscore.to_sym] = el_ref
end
locals[:order] = element
.sort_order if type.in? ReportExtends::SORTED_ELEMENTS
if type.in? ReportExtends::SORTED_ELEMENTS
locals[:order] = element.sort_order
end
end
(render partial: view, locals: locals).html_safe
@ -114,4 +156,19 @@ module ReportsHelper
end
html_doc.to_s
end
private
def obj_name_to_filename(obj, filename_suffix = '')
obj_name = if obj.class == Asset
obj_name, extension = obj.file_file_name.split('.')
extension&.prepend('.')
obj_name
elsif obj.class.in? [Table, Result, Repository]
extension = '.csv'
obj.name.present? ? obj.name : obj.class.name
end
obj_name = to_filesystem_name(obj_name)
obj_name + "#{filename_suffix}#{extension}"
end
end

23
app/helpers/string_utility.rb
View file

@ -1,3 +1,5 @@
# frozen_string_literal: true
module StringUtility
def ellipsize(
string,
@ -10,4 +12,25 @@ module StringUtility
mid_length = length - edge_length * 2
string.gsub(/(#{edge}).{#{mid_length},}(#{edge})/, '\1...\2')
end
# Convert string to filesystem compatible file/folder name
def to_filesystem_name(name)
# Handle reserved directories
if name == '..'
return '__'
elsif name == '.'
return '_'
end
# Truncate and replace reserved characters
name = name[0, Constants::EXPORTED_FILENAME_TRUNCATION_LENGTH]
.gsub(%r{[*":<>?/\\|~]}, '_')
# Remove control characters
name = name.chars.map(&:ord).select { |s| (s > 31 && s < 127) || s > 127 }
.pack('U*')
# Remove leading hyphens, trailing dots/spaces
name.gsub(/^-|\.+$| +$/, '_')
end
end

8
app/mailers/app_mailer.rb
View file

@ -8,9 +8,15 @@ class AppMailer < Devise::Mailer
def notification(user, notification, opts = {})
@user = user
@notification = notification
subject =
if notification.deliver?
I18n.t('notifications.deliver.email_subject')
else
I18n.t('notifications.email_title')
end
headers = {
to: @user.email,
subject: I18n.t('notifications.email_title')
subject: subject
}.merge(opts)
mail(headers)
end

24
app/models/concerns/variables_model.rb Normal file
View file

@ -0,0 +1,24 @@
# frozen_string_literal: true
module VariablesModel
extend ActiveSupport::Concern
@@default_variables = HashWithIndifferentAccess.new
included do
serialize :variables, JsonbHashSerializer
after_initialize :init_default_variables, if: :new_record?
end
class_methods do
def default_variables(dfs)
@@default_variables.merge!(dfs)
end
end
protected
def init_default_variables
self.variables = @@default_variables
end
end

68
app/models/project.rb
View file

@ -221,4 +221,72 @@ class Project < ApplicationRecord
end
res
end
def generate_report_pdf(user, team, pdf_name, obj_filenames = nil)
ActionController::Renderer::RACK_KEY_TRANSLATION['warden'] ||= 'warden'
proxy = Warden::Proxy.new({}, Warden::Manager.new({}))
renderer = ApplicationController.renderer.new(warden: proxy)
report = Report.generate_whole_project_report(self, user, team)
page_html_string =
renderer.render 'reports/new.html.erb',
locals: { export_all: true,
obj_filenames: obj_filenames },
assigns: { project: self, report: report }
parsed_page_html = Nokogiri::HTML(page_html_string)
parsed_pdf_html = parsed_page_html.at_css('#report-content')
report.destroy
tables = parsed_pdf_html.css('.hot-table-contents')
.zip(parsed_pdf_html.css('.hot-table-container'))
tables.each do |table_input, table_container|
table_vals = JSON.parse(table_input['value'])
table_data = table_vals['data']
table_headers = table_vals['headers']
table_headers ||= ('A'..'Z').first(table_data[0].count)
table_el = table_container
.add_child('<table class="handsontable"></table>').first
# Add header row
header_cell = '<th>'\
'<div class="relative">'\
'<span>%s</span>'\
'</div>'\
'</th>'
header_el = table_el.add_child('<thead></thead>').first
row_el = header_el.add_child('<tr></tr>').first
row_el.add_child(format(header_cell, '')).first
table_headers.each do |col|
row_el.add_child(format(header_cell, col)).first
end
# Add body rows
body_cell = '<td>%s</td>'
body_el = table_el.add_child('<tbody></tbody>').first
table_data.each.with_index(1) do |row, idx|
row_el = body_el.add_child('<tr></tr>').first
row_el.add_child(format(header_cell, idx)).first
row.each do |col|
row_el.add_child(format(body_cell, col)).first
end
end
end
parsed_pdf = ApplicationController.render(
pdf: pdf_name,
header: { right: '[page] of [topage]' },
locals: { content: parsed_pdf_html.to_s },
template: 'reports/report.pdf.erb',
disable_javascript: true,
disable_internal_links: false,
current_user: user,
current_team: team
)
# Dirty workaround to convert absolute links back to relative ones, since
# WickedPdf does the opposite, based on the path where the file parsing is
# done
parsed_pdf.gsub('/URI (file:////tmp/', '/URI (')
end
end

99
app/models/report.rb
View file

@ -20,7 +20,7 @@ class Report < ApplicationRecord
# Report either has many report elements (if grouped by timestamp),
# or many module elements (if grouped by module)
has_many :report_elements, inverse_of: :report, dependent: :destroy
has_many :report_elements, inverse_of: :report, dependent: :delete_all
after_commit do
Views::Datatables::DatatablesReport.refresh_materialized_view
@ -93,6 +93,103 @@ class Report < ApplicationRecord
end
end
def self.generate_whole_project_report(project, current_user, current_team)
report_contents = gen_element_content(project, nil, 'project_header', true)
project.experiments.each do |exp|
modules = []
exp.my_modules.each do |my_module|
module_children = []
my_module.protocol.steps.each do |step|
step_children =
gen_element_content(step, step.assets, 'step_asset')
step_children +=
gen_element_content(step, step.tables, 'step_table')
step_children +=
gen_element_content(step, step.checklists, 'step_checklist')
step_children +=
gen_element_content(step, nil, 'step_comments', true, 'asc')
module_children +=
gen_element_content(step, nil, 'step', true, nil, step_children)
end
my_module.results.each do |result|
result_children =
gen_element_content(result, nil, 'result_comments', true, 'asc')
result_type = if result.asset
'result_asset'
elsif result.table
'result_table'
elsif result.result_text
'result_text'
end
module_children +=
gen_element_content(result, nil, result_type, true, nil,
result_children)
end
module_children +=
gen_element_content(my_module, nil, 'my_module_activity', true, 'asc')
module_children +=
gen_element_content(my_module,
my_module.repository_rows.select(:repository_id)
.distinct.map(&:repository),
'my_module_repository', true, 'asc')
modules +=
gen_element_content(my_module, nil, 'my_module', true, nil,
module_children)
end
report_contents +=
gen_element_content(exp, nil, 'experiment', true, nil, modules)
end
report = Report.new
report.name = loop do
dummy_name = SecureRandom.hex(10)
break dummy_name unless Report.where(name: dummy_name).exists?
end
report.project = project
report.user = current_user
report.team = current_team
report.last_modified_by = current_user
report.save_with_contents(report_contents)
report
end
def self.gen_element_content(parent_obj, association_objs, type_of,
use_parent_id = false, sort_order = nil,
children = nil)
parent_type = parent_obj.class.name.underscore
type = type_of.split('_').last.singularize
extra_id_needed = use_parent_id && !association_objs.nil?
elements = []
association_objs ||= [nil]
association_objs.each do |obj|
elements << {
'type_of' => type_of,
'id' => {}.tap do |ids_hash|
if use_parent_id
ids_hash["#{parent_type}_id"] = parent_obj.id
else
ids_hash["#{type}_id"] = obj.id
end
ids_hash["#{type}_id"] = obj.id if extra_id_needed
end,
'sort_order' => sort_order.present? ? sort_order : nil,
'children' => children.present? ? children : []
}
end
elements
end
private
# Recursively save a single JSON element

11
app/models/table.rb
View file

@ -128,4 +128,15 @@ class Table < ApplicationRecord
Table.connection.execute(sql)
end
end
def to_csv
require 'csv'
data = JSON.parse(contents)['data']
CSV.generate do |csv|
data.each do |row|
csv << row
end
end
end
end

273
app/models/team_zip_export.rb Normal file
View file

@ -0,0 +1,273 @@
# frozen_string_literal: true
require 'zip'
require 'fileutils'
require 'csv'
class TeamZipExport < ZipExport
include StringUtility
# Override path only for S3
if ENV['PAPERCLIP_STORAGE'] == 's3'
has_attached_file :zip_file,
path: '/zip_exports/:attachment/:id_partition/' \
':hash/:style/:filename'
validates_attachment :zip_file,
content_type: { content_type: 'application/zip' }
end
def generate_exportable_zip(user, data, type, options = {})
@user = user
FileUtils.mkdir_p(File.join(Rails.root, 'tmp/zip-ready'))
dir_to_zip = FileUtils.mkdir_p(
File.join(Rails.root, "tmp/temp-zip-#{Time.now.to_i}")
).first
output_file = File.new(
File.join(Rails.root,
"tmp/zip-ready/projects-export-#{Time.now.to_i}.zip"),
'w+'
)
fill_content(dir_to_zip, data, type, options)
zip!(dir_to_zip, output_file.path)
self.zip_file = File.open(output_file)
generate_notification(user) if save
end
handle_asynchronously :generate_exportable_zip
private
# Export all functionality
def generate_teams_zip(tmp_dir, data, options = {})
# Create team folder
@team = options[:team]
team_path = "#{tmp_dir}/#{to_filesystem_name(@team.name)}"
FileUtils.mkdir_p(team_path)
# Create Projects folders
FileUtils.mkdir_p("#{team_path}/Projects")
FileUtils.mkdir_p("#{team_path}/Archived projects")
# Iterate through every project
data.each_with_index do |(_, p), ind|
obj_filenames = { my_module_repository: {}, step_asset: {},
step_table: {}, result_asset: {}, result_table: {} }
project_name = to_filesystem_name(p.name) + "_#{ind}"
root =
if p.archived
"#{team_path}/Archived projects"
else
"#{team_path}/Projects"
end
root += "/#{project_name}"
FileUtils.mkdir_p(root)
inventories = "#{root}/Inventories"
FileUtils.mkdir_p(inventories)
# Find all assigned inventories through all tasks in the project
task_ids = p.project_my_modules
repo_rows = RepositoryRow.joins(:my_modules)
.where(my_modules: { id: task_ids })
.distinct
# Iterate through every inventory repo and save it to CSV
repo_rows.map(&:repository).uniq.each_with_index do |repo, repo_idx|
curr_repo_rows = repo_rows.select { |x| x.repository_id == repo.id }
obj_filenames[:my_module_repository][repo.id] =
save_inventories_to_csv(inventories, repo, curr_repo_rows, repo_idx)
end
# Include all experiments
p.experiments.each_with_index do |ex, ex_ind|
experiment_path = "#{root}/#{to_filesystem_name(ex.name)}_#{ex_ind}"
FileUtils.mkdir_p(experiment_path)
# Include all modules
ex.my_modules.each_with_index do |my_module, mod_ind|
my_module_path = "#{experiment_path}/" \
"#{to_filesystem_name(my_module.name)}_#{mod_ind}"
FileUtils.mkdir_p(my_module_path)
# Create upper directories for both elements
protocol_path = "#{my_module_path}/Protocol attachments"
result_path = "#{my_module_path}/Result attachments"
FileUtils.mkdir_p(protocol_path)
FileUtils.mkdir_p(result_path)
# Export protocols
steps = my_module.protocols.map(&:steps).flatten
obj_filenames[:step_asset].merge!(
export_assets(StepAsset.where(step: steps), :step, protocol_path)
)
obj_filenames[:step_table].merge!(
export_tables(StepTable.where(step: steps), :step, protocol_path)
)
# Export results
obj_filenames[:result_asset].merge!(
export_assets(ResultAsset.where(result: my_module.results),
:result, result_path)
)
obj_filenames[:result_table].merge!(
export_tables(ResultTable.where(result: my_module.results),
:result, result_path)
)
end
end
# Generate and export whole project report PDF
pdf_name = "#{project_name}_Report.pdf"
project_report_pdf =
p.generate_report_pdf(@user, @team, pdf_name, obj_filenames)
file = FileUtils.touch("#{root}/#{pdf_name}").first
File.open(file, 'wb') { |f| f.write(project_report_pdf) }
end
end
def generate_notification(user)
notification = Notification.create(
type_of: :deliver,
title: I18n.t('zip_export.notification_title'),
message: "<a data-id='#{id}' " \
"href='#{Rails.application
.routes
.url_helpers
.zip_exports_download_export_all_path(self)}'>" \
"#{zip_file_file_name}</a>"
)
UserNotification.create(notification: notification, user: user)
end
# Appends given suffix to file_name and then adds original extension
def append_file_suffix(file_name, suffix)
ext = File.extname(file_name)
File.basename(file_name, ext) + suffix + ext
end
# Helper method to extract given assets to the directory
def export_assets(elements, type, directory)
asset_indexes = {}
elements.each_with_index do |element, i|
asset = element.asset
if type == :step
name = "#{directory}/" \
"#{append_file_suffix(asset.file_file_name,
"_#{i}_Step#{element.step.position + 1}")}"
elsif type == :result
name = "#{directory}/#{append_file_suffix(asset.file_file_name,
"_#{i}")}"
end
file = FileUtils.touch(name).first
File.open(file, 'wb') { |f| f.write(asset.open.read) }
asset_indexes[asset.id] = name
end
asset_indexes
end
# Helper method to extract given tables to the directory
def export_tables(elements, type, directory)
table_indexes = {}
elements.each_with_index do |element, i|
table = element.table
table_name = table.name.presence || 'Table'
table_name += i.to_s
if type == :step
name = "#{directory}/#{to_filesystem_name(table_name)}" \
"_#{i}_Step#{element.step.position + 1}.csv"
elsif type == :result
name = "#{directory}/#{to_filesystem_name(table_name)}.csv"
end
file = FileUtils.touch(name).first
File.open(file, 'wb') { |f| f.write(table.to_csv) }
table_indexes[table.id] = name
end
table_indexes
end
# Helper method for saving inventories to CSV
def save_inventories_to_csv(path, repo, repo_rows, id)
repo_name = "#{to_filesystem_name(repo.name)}_#{id}"
# Attachment folder
rel_attach_path = "#{repo_name}_attachments"
attach_path = "#{path}/#{rel_attach_path}"
FileUtils.mkdir_p(attach_path)
# CSV file
csv_file_path = "#{path}/#{to_filesystem_name(repo.name)}_#{id}.csv"
csv_file = FileUtils.touch(csv_file_path).first
# Define headers and columns IDs
col_ids = [-3, -4, -5, -6] + repo.repository_columns.map(&:id)
# Define callback function for file name
assets = {}
asset_counter = 0
handle_name_func = lambda do |asset|
file_name = append_file_suffix(asset.file_file_name,
"_#{asset_counter}").to_s
# Save pair for downloading it later
assets[asset] = "#{attach_path}/#{file_name}"
asset_counter += 1
rel_path = "#{rel_attach_path}/#{file_name}"
return "=HYPERLINK(\"#{rel_path}\", \"#{rel_path}\")"
end
# Generate CSV
csv_data = RepositoryZipExport.to_csv(repo_rows, col_ids, @user, @team,
handle_name_func)
File.open(csv_file, 'wb') { |f| f.write(csv_data) }
# Save all attachments (it doesn't work directly in callback function
assets.each do |asset, asset_path|
file = FileUtils.touch(asset_path).first
File.open(file, 'wb') { |f| f.write asset.open.read }
end
repo_name
end
# Recursive zipping
def zip!(input_dir, output_file)
files = Dir.entries(input_dir)
# Don't zip current/above directory
files.delete_if { |el| ['.', '..'].include?(el) }
Zip::File.open(output_file, Zip::File::CREATE) do |zipfile|
write_entries(input_dir, files, '', zipfile)
end
end
# A helper method to make the recursion work.
def write_entries(input_dir, entries, path, io)
entries.each do |e|
zip_file_path = path == '' ? e : File.join(path, e)
disk_file_path = File.join(input_dir, zip_file_path)
puts 'Deflating ' + disk_file_path
if File.directory?(disk_file_path)
io.mkdir(zip_file_path)
subdir = Dir.entries(disk_file_path)
# Remove current/above directory to prevent infinite recursion
subdir.delete_if { |el| ['.', '..'].include?(el) }
write_entries(input_dir, subdir, zip_file_path, io)
else
io.get_output_stream(zip_file_path) do |f|
f.puts File.open(disk_file_path, 'rb').read
end
end
end
end
end

15
app/models/user.rb
View file

@ -1,6 +1,9 @@
class User < ApplicationRecord
include SearchableModel, SettingsModel
include User::TeamRoles, User::ProjectRoles
include SearchableModel
include SettingsModel
include VariablesModel
include User::TeamRoles
include User::ProjectRoles
acts_as_token_authenticatable
devise :invitable, :confirmable, :database_authenticatable, :registerable,
@ -47,6 +50,14 @@ class User < ApplicationRecord
}
)
store_accessor :variables, :export_vars
default_variables(
export_vars: {
num_of_export_all_last_24_hours: 0
}
)
# Relations
has_many :user_identities, inverse_of: :user
has_many :user_teams, inverse_of: :user

1
app/models/zip_export.rb
View file

@ -1,5 +1,6 @@
require 'zip'
require 'fileutils'
require 'csv'
# To use ZipExport you have to define the generate_( type )_zip method!
# Example:

14
app/services/repository_zip_export.rb
View file

@ -16,7 +16,7 @@ module RepositoryZipExport
)
end
def self.to_csv(rows, column_ids, user, team)
def self.to_csv(rows, column_ids, user, team, handle_file_name_func = nil)
# Parse column names
csv_header = []
column_ids.each do |c_id|
@ -56,10 +56,16 @@ module RepositoryZipExport
else
cell = row.repository_cells
.find_by(repository_column_id: c_id)
if cell
SmartAnnotations::TagToText.new(
user, team, cell.value.formatted
).text
if cell.value_type == 'RepositoryAssetValue' &&
handle_file_name_func
handle_file_name_func.call(cell.value.asset)
else
SmartAnnotations::TagToText.new(
user, team, cell.value.formatted
).text
end
end
end
end

6
app/views/projects/export/_error.html.erb Normal file
View file

@ -0,0 +1,6 @@
<p>
It looks like you have exceeded your daily export limit. The number of exports is <strong>limited to <%= limit %> requests per day </strong>- you currently have 0 export requests left.
</p>
<p>Please repeat the desired action <strong>tomorrow</strong>, when your daily limit will reset back to <%= limit %> export requests.
</p>

12
app/views/projects/export/_success.html.erb Normal file
View file

@ -0,0 +1,12 @@
<p>Your export request for <strong><%= num_projects %> projects in <%= @team.name %> </strong> team is being processed and will soon be exported in a .zip file format. It may take anywhere from 5 minutes up to several hours, depending on the file size. Please also note, that any new data in the projects from this point will not be included in the exported file.
</p>
<p>
When it's ready, you will receive a confirmation e-mail and the <strong> link </strong> to your file will appear in your <strong>SciNote notifications</strong>. For security reasons, the link will <strong>expire in 7 days</strong>.
</p>
<p>
<i>
Please note that the number of exports is limited to <%= limit %> requests per day - you currently have <%= num_of_requests_left %> more.
</i>
</p>

25
app/views/projects/index.html.erb
View file

@ -77,6 +77,23 @@
</div>
</div>
<!-- Export projects modal -->
<div class="modal" id="export-projects-modal" tabindex="-1" role="dialog" aria-labelledby="export-projects-modal-label">
<div class="modal-dialog" role="document">
<div class="modal-content">
<div class="modal-header">
<button type="button" class="close" data-dismiss="modal" aria-label="Close"><span aria-hidden="true">&times;</span></button>
<h4 class="modal-title" id="export-projects-modal-label"></h4>
</div>
<div class="modal-body">
</div>
<div class="modal-footer">
<button type="button" class="btn btn-default" data-dismiss="modal"><%=t "general.close" %></button>
</div>
</div>
</div>
</div>
<div id="projects-toolbar">
<form class="form-inline" action="<%= projects_path %>">
@ -111,6 +128,14 @@
</ul>
</div>
<!-- export projects button -->
<button type="button" class="btn btn-default pull-right"
id="export-projects-button"
data-export-projects-url="<%= export_projects_team_path(current_team) %>">
<span class="fas fa-file-export"></span>
<span class="hidden-xs-custom"><%= t("projects.export_projects.export_button") %></span>
</button>
</div>
</form>
</div>

8
app/views/reports/elements/_my_module_repository_element.html.erb
View file

@ -13,6 +13,14 @@
<div class="pull-left repository-name">
<%=t "projects.reports.elements.module_repository.name", repository: repository.name, my_module: my_module.name %>
</div>
<% if defined? export_all and export_all %>
<div class="pull-left table-name">
<a href="<%= path %>">
<em><%=t "projects.reports.elements.module_repository.table_name",
name: filename %></em>
</a>
</div>
<% end %>
<div class="pull-right controls">
<%= render partial: "reports/elements/element_controls.html.erb", locals: { show_sort: true } %>
</div>

22
app/views/reports/elements/_my_module_result_asset_element.html.erb
View file

@ -1,25 +1,29 @@
<% if result.blank? and @result.present? then result = @result end %>
<% result ||= @result %>
<% asset = result.asset %>
<% is_image = result.asset.is_image? %>
<% comments = result.result_comments %>
<% timestamp = asset.created_at %>
<% name = result.name %>
<% icon_class = 'fas ' + (is_image ? 'fa-image' : 'fa-file') %>
<div class="report-element report-result-element report-result-asset-element" data-ts="<%= timestamp.to_i %>" data-type="result_asset" data-id='{ "result_id": <%= result.id %> }' data-scroll-id="<%= result.id %>" data-modal-title="<%=t "projects.reports.elements.modals.result_contents.head_title", result: result.name %>" data-name="<%= name %>" data-icon-class="<%= icon_class %>">
<div class="report-element report-result-element report-result-asset-element" data-ts="<%= timestamp.to_i %>" data-type="result_asset" data-id='{ "result_id": <%= result.id %> }' data-scroll-id="<%= result.id %>" data-modal-title="<%=t "projects.reports.elements.modals.result_contents.head_title", result: result.name %>" data-name="<%= result.name %>" data-icon-class="<%= icon_class %>">
<div class="report-element-header">
<div class="row">
<div class="pull-left result-icon">
<span class="<%= icon_class %>"></span>
</div>
<div class="pull-left result-name">
<%= name %>
<%= result.name %>
</div>
<div class="pull-left file-name">
<em><%=t "projects.reports.elements.result_asset.file_name",
file: truncate(asset.file_file_name,
length: Constants::FILENAME_TRUNCATION_LENGTH)
%>
</em>
<% if defined? export_all and export_all %>
<a href="<%= path %>">
<em><%=t "projects.reports.elements.result_asset.file_name",
file: filename %></em>
</a>
<% else %>
<em><%=t "projects.reports.elements.result_asset.file_name",
file: truncate(asset.file_file_name,
length: Constants::FILENAME_TRUNCATION_LENGTH) %></em>
<% end %>
</div>
<div class="pull-left user-time">
<%=t "projects.reports.elements.result_asset.user_time", user: result.user.full_name, timestamp: l(timestamp, format: :full) %>

29
app/views/reports/elements/_my_module_result_table_element.html.erb
View file

@ -1,24 +1,31 @@
<% if result.blank? and @result.present? then result = @result end %>
<% result ||= @result %>
<% table = result.table %>
<% comments = result.result_comments %>
<% timestamp = table.created_at %>
<% name = result.name %>
<div class="report-element report-result-element report-result-table-element" data-ts="<%= timestamp.to_i %>" data-type="result_table" data-id='{ "result_id": <%= result.id %> }' data-scroll-id="<%= result.id %>" data-modal-title="<%=t "projects.reports.elements.modals.result_contents.head_title", result: result.name %>" data-name="<%= name %>" data-icon-class="fas fa-table">
<div class="report-element report-result-element report-result-table-element" data-ts="<%= timestamp.to_i %>" data-type="result_table" data-id='{ "result_id": <%= result.id %> }' data-scroll-id="<%= result.id %>" data-modal-title="<%=t "projects.reports.elements.modals.result_contents.head_title", result: result.name %>" data-name="<%= result.name %>" data-icon-class="fas fa-table">
<div class="report-element-header">
<div class="row">
<div class="pull-left result-name-container">
<div class="result-icon">
<span class="fas fa-table"></span>
<div class="pull-left result-icon">
<span class="fas fa-table"></span>
</div>
<div class="result-name">
<%= name %>
<div class="pull-left result-name">
<%= result.name %>
</div>
<div class="user-time">
<% if defined? export_all and export_all %>
<div class="pull-left table-name">
<a href="<%= path %>">
<em><%=t "projects.reports.elements.result_table.table_name",
name: filename %></em>
</a>
</div>
<% end %>
<div class="pull-left user-time">
<%=t "projects.reports.elements.result_table.user_time", user: result.user.full_name , timestamp: l(timestamp, format: :full) %>
</div>
</div>
<div class="pull-right controls">
<%= render partial: "reports/elements/element_controls.html.erb" %>
<div class="pull-right controls">
<%= render partial: "reports/elements/element_controls.html.erb" %>
</div>
</div>
</div>
</div>

13
app/views/reports/elements/_step_asset_element.html.erb
View file

@ -1,4 +1,4 @@
<% if asset.blank? and @asset.present? then asset = @asset end %>
<% asset ||= @asset %>
<% is_image = asset.is_image? %>
<% timestamp = asset.created_at %>
<% icon_class = 'fas ' + (is_image ? 'fa-image' : 'fa-file') %>
@ -9,7 +9,16 @@
<span class="<%= icon_class %>"></span>
</div>
<div class="pull-left file-name">
<em><%=t 'projects.reports.elements.step_asset.file_name', file: truncate( asset.file_file_name, length: Constants::FILENAME_TRUNCATION_LENGTH) %></em>
<% if defined? export_all and export_all %>
<a href="<%= path %>">
<em><%=t 'projects.reports.elements.step_asset.file_name',
file: filename %></em>
</a>
<% else %>
<em><%=t 'projects.reports.elements.step_asset.file_name',
file: truncate(asset.file_file_name,
length: Constants::FILENAME_TRUNCATION_LENGTH) %></em>
<% end %>
</div>
<div class="pull-left user-time">
<%=t 'projects.reports.elements.step_asset.user_time', timestamp: l(timestamp, format: :full) %>

21
app/views/reports/elements/_step_table_element.html.erb
View file

@ -1,4 +1,4 @@
<% if table.blank? and @table.present? then table = @table end %>
<% table ||= @table %>
<% timestamp = table.created_at %>
<div class="report-element report-step-attachment-element report-step-table-element" data-ts="<%= timestamp.to_i %>" data-type="step_table" data-id='{ "table_id": <%= table.id %> }' data-scroll-id="<%= table.id %>" data-name="<%= table.name %>" data-icon-class="fas fa-table">
<div class="report-element-header">
@ -6,11 +6,20 @@
<div class="pull-left attachment-icon">
<span class="fas fa-table"></span>
</div>
<% if table && table.name %>
<div class="pull-left table-name">
<em><%=t 'projects.reports.elements.step_table.table_name', name: table.name %></em>
</div>
<% end %>
<div class="pull-left table-name">
<% if defined? export_all and export_all %>
<a href="<%= path %>">
<em><%=t 'projects.reports.elements.step_table.table_name',
name: filename %></em>
</a>
<% else %>
<% if table.try(:name) %>
<em><%=t 'projects.reports.elements.step_table.table_name',
name: truncate(table.name,
length: Constants::FILENAME_TRUNCATION_LENGTH) %></em>
<% end %>
<% end %>
</div>
<div class="pull-left user-time">
<%=t 'projects.reports.elements.step_table.user_time', timestamp: l(timestamp, format: :full) %>
</div>

14
app/views/reports/new.html.erb
View file

@ -14,15 +14,15 @@
id="data-holder"
class="hidden"
data-project-modal-title="<%=t "projects.reports.elements.modals.project_contents.head_title" %>"
data-add-project-contents-url="<%= project_contents_modal_project_reports_url %>"
data-add-experiment-contents-url="<%= experiment_contents_modal_project_reports_url %>"
data-add-module-contents-url="<%= module_contents_modal_project_reports_url %>"
data-add-step-contents-url="<%= step_contents_modal_project_reports_url %>"
data-add-result-contents-url="<%= result_contents_modal_project_reports_url %>"
data-add-project-contents-url="<%= project_contents_modal_project_reports_url(@project) %>"
data-add-experiment-contents-url="<%= experiment_contents_modal_project_reports_url(@project) %>"
data-add-module-contents-url="<%= module_contents_modal_project_reports_url(@project) %>"
data-add-step-contents-url="<%= step_contents_modal_project_reports_url(@project) %>"
data-add-result-contents-url="<%= result_contents_modal_project_reports_url(@project) %>"
data-stylesheet-url="<%= stylesheet_path "application" %>"
data-print-title="<%=t "projects.reports.print_title", project: @project.name %>"
data-project-id="<%= @project.id %>"
data-save-report-url="<%= save_modal_project_reports_url %>"
data-save-report-url="<%= save_modal_project_reports_url(@project) %>"
data-report-id="<%= @report.present? ? @report.id : "" %>"
data-unsaved-work-text="<%=t "projects.reports.new.unsaved_work" %>"
data-global-sort-text="<%=t "projects.reports.new.global_sort" %>"></div>
@ -32,7 +32,7 @@
<% if @report.present? %>
<% @report.root_elements.each do |el| %>
<%= render_report_element(el) %>
<%= render_report_element(el, local_assigns) %>
<%= render_new_element(false) %>
<% end %>
<% else %>

3
app/views/user_notifications/_list.html.erb
View file

@ -9,6 +9,9 @@
</div>
<div class="col-xs-9 col-md-11">
<strong><%= sanitize_input(notification.title) %></strong> <br>
<% if notification.deliver? %>
Click the link to download the file.</br>
<% end %>
<%= l(notification.created_at, format: :full) %> | <%= sanitize_input(notification.message) %>
</div>
</div>

3
app/views/user_notifications/_recent_notifications.html.erb
View file

@ -9,6 +9,9 @@
<div class="col-xs-10">
<strong><%= sanitize_input(notification.title) %></strong> <br>
<% if notification.deliver? %>
Click the link to download the file.</br>
<% end %>
<%= l(notification.created_at, format: :full) %> | <%= sanitize_input(notification.message) %>
</div>
</div>

17
app/views/users/mailer/notification.html.erb
View file

@ -1,13 +1,16 @@
<p>Hello <%= @user.name %>!</p>
<p>Hello <%= @user.name %>,</p>
<p><%= I18n.t("notifications.email_title") %></p>
<p>Type: <%= I18n.t("notifications.types.#{@notification.type_of}") %></p>
<% unless @notification.deliver? %>
<p><%= I18n.t("notifications.email_title") %></p>
<p>Type: <%= I18n.t("notifications.types.#{@notification.type_of}") %></p>
<% end %>
<p>
<% if @notification.system_message? || @notification.deliver? %>
<% if @notification.system_message? %>
<% # We assume the system notification is clean %>
<%= @notification.title.html_safe %>
<% elsif @notification.deliver? %>
<%= I18n.t("notifications.deliver.email_body").html_safe %>
<% else %>
<%= sanitize_input(prepend_server_url_to_links(@notification.title)) %>
<% end %>
@ -18,8 +21,12 @@
<% # We assume the system notification is clean %>
<%= @notification.message.html_safe %>
<% elsif @notification.deliver? %>
<p>
<%= I18n.t("notifications.deliver.email_end_greeting").html_safe %>
</p>
<% # work around the problem with inserting the link of zipExport %>
<% zip_id = /data-id='(\d*)'/.match(@notification.message)[1] %>
<%= I18n.t("notifications.deliver.download_link") %>
<%= @notification.message.gsub(/href='.+'/, "href='#{zip_exports_download_url(zip_id)}'").html_safe %>
<% else %>
<%= sanitize_input(prepend_server_url_to_links(@notification.message)) %>

3
config/initializers/constants.rb
View file

@ -25,6 +25,9 @@ class Constants
DROPDOWN_TEXT_MAX_LENGTH = 15
# Max characters for filenames, after which they get truncated
FILENAME_TRUNCATION_LENGTH = 50
# Max characters for names of exported files and folders, after which they get
# truncated
EXPORTED_FILENAME_TRUNCATION_LENGTH = 20
USER_INITIALS_MAX_LENGTH = 4
# Password 'key stretching' factor

13
config/locales/en.yml
View file

@ -288,6 +288,10 @@ en:
invite_users_link: "Invite users"
invite_users_details: "to team %{team}."
contact_admins: "To invite additional users to team %{team}, contact its administrator/s."
export_projects:
export_button: "Export projects..."
modal_title_success: 'Your export is being prepared'
modal_title_error: 'Your export is denied'
table:
status: "Status"
name: "Project name"
@ -464,11 +468,13 @@ en:
no_samples: "No samples"
module_repository:
name: "%{repository} of task %{my_module}"
table_name: "[ %{name} ]"
no_items: "No items"
result_asset:
file_name: "[ %{file} ]"
user_time: "Uploaded by %{user} on %{timestamp}."
result_table:
table_name: "[ %{name} ]"
user_time: "Created by %{user} on %{timestamp}."
result_text:
user_time: "Created by %{user} on %{timestamp}."
@ -1829,6 +1835,11 @@ en:
recent_notification_description: 'Recent changes notifications appear whenever there is a change on a task you are assigned to.'
system_message: 'System message'
system_message_description: 'System message notifications are specifically sent by site maintainers to notify all users about a system update.'
deliver:
download_link: "Download link:"
email_subject: "Your SciNote export is ready!"
email_body: "<p>The export of SciNote project(s) that you requested is ready!</p><p>You can find the link to download the file below or in your SciNote notifications. Please keep in mind that the link will expire in 7 days for security reasons.</p>"
email_end_greeting: "<p>Best regards,</p><p>Your SciNote team</p>"
show_all: "Show all notifications"
show_more: "Show more notifications"
no_notifications: "No notifications."
@ -1896,7 +1907,7 @@ en:
zip_export:
modal_label: 'Export inventory'
notification_title: 'Your package is ready to be exported!'
notification_title: 'Your requested export package is ready!'
expired_title: 'Looks like your link has expired.'
expired_description: 'Please export the data again in order to receive a new link.'
modal_label: 'Export request received'

6
config/routes.rb
View file

@ -137,6 +137,11 @@ Rails.application.routes.draw do
to: 'zip_exports#download',
as: 'zip_exports_download'
# Get Team Zip Export
get 'zip_exports/download_export_all_zip/:id',
to: 'zip_exports#download_export_all_zip',
as: 'zip_exports_download_export_all'
get 'zip_exports/file_expired',
to: 'zip_exports#file_expired',
as: 'file_expired'
@ -173,6 +178,7 @@ Rails.application.routes.draw do
# post 'import_samples'
# post 'export_samples'
post 'export_repository', to: 'repositories#export_repository'
post 'export_projects'
# Used for atwho (smart annotations)
get 'atwho_users', to: 'at_who#users'
get 'atwho_repositories', to: 'at_who#repositories'

21
db/migrate/20180930205254_add_variables_to_users.rb Normal file
View file

@ -0,0 +1,21 @@
# frozen_string_literal: true
class AddVariablesToUsers < ActiveRecord::Migration[5.1]
def up
add_column :users, :variables, :jsonb, default: {}, null: false
User.find_each do |user|
variables = {
export_vars: {
num_of_export_all_last_24_hours: 0
}
}
user.update(variables: variables)
end
end
def down
remove_column :users, :variables, :jsonb
end
end

1
db/schema.rb
View file

@ -786,6 +786,7 @@ ActiveRecord::Schema.define(version: 20181008130519) do
t.integer "current_team_id"
t.string "authentication_token", limit: 30
t.jsonb "settings", default: {}, null: false
t.jsonb "variables", default: {}, null: false
t.index "trim_html_tags((full_name)::text) gin_trgm_ops", name: "index_users_on_full_name", using: :gin
t.index ["authentication_token"], name: "index_users_on_authentication_token", unique: true
t.index ["confirmation_token"], name: "index_users_on_confirmation_token", unique: true

19
lib/tasks/exportable_items.rake
View file

@ -6,4 +6,23 @@ namespace :exportable_items do
puts "All exportable zip files older than " \
"'#{num.days.ago}' have been removed"
end
desc 'Resets export project counter to 0'
task reset_export_projects_counter: :environment do
User.find_each do |user|
User.transaction do
begin
user.export_vars['num_of_export_all_last_24_hours'] = 0
user.save
rescue ActiveRecord::ActiveRecordError,
ArgumentError,
ActiveRecord::RecordNotSaved => e
puts "Error resetting users num_of_export_all_last_24_hours " \
"variable to 0, transaction reverted: #{e}"
end
end
end
puts 'Export project counter successfully ' \
'reset on all users'
end
end

2
package.json
View file

@ -24,7 +24,7 @@
"@fortawesome/fontawesome-free": "^5.2.0",
"babel-eslint": "^8.2.6",
"babel-plugin-transform-react-jsx-source": "^6.22.0",
"eslint": "^5.2.0",
"eslint": "^5.3.0",
"eslint-config-airbnb": "^15.1.0",
"eslint-config-airbnb-base": "^13.0.0",
"eslint-config-google": "^0.9.1",

5
spec/models/user_spec.rb
View file

@ -37,6 +37,7 @@ describe User, type: :model do
it { should have_db_column :invited_by_type }
it { should have_db_column :invitations_count }
it { should have_db_column :settings }
it { should have_db_column :variables }
it { should have_db_column :current_team_id }
it { should have_db_column :authentication_token }
end
@ -179,6 +180,10 @@ describe User, type: :model do
it { is_expected.to respond_to(:system_message_email_notification) }
end
describe 'user variables' do
it { is_expected.to respond_to(:export_vars) }
end
describe '#last_activities' do
let!(:user) { create :user }
let!(:project) { create :project }