Implement search in file repository cells and improve performance [SCI-2111]

This commit is contained in:
Oleksii Kriuchykhin 2018-03-26 13:19:09 +02:00
parent 35afc06cbd
commit e42fc3117b
10 changed files with 174 additions and 102 deletions

View file

@ -1,10 +1,11 @@
GIT GIT
remote: https://github.com/biosistemika/canaid remote: https://github.com/biosistemika/canaid
revision: f2000c19b75e66ea929a44cb0575262b7f5fc13e revision: 943ae9b9801819fd2513f6ab9e1143ad8de523ce
branch: master branch: master
specs: specs:
canaid (1.0.1) canaid (1.0.2)
devise (>= 3.4.1) devise (>= 3.4.1)
docile (>= 1.1.0)
rails (>= 4) rails (>= 4)
GIT GIT

View file

@ -4,6 +4,8 @@ class SearchController < ApplicationController
def index def index
redirect_to new_search_path unless @search_query redirect_to new_search_path unless @search_query
@search_id = params[:search_id] ? params[:search_id] : generate_search_id
count_search_results count_search_results
search_projects if @search_category == :projects search_projects if @search_category == :projects
@ -86,6 +88,10 @@ class SearchController < ApplicationController
protected protected
def generate_search_id
SecureRandom.urlsafe_base64(32)
end
def search_by_name(model) def search_by_name(model)
model.search(current_user, model.search(current_user,
true, true,
@ -109,52 +115,61 @@ class SearchController < ApplicationController
end end
def count_by_repository def count_by_repository
count_total = 0 @repository_search_count =
search_results = Repository.search(current_user, Rails.cache.fetch("#{@search_id}/repository_search_count",
@search_query, expires_in: 5.minutes) do
Constants::SEARCH_NO_LIMIT, search_count = {}
nil, search_results = Repository.search(current_user,
match_case: @search_case, @search_query,
whole_word: @search_whole_word, Constants::SEARCH_NO_LIMIT,
whole_phrase: @search_whole_phrase) nil,
@repository_search_count = {} match_case: @search_case,
current_user.teams.includes(:repositories).each do |team| whole_word: @search_whole_word,
team_results = {} whole_phrase: @search_whole_phrase)
team_results[:count] = 0
team_results[:repositories] = {} current_user.teams.includes(:repositories).each do |team|
team.repositories.each do |repository| team_results = {}
repository_results = {} team_results[:count] = 0
repository_results[:id] = repository.id team_results[:repositories] = {}
repository_results[:count] = 0 team.repositories.each do |repository|
search_results.each do |result| repository_results = {}
if repository.id == result.id repository_results[:id] = repository.id
count_total += result.counter repository_results[:count] = 0
repository_results[:count] += result.counter search_results.each do |result|
if repository.id == result.id
repository_results[:count] += result.counter
end
end
team_results[:repositories][repository.name] = repository_results
team_results[:count] += repository_results[:count]
end end
search_count[team.name] = team_results
end end
team_results[:repositories][repository.name] = repository_results search_count
team_results[:count] += repository_results[:count]
end end
@repository_search_count[team.name] = team_results
count_total = 0
@repository_search_count.each_value do |team_results|
count_total += team_results[:count]
end end
count_total count_total
end end
def count_search_results def count_search_results
@project_search_count = count_by_name Project @project_search_count = fetch_cached_count Project
@experiment_search_count = count_by_name Experiment @experiment_search_count = fetch_cached_count Experiment
@module_search_count = count_by_name MyModule @module_search_count = fetch_cached_count MyModule
@result_search_count = count_by_name Result @result_search_count = fetch_cached_count Result
@tag_search_count = count_by_name Tag @tag_search_count = fetch_cached_count Tag
@report_search_count = count_by_name Report @report_search_count = fetch_cached_count Report
@protocol_search_count = count_by_name Protocol @protocol_search_count = fetch_cached_count Protocol
@step_search_count = count_by_name Step @step_search_count = fetch_cached_count Step
@checklist_search_count = count_by_name Checklist @checklist_search_count = fetch_cached_count Checklist
@sample_search_count = count_by_name Sample @sample_search_count = fetch_cached_count Sample
@repository_search_count_total = count_by_repository @repository_search_count_total = count_by_repository
@asset_search_count = count_by_name Asset @asset_search_count = fetch_cached_count Asset
@table_search_count = count_by_name Table @table_search_count = fetch_cached_count Table
@comment_search_count = count_by_name Comment @comment_search_count = fetch_cached_count Comment
@search_results_count = @project_search_count @search_results_count = @project_search_count
@search_results_count += @experiment_search_count @search_results_count += @experiment_search_count
@ -172,6 +187,15 @@ class SearchController < ApplicationController
@search_results_count += @comment_search_count @search_results_count += @comment_search_count
end end
def fetch_cached_count(type)
exp = 5.minutes
Rails.cache.fetch(
"#{@search_id}/#{type.name.underscore}_search_count", expires_in: exp
) do
count_by_name type
end
end
def search_projects def search_projects
@project_results = [] @project_results = []
@project_results = search_by_name(Project) if @project_search_count > 0 @project_results = search_by_name(Project) if @project_search_count > 0

View file

@ -91,37 +91,19 @@ class Asset < ApplicationRecord
def self.search( def self.search(
user, user,
include_archived, _include_archived,
query = nil, query = nil,
page = 1, page = 1,
_current_team = nil, _current_team = nil,
options = {} options = {}
) )
step_ids =
Step
.search(user, include_archived, nil, Constants::SEARCH_NO_LIMIT)
.joins(:step_assets)
.distinct
.pluck('step_assets.id')
result_ids = new_query =
Result
.search(user, include_archived, nil, Constants::SEARCH_NO_LIMIT)
.joins(:result_asset)
.distinct
.pluck('result_assets.id')
ids =
Asset Asset
.select(:id)
.distinct .distinct
.joins('LEFT OUTER JOIN step_assets ON step_assets.asset_id = assets.id') .select('assets.*')
.joins('LEFT OUTER JOIN result_assets ON ' \ .left_outer_joins(:asset_text_datum)
'result_assets.asset_id = assets.id') .where(team: user.teams)
.joins('LEFT JOIN asset_text_data ON ' \
'assets.id = asset_text_data.asset_id')
.where('(step_assets.id IN (?) OR result_assets.id IN (?))',
step_ids, result_ids)
a_query = s_query = '' a_query = s_query = ''
@ -144,7 +126,7 @@ class Asset < ApplicationRecord
a_query = '\\y(' + a_query + ')\\y' a_query = '\\y(' + a_query + ')\\y'
s_query = s_query.tr('\'', '"') s_query = s_query.tr('\'', '"')
ids = ids.where( new_query = new_query.where(
"(trim_html_tags(assets.file_file_name) #{like} ? " \ "(trim_html_tags(assets.file_file_name) #{like} ? " \
"OR asset_text_data.data_vector @@ to_tsquery(?))", "OR asset_text_data.data_vector @@ to_tsquery(?))",
a_query, a_query,
@ -165,7 +147,7 @@ class Asset < ApplicationRecord
.map { |t| t + ':*' } .map { |t| t + ':*' }
.join('|') .join('|')
.tr('\'', '"') .tr('\'', '"')
ids = ids.where( new_query = new_query.where(
"(trim_html_tags(assets.file_file_name) #{like} ANY (array[?]) " \ "(trim_html_tags(assets.file_file_name) #{like} ANY (array[?]) " \
"OR asset_text_data.data_vector @@ to_tsquery(?))", "OR asset_text_data.data_vector @@ to_tsquery(?))",
a_query, a_query,
@ -175,19 +157,14 @@ class Asset < ApplicationRecord
# Show all results if needed # Show all results if needed
if page != Constants::SEARCH_NO_LIMIT if page != Constants::SEARCH_NO_LIMIT
ids = ids new_query.select("ts_headline(data, to_tsquery('" +
.limit(Constants::SEARCH_LIMIT) sanitize_sql_for_conditions(s_query) +
.offset((page - 1) * Constants::SEARCH_LIMIT) "'), 'StartSel=<mark>, StopSel=</mark>') headline")
.limit(Constants::SEARCH_LIMIT)
.offset((page - 1) * Constants::SEARCH_LIMIT)
else
new_query
end end
Asset
.joins('LEFT JOIN asset_text_data ON ' \
' assets.id = asset_text_data.asset_id')
.select('assets.*')
.select("ts_headline(data, to_tsquery('" +
sanitize_sql_for_conditions(s_query) +
"'), 'StartSel=<mark>, StopSel=</mark>') headline")
.where('assets.id IN (?)', ids)
end end
def is_image? def is_image?

View file

@ -51,7 +51,8 @@
> >
<a href="?<%= {category: 'projects', q: @search_query, <a href="?<%= {category: 'projects', q: @search_query,
whole_word: @search_whole_word, whole_phrase: @search_whole_phrase, whole_word: @search_whole_word, whole_phrase: @search_whole_phrase,
match_case: @search_case, utf8: '✓'}.to_query %>"> match_case: @search_case, utf8: '✓',
search_id: @search_id}.to_query %>">
<span class="badge pull-right"><%= @project_search_count %></span> <span class="badge pull-right"><%= @project_search_count %></span>
<span class="glyphicon glyphicon-blackboard"></span> <span class="glyphicon glyphicon-blackboard"></span>
<%= t'Projects' %> <%= t'Projects' %>
@ -64,7 +65,8 @@
> >
<a href="?<%= {category: 'experiments', q: @search_query, <a href="?<%= {category: 'experiments', q: @search_query,
whole_word: @search_whole_word, whole_phrase: @search_whole_phrase, whole_word: @search_whole_word, whole_phrase: @search_whole_phrase,
match_case: @search_case, utf8: '✓'}.to_query %>"> match_case: @search_case, utf8: '✓',
search_id: @search_id}.to_query %>">
<span class="badge pull-right"><%= @experiment_search_count %></span> <span class="badge pull-right"><%= @experiment_search_count %></span>
<%= fa_icon 'flask' %> <%= fa_icon 'flask' %>
<%= t'Experiments' %> <%= t'Experiments' %>
@ -77,7 +79,8 @@
> >
<a href="?<%= {category: 'modules', q: @search_query, <a href="?<%= {category: 'modules', q: @search_query,
whole_word: @search_whole_word, whole_phrase: @search_whole_phrase, whole_word: @search_whole_word, whole_phrase: @search_whole_phrase,
match_case: @search_case, utf8: '✓'}.to_query %>"> match_case: @search_case, utf8: '✓',
search_id: @search_id}.to_query %>">
<span class="badge pull-right"><%= @module_search_count %></span> <span class="badge pull-right"><%= @module_search_count %></span>
<span class="glyphicon glyphicon-credit-card"></span> <span class="glyphicon glyphicon-credit-card"></span>
<%= t'Modules' %> <%= t'Modules' %>
@ -90,7 +93,8 @@
> >
<a href="?<%= {category: 'results', q: @search_query, <a href="?<%= {category: 'results', q: @search_query,
whole_word: @search_whole_word, whole_phrase: @search_whole_phrase, whole_word: @search_whole_word, whole_phrase: @search_whole_phrase,
match_case: @search_case, utf8: '✓'}.to_query %>"> match_case: @search_case, utf8: '✓',
search_id: @search_id}.to_query %>">
<span class="badge pull-right"><%= @result_search_count %></span> <span class="badge pull-right"><%= @result_search_count %></span>
<span class="glyphicon glyphicon-modal-window"></span> <span class="glyphicon glyphicon-modal-window"></span>
<%= t'Results' %> <%= t'Results' %>
@ -103,7 +107,8 @@
> >
<a href="?<%= {category: 'tags', q: @search_query, <a href="?<%= {category: 'tags', q: @search_query,
whole_word: @search_whole_word, whole_phrase: @search_whole_phrase, whole_word: @search_whole_word, whole_phrase: @search_whole_phrase,
match_case: @search_case, utf8: '✓'}.to_query %>"> match_case: @search_case, utf8: '✓',
search_id: @search_id}.to_query %>">
<span class="badge pull-right"><%= @tag_search_count %></span> <span class="badge pull-right"><%= @tag_search_count %></span>
<span class="glyphicon glyphicon-tags"></span> <span class="glyphicon glyphicon-tags"></span>
<%= t'Tags' %> <%= t'Tags' %>
@ -116,7 +121,8 @@
> >
<a href="?<%= {category: 'reports', q: @search_query, <a href="?<%= {category: 'reports', q: @search_query,
whole_word: @search_whole_word, whole_phrase: @search_whole_phrase, whole_word: @search_whole_word, whole_phrase: @search_whole_phrase,
match_case: @search_case, utf8: '✓'}.to_query %>"> match_case: @search_case, utf8: '✓',
search_id: @search_id}.to_query %>">
<span class="badge pull-right"><%= @report_search_count %></span> <span class="badge pull-right"><%= @report_search_count %></span>
<span class="glyphicon glyphicon-align-left"></span> <span class="glyphicon glyphicon-align-left"></span>
<%= t'Reports' %> <%= t'Reports' %>
@ -129,7 +135,8 @@
> >
<a href="?<%= {category: 'protocols', q: @search_query, <a href="?<%= {category: 'protocols', q: @search_query,
whole_word: @search_whole_word, whole_phrase: @search_whole_phrase, whole_word: @search_whole_word, whole_phrase: @search_whole_phrase,
match_case: @search_case, utf8: '✓'}.to_query %>"> match_case: @search_case, utf8: '✓',
search_id: @search_id}.to_query %>">
<span class="badge pull-right"><%= @protocol_search_count %></span> <span class="badge pull-right"><%= @protocol_search_count %></span>
<span class="glyphicon glyphicon-list-alt"></span> <span class="glyphicon glyphicon-list-alt"></span>
<%= t'Protocols' %> <%= t'Protocols' %>
@ -142,7 +149,8 @@
> >
<a href="?<%= {category: 'steps', q: @search_query, <a href="?<%= {category: 'steps', q: @search_query,
whole_word: @search_whole_word, whole_phrase: @search_whole_phrase, whole_word: @search_whole_word, whole_phrase: @search_whole_phrase,
match_case: @search_case, utf8: '✓'}.to_query %>"> match_case: @search_case, utf8: '✓',
search_id: @search_id}.to_query %>">
<span class="badge pull-right"><%= @step_search_count %></span> <span class="badge pull-right"><%= @step_search_count %></span>
<span class="glyphicon glyphicon-circle-arrow-right"></span> <span class="glyphicon glyphicon-circle-arrow-right"></span>
<%= t'Steps' %> <%= t'Steps' %>
@ -155,7 +163,8 @@
> >
<a href="?<%= {category: 'checklists', q: @search_query, <a href="?<%= {category: 'checklists', q: @search_query,
whole_word: @search_whole_word, whole_phrase: @search_whole_phrase, whole_word: @search_whole_word, whole_phrase: @search_whole_phrase,
match_case: @search_case, utf8: '✓'}.to_query %>"> match_case: @search_case, utf8: '✓',
search_id: @search_id}.to_query %>">
<span class="badge pull-right"><%= @checklist_search_count %></span> <span class="badge pull-right"><%= @checklist_search_count %></span>
<span class="glyphicon glyphicon-list"></span> <span class="glyphicon glyphicon-list"></span>
<%= t'Checklists' %> <%= t'Checklists' %>
@ -168,7 +177,8 @@
> >
<a href="?<%= {category: 'samples', q: @search_query, <a href="?<%= {category: 'samples', q: @search_query,
whole_word: @search_whole_word, whole_phrase: @search_whole_phrase, whole_word: @search_whole_word, whole_phrase: @search_whole_phrase,
match_case: @search_case, utf8: '✓'}.to_query %>"> match_case: @search_case, utf8: '✓',
search_id: @search_id}.to_query %>">
<span class="badge pull-right"><%= @sample_search_count %></span> <span class="badge pull-right"><%= @sample_search_count %></span>
<span class="glyphicon glyphicon-tint"></span> <span class="glyphicon glyphicon-tint"></span>
<%= t'Samples' %> <%= t'Samples' %>
@ -182,7 +192,8 @@
> >
<a href="?<%= {category: 'assets', q: @search_query, <a href="?<%= {category: 'assets', q: @search_query,
whole_word: @search_whole_word, whole_phrase: @search_whole_phrase, whole_word: @search_whole_word, whole_phrase: @search_whole_phrase,
match_case: @search_case, utf8: '✓'}.to_query %>"> match_case: @search_case, utf8: '✓',
search_id: @search_id}.to_query %>">
<span class="badge pull-right"><%= @asset_search_count %></span> <span class="badge pull-right"><%= @asset_search_count %></span>
<span class="glyphicon glyphicon-file"></span> <span class="glyphicon glyphicon-file"></span>
<%= t'Assets' %> <%= t'Assets' %>
@ -195,7 +206,8 @@
> >
<a href="?<%= {category: 'tables', q: @search_query, <a href="?<%= {category: 'tables', q: @search_query,
whole_word: @search_whole_word, whole_phrase: @search_whole_phrase, whole_word: @search_whole_word, whole_phrase: @search_whole_phrase,
match_case: @search_case, utf8: '✓'}.to_query %>"> match_case: @search_case, utf8: '✓',
search_id: @search_id}.to_query %>">
<span class="badge pull-right"><%= @table_search_count %></span> <span class="badge pull-right"><%= @table_search_count %></span>
<span class="glyphicon glyphicon-th"></span> <span class="glyphicon glyphicon-th"></span>
<%= t'Tables' %> <%= t'Tables' %>
@ -208,7 +220,8 @@
> >
<a href="?<%= {category: 'comments', q: @search_query, <a href="?<%= {category: 'comments', q: @search_query,
whole_word: @search_whole_word, whole_phrase: @search_whole_phrase, whole_word: @search_whole_word, whole_phrase: @search_whole_phrase,
match_case: @search_case, utf8: '✓'}.to_query %>"> match_case: @search_case, utf8: '✓',
search_id: @search_id}.to_query %>">
<span class="badge pull-right"><%= @comment_search_count %></span> <span class="badge pull-right"><%= @comment_search_count %></span>
<span class="glyphicon glyphicon-comment"></span> <span class="glyphicon glyphicon-comment"></span>
<%= t'Comments' %> <%= t'Comments' %>
@ -229,7 +242,8 @@
<a href="?<%= {category: 'repositories', <a href="?<%= {category: 'repositories',
repository: values[:id], q: @search_query, repository: values[:id], q: @search_query,
whole_word: @search_whole_word, whole_phrase: @search_whole_phrase, whole_word: @search_whole_word, whole_phrase: @search_whole_phrase,
match_case: @search_case, utf8: '✓'}.to_query %>"> match_case: @search_case, utf8: '✓',
search_id: @search_id}.to_query %>">
<span class="badge pull-right"><%= values[:count] %></span> <span class="badge pull-right"><%= values[:count] %></span>
<%= repository %> <%= repository %>
</a> </a>

View file

@ -12,13 +12,6 @@
<%= render partial: "search/results/partials/asset_text.html.erb", locals: { asset: asset, query: search_query } %> <%= render partial: "search/results/partials/asset_text.html.erb", locals: { asset: asset, query: search_query } %>
</h5> </h5>
<!-- Display asset contents if it exists -->
<% if asset.headline.present? && !asset.headline.empty? && asset.headline.include?("<mark>") %>
<blockquote class="blockquote-search">
<p><%= highlight(sanitize_input(asset.headline), search_query.strip.split(/\s+/)) %></p>
</blockquote>
<% end %>
<p> <p>
<span> <span>
<%=t "search.index.created_at" %> <%=t "search.index.created_at" %>
@ -93,6 +86,24 @@
<%= render partial: "search/results/partials/team_text.html.erb", <%= render partial: "search/results/partials/team_text.html.erb",
locals: { team: asset.result.my_module.experiment.project.team } %> locals: { team: asset.result.my_module.experiment.project.team } %>
</span> </span>
<% elsif asset.repository_asset_value %>
<span>
<%=t "search.index.repository_row" %>
<%= render partial: "search/results/partials/repository_row_text.html.erb",
locals: { repository_row: asset.repository_asset_value.repository_cell.repository_row } %>
</span>
<br>
<span>
<%=t "search.index.repository" %>
<%= render partial: "search/results/partials/repository_text.html.erb",
locals: { repository: asset.repository_asset_value.repository_cell.repository_row.repository } %>
</span>
<br>
<span>
<%=t "search.index.team" %>
<%= render partial: "search/results/partials/team_text.html.erb",
locals: { team: asset.team } %>
</span>
<% end %> <% end %>
</p> </p>

View file

@ -1,15 +1,43 @@
<% query ||= nil %> <% query ||= nil %>
<% asset_read_allowed = false %>
<% text = query.present? ? highlight(asset.file_file_name, query.strip.split(/\s+/)) : asset.file_file_name %> <% text = query.present? ? highlight(asset.file_file_name, query.strip.split(/\s+/)) : asset.file_file_name %>
<% if asset.step %> <% if asset.step %>
<% protocol = asset.step.protocol %> <% protocol = asset.step.protocol %>
<% if can_read_protocol_in_module?(protocol) || <% if can_read_protocol_in_module?(protocol) ||
can_read_protocol_in_repository?(protocol) || can_read_protocol_in_repository?(protocol) %>
(asset.result && can_read_experiment?(protocol.my_module.experiment)) %> <% asset_read_allowed = true %>
<a href="<%= download_asset_path asset %>" target="_blank"> <a href="<%= download_asset_path asset %>" target="_blank">
<%= text %> <%= text %>
</a> </a>
<% else %>
<%= text %>
<% end %>
<% elsif asset.result %>
<% if can_read_experiment?(asset.result.my_module.experiment) %>
<% asset_read_allowed = true %>
<a href="<%= download_asset_path asset %>" target="_blank">
<%= text %>
</a>
<% else %>
<%= text %>
<% end %>
<% elsif asset.repository_asset_value %>
<% if can_read_team?(asset.repository_asset_value.repository_cell.repository_row.repository.team) %>
<% asset_read_allowed = true %>
<a href="<%= download_asset_path asset %>" target="_blank">
<%= text %>
</a>
<% else %>
<%= text %>
<% end %> <% end %>
<% else %> <% else %>
<%= text %> <%= text %>
<% end %> <% end %>
<!-- Display asset contents if it exists -->
<% if asset_read_allowed && asset.headline.present? && asset.headline.include?("<mark>") %>
<blockquote class="blockquote-search">
<p><%= highlight(sanitize_input(asset.headline), query.strip.split(/\s+/)) %></p>
</blockquote>
<% end %>

View file

@ -0,0 +1,7 @@
<% if can_read_team?(repository_row.repository.team) %>
<%= route_to_other_team repository_path(id: repository_row.repository.id),
repository_row.repository.team,
repository_row.name %>
<% else %>
<%= repository_row.name %>
<% end %>

View file

@ -0,0 +1,7 @@
<% if can_read_team?(repository.team) %>
<%= route_to_other_team repository_path(id: repository.id),
repository.team,
repository.name %>
<% else %>
<%= repository.name %>
<% end %>

View file

@ -52,11 +52,13 @@ class Extends
# Extra attributes used for search in repositories, text columns # Extra attributes used for search in repositories, text columns
# are only supported # are only supported
REPOSITORY_EXTRA_SEARCH_ATTR = ['repository_text_values.data', REPOSITORY_EXTRA_SEARCH_ATTR = ['repository_text_values.data',
'repository_list_items.data'] 'repository_list_items.data',
'assets.file_file_name']
# Array of includes used in search query for repository rows # Array of includes used in search query for repository rows
REPOSITORY_SEARCH_INCLUDES = [:repository_text_value, REPOSITORY_SEARCH_INCLUDES = [:repository_text_value,
repository_list_value: :repository_list_item] repository_list_value: :repository_list_item,
repository_asset_value: :asset]
# List of implemented core API versions # List of implemented core API versions
API_VERSIONS = ['20170715'] API_VERSIONS = ['20170715']

View file

@ -152,6 +152,8 @@ en:
description: "Description: " description: "Description: "
no_description: "No description" no_description: "No description"
team: "Team: " team: "Team: "
repository: "Repository: "
repository_row: "Repository item: "
project: "Project: " project: "Project: "
experiment: "Experiment: " experiment: "Experiment: "
protocol: "Protocol: " protocol: "Protocol: "
@ -211,7 +213,6 @@ en:
one: "1 day" one: "1 day"
other: "%{count} days" other: "%{count} days"
module_one_day_due_html: "Task <em>%{module}</em> is due in less than 1 day." module_one_day_due_html: "Task <em>%{module}</em> is due in less than 1 day."
new_comment: "New comment"
user_role: "Role: " user_role: "Role: "
user_full_name: "User: " user_full_name: "User: "
edit_user: "Edit role" edit_user: "Edit role"