Implement search in file repository cells and improve performance [SCI-2111]

Gemfile.lock

@@ -1,10 +1,11 @@
 GIT
   remote: https://github.com/biosistemika/canaid
-  revision: f2000c19b75e66ea929a44cb0575262b7f5fc13e
+  revision: 943ae9b9801819fd2513f6ab9e1143ad8de523ce
   branch: master
   specs:
-    canaid (1.0.1)
+    canaid (1.0.2)
       devise (>= 3.4.1)
+      docile (>= 1.1.0)
       rails (>= 4)
 
 GIT

app/controllers/search_controller.rb

@@ -4,6 +4,8 @@ class SearchController < ApplicationController
   def index
     redirect_to new_search_path unless @search_query
 
+    @search_id = params[:search_id] ? params[:search_id] : generate_search_id
+
     count_search_results
 
     search_projects if @search_category == :projects
@@ -86,6 +88,10 @@ class SearchController < ApplicationController
 
   protected
 
+  def generate_search_id
+    SecureRandom.urlsafe_base64(32)
+  end
+
   def search_by_name(model)
     model.search(current_user,
                  true,
@@ -109,52 +115,61 @@ class SearchController < ApplicationController
   end
 
   def count_by_repository
-    count_total = 0
-    search_results = Repository.search(current_user,
-                                       @search_query,
-                                       Constants::SEARCH_NO_LIMIT,
-                                       nil,
-                                       match_case: @search_case,
-                                       whole_word: @search_whole_word,
-                                       whole_phrase: @search_whole_phrase)
-    @repository_search_count = {}
-    current_user.teams.includes(:repositories).each do |team|
-      team_results = {}
-      team_results[:count] = 0
-      team_results[:repositories] = {}
-      team.repositories.each do |repository|
-        repository_results = {}
-        repository_results[:id] = repository.id
-        repository_results[:count] = 0
-        search_results.each do |result|
-          if repository.id == result.id
-            count_total += result.counter
-            repository_results[:count] += result.counter
+    @repository_search_count =
+      Rails.cache.fetch("#{@search_id}/repository_search_count",
+                        expires_in: 5.minutes) do
+        search_count = {}
+        search_results = Repository.search(current_user,
+                                           @search_query,
+                                           Constants::SEARCH_NO_LIMIT,
+                                           nil,
+                                           match_case: @search_case,
+                                           whole_word: @search_whole_word,
+                                           whole_phrase: @search_whole_phrase)
+
+        current_user.teams.includes(:repositories).each do |team|
+          team_results = {}
+          team_results[:count] = 0
+          team_results[:repositories] = {}
+          team.repositories.each do |repository|
+            repository_results = {}
+            repository_results[:id] = repository.id
+            repository_results[:count] = 0
+            search_results.each do |result|
+              if repository.id == result.id
+                repository_results[:count] += result.counter
+              end
+            end
+            team_results[:repositories][repository.name] = repository_results
+            team_results[:count] += repository_results[:count]
           end
+          search_count[team.name] = team_results
         end
-        team_results[:repositories][repository.name] = repository_results
-        team_results[:count] += repository_results[:count]
+        search_count
       end
-      @repository_search_count[team.name] = team_results
+
+    count_total = 0
+    @repository_search_count.each_value do |team_results|
+      count_total += team_results[:count]
     end
     count_total
   end
 
   def count_search_results
-    @project_search_count = count_by_name Project
-    @experiment_search_count = count_by_name Experiment
-    @module_search_count = count_by_name MyModule
-    @result_search_count = count_by_name Result
-    @tag_search_count = count_by_name Tag
-    @report_search_count = count_by_name Report
-    @protocol_search_count = count_by_name Protocol
-    @step_search_count = count_by_name Step
-    @checklist_search_count = count_by_name Checklist
-    @sample_search_count = count_by_name Sample
+    @project_search_count = fetch_cached_count Project
+    @experiment_search_count = fetch_cached_count Experiment
+    @module_search_count = fetch_cached_count MyModule
+    @result_search_count = fetch_cached_count Result
+    @tag_search_count = fetch_cached_count Tag
+    @report_search_count = fetch_cached_count Report
+    @protocol_search_count = fetch_cached_count Protocol
+    @step_search_count = fetch_cached_count Step
+    @checklist_search_count = fetch_cached_count Checklist
+    @sample_search_count = fetch_cached_count Sample
     @repository_search_count_total = count_by_repository
-    @asset_search_count = count_by_name Asset
-    @table_search_count = count_by_name Table
-    @comment_search_count = count_by_name Comment
+    @asset_search_count = fetch_cached_count Asset
+    @table_search_count = fetch_cached_count Table
+    @comment_search_count = fetch_cached_count Comment
 
     @search_results_count = @project_search_count
     @search_results_count += @experiment_search_count
@@ -172,6 +187,15 @@ class SearchController < ApplicationController
     @search_results_count += @comment_search_count
   end
 
+  def fetch_cached_count(type)
+    exp = 5.minutes
+    Rails.cache.fetch(
+      "#{@search_id}/#{type.name.underscore}_search_count", expires_in: exp
+    ) do
+      count_by_name type
+    end
+  end
+
   def search_projects
     @project_results = []
     @project_results = search_by_name(Project) if @project_search_count > 0

app/models/asset.rb

@@ -91,37 +91,19 @@ class Asset < ApplicationRecord
 
   def self.search(
     user,
-    include_archived,
+    _include_archived,
     query = nil,
     page = 1,
     _current_team = nil,
     options = {}
   )
-    step_ids =
-      Step
-      .search(user, include_archived, nil, Constants::SEARCH_NO_LIMIT)
-      .joins(:step_assets)
-      .distinct
-      .pluck('step_assets.id')
 
-    result_ids =
-      Result
-      .search(user, include_archived, nil, Constants::SEARCH_NO_LIMIT)
-      .joins(:result_asset)
-      .distinct
-      .pluck('result_assets.id')
-
-    ids =
+    new_query =
       Asset
-      .select(:id)
       .distinct
-      .joins('LEFT OUTER JOIN step_assets ON step_assets.asset_id = assets.id')
-      .joins('LEFT OUTER JOIN result_assets ON ' \
-             'result_assets.asset_id = assets.id')
-      .joins('LEFT JOIN asset_text_data ON ' \
-             'assets.id = asset_text_data.asset_id')
-      .where('(step_assets.id IN (?) OR result_assets.id IN (?))',
-             step_ids, result_ids)
+      .select('assets.*')
+      .left_outer_joins(:asset_text_datum)
+      .where(team: user.teams)
 
     a_query = s_query = ''
 
@@ -144,7 +126,7 @@ class Asset < ApplicationRecord
       a_query = '\\y(' + a_query + ')\\y'
       s_query = s_query.tr('\'', '"')
 
-      ids = ids.where(
+      new_query = new_query.where(
         "(trim_html_tags(assets.file_file_name) #{like} ? " \
         "OR asset_text_data.data_vector @@ to_tsquery(?))",
         a_query,
@@ -165,7 +147,7 @@ class Asset < ApplicationRecord
                      .map { |t| t + ':*' }
                      .join('|')
                      .tr('\'', '"')
-      ids = ids.where(
+      new_query = new_query.where(
         "(trim_html_tags(assets.file_file_name) #{like} ANY (array[?]) " \
         "OR asset_text_data.data_vector @@ to_tsquery(?))",
         a_query,
@@ -175,19 +157,14 @@ class Asset < ApplicationRecord
 
     # Show all results if needed
     if page != Constants::SEARCH_NO_LIMIT
-      ids = ids
-            .limit(Constants::SEARCH_LIMIT)
-            .offset((page - 1) * Constants::SEARCH_LIMIT)
+      new_query.select("ts_headline(data, to_tsquery('" +
+                       sanitize_sql_for_conditions(s_query) +
+                       "'), 'StartSel=<mark>, StopSel=</mark>') headline")
+               .limit(Constants::SEARCH_LIMIT)
+               .offset((page - 1) * Constants::SEARCH_LIMIT)
+    else
+      new_query
     end
-
-    Asset
-      .joins('LEFT JOIN asset_text_data ON ' \
-             ' assets.id = asset_text_data.asset_id')
-      .select('assets.*')
-      .select("ts_headline(data, to_tsquery('" +
-              sanitize_sql_for_conditions(s_query) +
-              "'), 'StartSel=<mark>, StopSel=</mark>') headline")
-      .where('assets.id IN (?)', ids)
   end
 
   def is_image?

app/views/search/index.html.erb

@@ -51,7 +51,8 @@
         >
           <a href="?<%= {category: 'projects', q: @search_query,
                          whole_word: @search_whole_word, whole_phrase: @search_whole_phrase,
-                         match_case: @search_case, utf8: '✓'}.to_query %>">
+                         match_case: @search_case, utf8: '✓',
+                         search_id: @search_id}.to_query %>">
             <span class="badge pull-right"><%= @project_search_count %></span>
             <span class="glyphicon glyphicon-blackboard"></span>
             <%= t'Projects' %>
@@ -64,7 +65,8 @@
         >
           <a href="?<%= {category: 'experiments', q: @search_query,
                          whole_word: @search_whole_word, whole_phrase: @search_whole_phrase,
-                         match_case: @search_case, utf8: '✓'}.to_query %>">
+                         match_case: @search_case, utf8: '✓',
+                         search_id: @search_id}.to_query %>">
             <span class="badge pull-right"><%= @experiment_search_count %></span>
             <%= fa_icon 'flask' %>
             <%= t'Experiments' %>
@@ -77,7 +79,8 @@
         >
           <a href="?<%= {category: 'modules', q: @search_query,
                          whole_word: @search_whole_word, whole_phrase: @search_whole_phrase,
-                         match_case: @search_case, utf8: '✓'}.to_query %>">
+                         match_case: @search_case, utf8: '✓',
+                         search_id: @search_id}.to_query %>">
             <span class="badge pull-right"><%= @module_search_count %></span>
             <span class="glyphicon glyphicon-credit-card"></span>
             <%= t'Modules' %>
@@ -90,7 +93,8 @@
         >
           <a href="?<%= {category: 'results', q: @search_query,
                          whole_word: @search_whole_word, whole_phrase: @search_whole_phrase,
-                         match_case: @search_case, utf8: '✓'}.to_query %>">
+                         match_case: @search_case, utf8: '✓',
+                         search_id: @search_id}.to_query %>">
             <span class="badge pull-right"><%= @result_search_count %></span>
             <span class="glyphicon glyphicon-modal-window"></span>
             <%= t'Results' %>
@@ -103,7 +107,8 @@
         >
           <a href="?<%= {category: 'tags', q: @search_query,
                          whole_word: @search_whole_word, whole_phrase: @search_whole_phrase,
-                         match_case: @search_case, utf8: '✓'}.to_query %>">
+                         match_case: @search_case, utf8: '✓',
+                         search_id: @search_id}.to_query %>">
             <span class="badge pull-right"><%= @tag_search_count %></span>
             <span class="glyphicon glyphicon-tags"></span>
             <%= t'Tags' %>
@@ -116,7 +121,8 @@
         >
           <a href="?<%= {category: 'reports', q: @search_query,
                          whole_word: @search_whole_word, whole_phrase: @search_whole_phrase,
-                         match_case: @search_case, utf8: '✓'}.to_query %>">
+                         match_case: @search_case, utf8: '✓',
+                         search_id: @search_id}.to_query %>">
             <span class="badge pull-right"><%= @report_search_count %></span>
             <span class="glyphicon glyphicon-align-left"></span>
             <%= t'Reports' %>
@@ -129,7 +135,8 @@
         >
           <a href="?<%= {category: 'protocols', q: @search_query,
                          whole_word: @search_whole_word, whole_phrase: @search_whole_phrase,
-                         match_case: @search_case, utf8: '✓'}.to_query %>">
+                         match_case: @search_case, utf8: '✓',
+                         search_id: @search_id}.to_query %>">
             <span class="badge pull-right"><%= @protocol_search_count %></span>
             <span class="glyphicon glyphicon-list-alt"></span>
             <%= t'Protocols' %>
@@ -142,7 +149,8 @@
         >
           <a href="?<%= {category: 'steps', q: @search_query,
                          whole_word: @search_whole_word, whole_phrase: @search_whole_phrase,
-                         match_case: @search_case, utf8: '✓'}.to_query %>">
+                         match_case: @search_case, utf8: '✓',
+                         search_id: @search_id}.to_query %>">
             <span class="badge pull-right"><%= @step_search_count %></span>
             <span class="glyphicon glyphicon-circle-arrow-right"></span>
             <%= t'Steps' %>
@@ -155,7 +163,8 @@
         >
           <a href="?<%= {category: 'checklists', q: @search_query,
                          whole_word: @search_whole_word, whole_phrase: @search_whole_phrase,
-                         match_case: @search_case, utf8: '✓'}.to_query %>">
+                         match_case: @search_case, utf8: '✓',
+                         search_id: @search_id}.to_query %>">
             <span class="badge pull-right"><%= @checklist_search_count %></span>
             <span class="glyphicon glyphicon-list"></span>
             <%= t'Checklists' %>
@@ -168,7 +177,8 @@
         >
           <a href="?<%= {category: 'samples', q: @search_query,
                          whole_word: @search_whole_word, whole_phrase: @search_whole_phrase,
-                         match_case: @search_case, utf8: '✓'}.to_query %>">
+                         match_case: @search_case, utf8: '✓',
+                         search_id: @search_id}.to_query %>">
             <span class="badge pull-right"><%= @sample_search_count %></span>
             <span class="glyphicon glyphicon-tint"></span>
             <%= t'Samples' %>
@@ -182,7 +192,8 @@
         >
           <a href="?<%= {category: 'assets', q: @search_query,
                          whole_word: @search_whole_word, whole_phrase: @search_whole_phrase,
-                         match_case: @search_case, utf8: '✓'}.to_query %>">
+                         match_case: @search_case, utf8: '✓',
+                         search_id: @search_id}.to_query %>">
             <span class="badge pull-right"><%= @asset_search_count %></span>
             <span class="glyphicon glyphicon-file"></span>
             <%= t'Assets' %>
@@ -195,7 +206,8 @@
         >
           <a href="?<%= {category: 'tables', q: @search_query,
                          whole_word: @search_whole_word, whole_phrase: @search_whole_phrase,
-                         match_case: @search_case, utf8: '✓'}.to_query %>">
+                         match_case: @search_case, utf8: '✓',
+                         search_id: @search_id}.to_query %>">
             <span class="badge pull-right"><%= @table_search_count %></span>
             <span class="glyphicon glyphicon-th"></span>
             <%= t'Tables' %>
@@ -208,7 +220,8 @@
         >
           <a href="?<%= {category: 'comments', q: @search_query,
                          whole_word: @search_whole_word, whole_phrase: @search_whole_phrase,
-                         match_case: @search_case, utf8: '✓'}.to_query %>">
+                         match_case: @search_case, utf8: '✓',
+                         search_id: @search_id}.to_query %>">
             <span class="badge pull-right"><%= @comment_search_count %></span>
             <span class="glyphicon glyphicon-comment"></span>
             <%= t'Comments' %>
@@ -229,7 +242,8 @@
               <a href="?<%= {category: 'repositories',
                              repository: values[:id], q: @search_query,
                              whole_word: @search_whole_word, whole_phrase: @search_whole_phrase,
-                             match_case: @search_case, utf8: '✓'}.to_query %>">
+                             match_case: @search_case, utf8: '✓',
+                             search_id: @search_id}.to_query %>">
                 <span class="badge pull-right"><%= values[:count] %></span>
                 <%= repository %>
               </a>

app/views/search/results/_assets.html.erb

@@ -12,13 +12,6 @@
     <%= render partial: "search/results/partials/asset_text.html.erb", locals: { asset: asset, query: search_query } %>
   </h5>
 
-  <!-- Display asset contents if it exists -->
-  <% if asset.headline.present? && !asset.headline.empty? && asset.headline.include?("<mark>") %>
-    <blockquote class="blockquote-search">
-      <p><%= highlight(sanitize_input(asset.headline), search_query.strip.split(/\s+/)) %></p>
-    </blockquote>
-  <% end %>
-
   <p>
     <span>
       <%=t "search.index.created_at" %>
@@ -93,6 +86,24 @@
         <%= render partial: "search/results/partials/team_text.html.erb",
                    locals: { team: asset.result.my_module.experiment.project.team } %>
       </span>
+    <% elsif asset.repository_asset_value %>
+      <span>
+        <%=t "search.index.repository_row" %>
+        <%= render partial: "search/results/partials/repository_row_text.html.erb",
+                   locals: { repository_row: asset.repository_asset_value.repository_cell.repository_row } %>
+      </span>
+      <br>
+      <span>
+        <%=t "search.index.repository" %>
+        <%= render partial: "search/results/partials/repository_text.html.erb",
+                   locals: { repository: asset.repository_asset_value.repository_cell.repository_row.repository } %>
+      </span>
+      <br>
+      <span>
+        <%=t "search.index.team" %>
+        <%= render partial: "search/results/partials/team_text.html.erb",
+                   locals: { team: asset.team } %>
+      </span>
     <% end %>
   </p>
 

app/views/search/results/partials/_asset_text.html.erb

@@ -1,15 +1,43 @@
 <% query ||= nil %>
+<% asset_read_allowed = false %>
 <% text = query.present? ? highlight(asset.file_file_name, query.strip.split(/\s+/)) : asset.file_file_name %>
 
 <% if asset.step %>
   <% protocol = asset.step.protocol %>
   <% if can_read_protocol_in_module?(protocol) ||
-        can_read_protocol_in_repository?(protocol) ||
-        (asset.result && can_read_experiment?(protocol.my_module.experiment)) %>
+        can_read_protocol_in_repository?(protocol) %>
+    <% asset_read_allowed = true %>
     <a href="<%= download_asset_path asset %>" target="_blank">
       <%= text %>
     </a>
+  <% else %>
+    <%= text %>
+  <% end %>
+<% elsif asset.result %>
+  <% if can_read_experiment?(asset.result.my_module.experiment) %>
+    <% asset_read_allowed = true %>
+    <a href="<%= download_asset_path asset %>" target="_blank">
+      <%= text %>
+    </a>
+  <% else %>
+    <%= text %>
+  <% end %>
+<% elsif asset.repository_asset_value %>
+  <% if can_read_team?(asset.repository_asset_value.repository_cell.repository_row.repository.team) %>
+    <% asset_read_allowed = true %>
+    <a href="<%= download_asset_path asset %>" target="_blank">
+      <%= text %>
+    </a>
+  <% else %>
+    <%= text %>
   <% end %>
 <% else %>
   <%= text %>
 <% end %>
+
+<!-- Display asset contents if it exists -->
+<% if asset_read_allowed && asset.headline.present? && asset.headline.include?("<mark>") %>
+  <blockquote class="blockquote-search">
+    <p><%= highlight(sanitize_input(asset.headline), query.strip.split(/\s+/)) %></p>
+  </blockquote>
+<% end %>

app/views/search/results/partials/_repository_row_text.html.erb

@@ -0,0 +1,7 @@
+<% if can_read_team?(repository_row.repository.team) %>
+  <%= route_to_other_team repository_path(id: repository_row.repository.id),
+                          repository_row.repository.team,
+                          repository_row.name %>
+<% else %>
+  <%= repository_row.name %>
+<% end %>

app/views/search/results/partials/_repository_text.html.erb

@@ -0,0 +1,7 @@
+<% if can_read_team?(repository.team) %>
+  <%= route_to_other_team repository_path(id: repository.id),
+                          repository.team,
+                          repository.name %>
+<% else %>
+  <%= repository.name %>
+<% end %>

config/initializers/extends.rb

@@ -52,11 +52,13 @@ class Extends
   # Extra attributes used for search in repositories, text columns
   # are only supported
   REPOSITORY_EXTRA_SEARCH_ATTR = ['repository_text_values.data',
-                                  'repository_list_items.data']
+                                  'repository_list_items.data',
+                                  'assets.file_file_name']
 
   # Array of includes used in search query for repository rows
   REPOSITORY_SEARCH_INCLUDES = [:repository_text_value,
-                                repository_list_value: :repository_list_item]
+                                repository_list_value: :repository_list_item,
+                                repository_asset_value: :asset]
 
   # List of implemented core API versions
   API_VERSIONS = ['20170715']

config/locales/en.yml

@@ -152,6 +152,8 @@ en:
       description: "Description: "
       no_description: "No description"
       team: "Team: "
+      repository: "Repository: "
+      repository_row: "Repository item: "
       project: "Project: "
       experiment: "Experiment: "
       protocol: "Protocol: "
@@ -211,7 +213,6 @@ en:
         one: "1 day"
         other: "%{count} days"
       module_one_day_due_html: "Task <em>%{module}</em> is due in less than 1 day."
-      new_comment: "New comment"
       user_role: "Role: "
       user_full_name: "User: "
       edit_user: "Edit role"