ES: search through the wiki blobs

Gemfile

@@ -97,7 +97,7 @@ gem "seed-fu", '~> 2.3.5'
 # Search
 gem 'elasticsearch-model'
 gem 'elasticsearch-rails'
-gem 'gitlab-elasticsearch-git', path: '../elasticsearch-git', require: "elasticsearch/git"
+gem 'gitlab-elasticsearch-git', require: "elasticsearch/git"
 # Markdown and HTML processing
 gem 'html-pipeline', '~> 1.11.0'

Gemfile.lock

-PATH
-  remote: ../elasticsearch-git
-  specs:
-    gitlab-elasticsearch-git (0.0.6)
-      activemodel (~> 4.2.0)
-      activesupport (~> 4.2.0)
-      charlock_holmes (~> 0.7.3)
-      elasticsearch-api (> 0.4.0)
-      elasticsearch-model
-      github-linguist (> 4.7.0)
-      rugged (~> 0.23.3)
-
 GEM
   remote: https://rubygems.org/
   specs:
@@ -308,12 +296,20 @@ GEM
       json
     get_process_mem (0.2.0)
     gherkin-ruby (0.3.2)
-    github-linguist (4.7.2)
+    github-linguist (4.7.4)
       charlock_holmes (~> 0.7.3)
       escape_utils (~> 1.1.0)
       mime-types (>= 1.19)
       rugged (>= 0.23.0b)
     github-markup (1.3.3)
+    gitlab-elasticsearch-git (0.0.7)
+      activemodel (~> 4.2.0)
+      activesupport (~> 4.2.0)
+      charlock_holmes (~> 0.7.3)
+      elasticsearch-api (~> 1.0.15)
+      elasticsearch-model
+      github-linguist (~> 4.7.0)
+      rugged (~> 0.23.3)
     gitlab-flowdock-git-hook (1.0.1)
       flowdock (~> 0.7)
       gitlab-grit (>= 2.4.1)
@@ -899,7 +895,7 @@ DEPENDENCIES
   gemnasium-gitlab-service (~> 0.2)
   github-linguist (~> 4.7.0)
   github-markup (~> 1.3.1)
-  gitlab-elasticsearch-git!
+  gitlab-elasticsearch-git
   gitlab-flowdock-git-hook (~> 1.0.1)
   gitlab-license (~> 0.0.4)
   gitlab_emoji (~> 0.2.0)

app/elastic/repositories_search.rb

@@ -23,15 +23,8 @@ def self.import
       Project.find_each do |project|
         if project.repository.exists? && !project.repository.empty?
-          begin
-            project.repository.index_commits
-          rescue
-          end
-
-          begin
-            project.repository.index_blobs
-          rescue
-          end
+          project.repository.index_commits
+          project.repository.index_blobs
         end
       end
     end

app/elastic/wiki_repositories_search.rb

+module WikiRepositoriesSearch
+  extend ActiveSupport::Concern
+
+  included do
+    include Elasticsearch::Git::Repository
+
+    self.__elasticsearch__.client = Elasticsearch::Client.new host: Gitlab.config.elasticsearch.host, port: Gitlab.config.elasticsearch.port
+
+    def repository_id
+      "wiki_#{project.id}"
+    end
+
+    def self.repositories_count
+      Project.where(wiki_enabled: true).count
+    end
+
+    def client_for_indexing
+      self.__elasticsearch__.client
+    end
+
+    def self.import
+      ProjectWiki.__elasticsearch__.create_index! force: true
+
+      Project.where(wiki_enabled: true).find_each do |project|
+        unless project.wiki.empty?
+          project.wiki.index_blobs
+        end
+      end
+    end
+  end
+end

app/models/project.rb

@@ -927,6 +927,10 @@ def create_wiki
     false
   end
+  def wiki
+    ProjectWiki.new(self, self.owner)
+  end
+
   def reference_issue_tracker?
     default_issues_tracker? || jira_tracker_active?
   end

app/models/project_wiki.rb

 class ProjectWiki
   include Gitlab::ShellAdapter
+  include WikiRepositoriesSearch
   MARKUPS = {
     'Markdown' => :md,
@@ -12,6 +13,8 @@ class CouldNotCreateWikiError < StandardError; end
   # Returns a string describing what went wrong after
   # an operation fails.
   attr_reader :error_message
+  attr_reader :project
+
   def initialize(project, user = nil)
     @project = project

app/views/search/results/_wiki_blob.html.haml

-- wiki_blob = @project.repository.parse_search_result(wiki_blob)
+- wiki_blob = @project.repository.parse_search_result(wiki_blob, true)
 .blob-result
   .file-holder
     .file-title

lib/gitlab/elastic/project_search_results.rb

@@ -21,7 +21,7 @@ def objects(scope, page = nil)
         when 'blobs'
           blobs.response.page(page).per(per_page)
         when 'wiki_blobs'
-          Kaminari.paginate_array(wiki_blobs).page(page).per(per_page)
+          wiki_blobs.response.page(page).per(per_page)
         when 'commits'
           Kaminari.paginate_array(commits).page(page).per(per_page)
         else
@@ -54,25 +54,18 @@ def commits_count
       def blobs
         if project.empty_repo? || query.blank?
-          []
+          Kaminari.paginate_array([])
         else
           project.repository.search(query, type: :blob, options: {highlight: true})[:blobs][:results]
         end
       end
       def wiki_blobs
-        Kaminari.paginate_array([])
-        # if project.wiki_enabled? && query.present?
-        #   project_wiki = ProjectWiki.new(project)
-
-        #   unless project_wiki.empty?
-        #     project_wiki.search_files(query)
-        #   else
-        #     []
-        #   end
-        # else
-        #   []
-        # end
+        if project.wiki_enabled? && !project.wiki.empty? && query.present?
+          project.wiki.search(query, type: :blob, options: {highlight: true})[:blobs][:results]
+        else
+          Kaminari.paginate_array([])
+        end
       end
       def notes
@@ -96,4 +89,4 @@ def limit_project_ids
       end
     end
   end
-end
+end
\ No newline at end of file

lib/tasks/gitlab/elastic.rake

@@ -5,6 +5,11 @@ namespace :gitlab do
       Repository.import
     end
+    desc "Indexing all wikis"
+    task index_wiki: :environment  do
+      ProjectWiki.import
+    end
+
     desc "Create indexes in the Elasticsearch from database records"
     task create_index: :environment do
       [Project, User, Issue, MergeRequest, Snippet, Note, Milestone].each do |klass|