elastic: almost all integrated exept view logic

Gemfile

@@ -94,6 +94,10 @@ gem "six", '~> 0.2.0'
 # Seed data
 gem "seed-fu", '~> 2.3.5'
+# Search
+gem 'elasticsearch-model',  github: 'elasticsearch/elasticsearch-rails',  ref: '88b6597e47c9f45024b603faeddb0a85b47e1fce'
+gem 'elasticsearch-rails',  github: 'elasticsearch/elasticsearch-rails'
+
 # Markdown and HTML processing
 gem 'html-pipeline', '~> 1.11.0'
 gem 'task_list',     '~> 1.0.2', require: 'task_list/railtie'

Gemfile.lock

+GIT
+  remote: git://github.com/elasticsearch/elasticsearch-rails.git
+  revision: 88b6597e47c9f45024b603faeddb0a85b47e1fce
+  ref: 88b6597e47c9f45024b603faeddb0a85b47e1fce
+  specs:
+    elasticsearch-model (0.1.0.rc1)
+      activesupport (> 3)
+      elasticsearch (> 0.4)
+      hashie
+
+GIT
+  remote: git://github.com/elasticsearch/elasticsearch-rails.git
+  revision: 5f32e484a6d0458f26dc9acfa9dd5fed4e5d6453
+  specs:
+    elasticsearch-rails (0.1.0.rc1)
+
 GEM
   remote: https://rubygems.org/
   specs:
@@ -177,6 +193,14 @@ GEM
       railties (>= 3.2)
     dropzonejs-rails (0.7.2)
       rails (> 3.1)
+    elasticsearch (1.0.15)
+      elasticsearch-api (= 1.0.15)
+      elasticsearch-transport (= 1.0.15)
+    elasticsearch-api (1.0.15)
+      multi_json
+    elasticsearch-transport (1.0.15)
+      faraday
+      multi_json
     email_reply_parser (0.5.8)
     email_spec (1.6.0)
       launchy (~> 2.1)
@@ -858,6 +882,8 @@ DEPENDENCIES
   diffy (~> 3.0.3)
   doorkeeper (~> 2.2.0)
   dropzonejs-rails (~> 0.7.1)
+  elasticsearch-model!
+  elasticsearch-rails!
   email_reply_parser (~> 0.5.8)
   email_spec (~> 1.6.0)
   enumerize (~> 0.7.0)

app/controllers/search_controller.rb

@@ -48,9 +48,9 @@ class SearchController < ApplicationController
     if project
       return access_denied! unless can?(current_user, :download_code, project)
-      @search_results = SearchService.new(current_user, params).project_search(project)
+      @search_results = SearchService.new(nil, current_user, params).project_search(project)
     else
-      @search_results = SearchService.new(current_user, params).global_search
+      @search_results = SearchService.new(nil, current_user, params).global_search
     end
     @search_results = SearchDecorator.new(@search_results, params[:type])

app/elastic/application_search.rb

@@ -46,10 +46,10 @@ module ApplicationSearch
       }
     }
-    after_commit lambda { Resque.enqueue(Elastic::BaseIndexer, :index,  self.class.to_s, self.id) }, on: :create
-    after_commit lambda { Resque.enqueue(Elastic::BaseIndexer, :update, self.class.to_s, self.id) }, on: :update
-    after_commit lambda { Resque.enqueue(Elastic::BaseIndexer, :delete, self.class.to_s, self.id) }, on: :destroy
-    after_touch  lambda { Resque.enqueue(Elastic::BaseIndexer, :update, self.class.to_s, self.id) }
+    # after_commit lambda { Resque.enqueue(Elastic::BaseIndexer, :index,  self.class.to_s, self.id) }, on: :create
+    # after_commit lambda { Resque.enqueue(Elastic::BaseIndexer, :update, self.class.to_s, self.id) }, on: :update
+    # after_commit lambda { Resque.enqueue(Elastic::BaseIndexer, :delete, self.class.to_s, self.id) }, on: :destroy
+    # after_touch  lambda { Resque.enqueue(Elastic::BaseIndexer, :update, self.class.to_s, self.id) }
   end
    module ClassMethods

app/elastic/groups_search.rb

@@ -24,7 +24,7 @@ module GroupsSearch
       })
     end
-    def self.search(query, page: 1, per: 20, options: {})
+    def self.elastic_search(query, page: 1, per: 20, options: {})
       page ||= 1

app/elastic/issues_search.rb

@@ -41,7 +41,7 @@ module IssuesSearch
       })
     end
-    def self.search(query, page: 1, per: 20, options: {})
+    def self.elastic_search(query, page: 1, per: 20, options: {})
       page ||= 1

app/elastic/merge_requests_search.rb

@@ -47,7 +47,7 @@ module MergeRequestsSearch
       })
     end
-    def self.search(query, page: 1, per: 20, options: {})
+    def self.elastic_search(query, page: 1, per: 20, options: {})
       page ||= 1

app/elastic/projects_search.rb

@@ -54,7 +54,7 @@ module ProjectsSearch
       })
     end
-    def self.search(query, page: 1, per: 20, options: {})
+    def self.elastic_search(query, page: 1, per: 20, options: {})
       page ||= 1

app/elastic/repositories_search.rb

-module RepositoriesSearch
-  extend ActiveSupport::Concern
-
-  included do
-    include Elasticsearch::Git::Repository
-
-    self.__elasticsearch__.client = Elasticsearch::Client.new host: Gitlab.config.elasticsearch.host, port: Gitlab.config.elasticsearch.port
-
-    def repository_id
-      project.id
-    end
-
-    def self.repositories_count
-      Project.count
-    end
-
-    def client_for_indexing
-      self.__elasticsearch__.client
-    end
-
-    def self.import
-      Repository.__elasticsearch__.create_index! force: true
-
-      Project.find_each do |project|
-        if project.repository.exists? && !project.repository.empty?
-          begin
-            project.repository.index_commits
-          rescue
-          end
-          begin
-            project.repository.index_blobs
-          rescue
-          end
-        end
-      end
-    end
-  end
-end

app/elastic/users_search.rb

@@ -31,7 +31,7 @@ module UsersSearch
       })
     end
-    def self.search(query, page: 1, per: 20, options: {})
+    def self.elastic_search(query, page: 1, per: 20, options: {})
       page ||= 1
       per ||= 20

app/models/repository.rb

 require 'securerandom'
 class Repository
-  include RepositoriesSearch
-
   class CommitError < StandardError; end
   MIRROR_REMOTE = "upstream"

app/services/search_service.rb

@@ -7,20 +7,19 @@ class SearchService < BaseService
         users: search_in_users(query),
         projects: search_in_projects(query),
         merge_requests: search_in_merge_requests(query),
-        issues: search_in_issues(query),
-        repositories: search_in_repository(query),
+        issues: search_in_issues(query)
     }
   end
   def project_search(project)
     query = params[:search]
+
     {
         groups: {},
         users: {},
         projects: {},
         merge_requests: search_in_merge_requests(query, project),
-        issues: search_in_issues(query, project),
-        repositories: search_in_repository(query, project),
+        issues: search_in_issues(query, project)
     }
   end
@@ -36,12 +35,13 @@ class SearchService < BaseService
     }
     group = Group.find_by(id: params[:group_id]) if params[:group_id].present?
+
     opt[:namespace_id] = group.id if group
     opt[:category] = params[:category] if params[:category].present?
     begin
-      response = Project.search(query, options: opt, page: page)
+      response = Project.elastic_search(query, options: opt, page: page)
       categories_list = if query.blank?
                           Project.category_counts.map do |category|
@@ -75,7 +75,7 @@ class SearchService < BaseService
     }
     begin
-      response = Group.search(query, options: opt, page: page)
+      response = Group.elastic_search(query, options: opt, page: page)
       {
         records: response.records,
@@ -96,7 +96,7 @@ class SearchService < BaseService
     }
     begin
-      response = User.search(query, options: opt, page: page)
+      response = User.elastic_search(query, options: opt, page: page)
       {
         records: response.records,
@@ -117,7 +117,7 @@ class SearchService < BaseService
     }
     begin
-      response = MergeRequest.search(query, options: opt, page: page)
+      response = MergeRequest.elastic_search(query, options: opt, page: page)
       {
         records: response.records,
@@ -137,7 +137,7 @@ class SearchService < BaseService
     }
     begin
-      response = Issue.search(query, options: opt, page: page)
+      response = Issue.elastic_search(query, options: opt, page: page)
       {
         records: response.records,
@@ -150,33 +150,11 @@ class SearchService < BaseService
     end
   end
-  def search_in_repository(query, project = nil)
-    opt = {
-      repository_id: project ? [project.id] : projects_ids,
-      highlight: true,
-      order: params[:order]
-    }
-
-    if params[:language].present? && params[:language] != 'All'
-      opt.merge!({ language: params[:language] })
-    end
-
-    begin
-      res = Repository.search(query, options: opt, page: page)
-
-      res[:blobs][:projects] = project_filter(res[:blobs][:repositories]) || []
-      res[:commits][:projects]  = project_filter(res[:commits][:repositories]) || []
-
-      res
-    rescue Exception => e
-      {}
-    end
-  end
-
   def projects_ids
     @allowed_projects_ids ||= begin
       if params[:namespace].present?
         namespace = Namespace.find_by(path: params[:namespace])
+
         if namespace
           return namespace.projects.where(id: known_projects_ids).pluck(:id)
         end
@@ -196,8 +174,8 @@ class SearchService < BaseService
   def known_projects_ids
     known_projects_ids = []
-    known_projects_ids += current_user.known_projects.pluck(:id) if current_user
-    known_projects_ids + Project.public_or_internal_only(current_user).pluck(:id)
+    known_projects_ids += current_user.authorized_projects.pluck(:id) if current_user
+    known_projects_ids + Project.public_and_internal_only.pluck(:id)
   end
   def project_filter(es_results)

config/gitlab.yml.example

@@ -136,6 +136,11 @@ production: &base
     # The location where LFS objects are stored (default: shared/lfs-objects).
     # storage_path: shared/lfs-objects
+  ## Elasticsearch (EE only)
+  elasticsearch:
+    host: localhost
+    port: 9200
+
   ## GitLab Pages
   pages:
     enabled: false