Merge branch 'feature/geo-reposync-systemhook' into 'master'

Geo: repository sync changed to use push event from system hook

Changed from our own custom implementation, to use GitLab's system hook push event to trigger repository synchronization for Geo (#76). 

See merge request !347

app/models/geo_node.rb

@@ -118,5 +118,7 @@ def update_system_hook!
     self.build_system_hook if system_hook.nil?
     self.system_hook.token = SecureRandom.hex(20) unless self.system_hook.token.present?
     self.system_hook.url = geo_events_url if uri.present?
+    self.system_hook.push_events = true
+    self.system_hook.tag_push_events = true
   end
 end

app/services/geo/enqueue_project_update_service.rb

-module Geo
-  class EnqueueProjectUpdateService
-    attr_reader :project
-
-    def initialize(project)
-      @queue = Gitlab::Geo::UpdateQueue.new('updated_projects')
-      @project = project
-    end
-
-    def execute
-      @queue.store({ id: @project.id, clone_url: @project.url_to_repo })
-    end
-  end
-end

app/services/geo/notify_nodes_service.rb

@@ -2,12 +2,10 @@ module Geo
   class NotifyNodesService < BaseNotify
     def initialize
-      @proj_queue = Gitlab::Geo::UpdateQueue.new('updated_projects')
       @wiki_queue = Gitlab::Geo::UpdateQueue.new('updated_wikis')
     end
     def execute
-      process(@proj_queue, :notify_projects_url)
       process(@wiki_queue, :notify_wikis_url)
     end

app/services/geo/schedule_repo_update_service.rb

 module Geo
   class ScheduleRepoUpdateService
-    attr_reader :projects
-    def initialize(projects)
-      @projects = projects
+    attr_reader :id, :clone_url
+    def initialize(params)
+      @id = params[:project_id]
+      @clone_url = params[:project][:git_ssh_url]
     end
     def execute
-      @projects.each do |project|
-        GeoRepositoryUpdateWorker.perform_async(project['id'], project['clone_url'])
-      end
+      GeoRepositoryUpdateWorker.perform_async(@id, @clone_url)
     end
   end
 end

app/workers/post_receive.rb

@@ -23,9 +23,6 @@ def perform(repo_path, identifier, changes)
       # Triggers repository update on secondary nodes when Geo is enabled
       Gitlab::Geo.notify_wiki_update(post_received.project) if Gitlab::Geo.enabled?
     elsif post_received.regular_project?
-      # Triggers repository update on secondary nodes when Geo is enabled
-      Gitlab::Geo.notify_project_update(post_received.project) if Gitlab::Geo.enabled?
-
       process_project_changes(post_received)
     else
       log("Triggered hook for unidentifiable repository type with full path \"#{repo_path} \"")

lib/api/geo.rb

 module API
   class Geo < Grape::API
     resource :geo do
-      # Enqueue a batch of IDs of modified projects to have their
-      # repositories updated
-      #
-      # Example request:
-      #   POST /geo/refresh_projects
-      post 'refresh_projects' do
-        authenticated_as_admin!
-        required_attributes! [:projects]
-        ::Geo::ScheduleRepoUpdateService.new(params[:projects]).execute
-      end
-
       # Enqueue a batch of IDs of wiki's projects to have their
       # wiki repositories updated
       #
@@ -35,6 +24,12 @@ class Geo < Grape::API
         when 'key_create', 'key_destroy'
           required_attributes! %w(key id)
           ::Geo::ScheduleKeyChangeService.new(params).execute
+        when 'push'
+          required_attributes! %w(event_name project_id project)
+          ::Geo::ScheduleRepoUpdateService.new(params).execute
+        when 'tag_push'
+          required_attributes! %w(event_name project_id project)
+          ::Geo::ScheduleWikiRepoUpdateService.new(params).execute
         end
       end
     end

lib/gitlab/geo.rb

@@ -34,10 +34,6 @@ def self.geo_node?(host:, port:)
       GeoNode.where(host: host, port: port).exists?
     end
-    def self.notify_project_update(project)
-      ::Geo::EnqueueProjectUpdateService.new(project).execute
-    end
-
     def self.notify_wiki_update(project)
       ::Geo::EnqueueWikiUpdateService.new(project).execute
     end

lib/gitlab/middleware/readonly_geo.rb

@@ -2,7 +2,7 @@ module Gitlab
   module Middleware
     class ReadonlyGeo
       DISALLOWED_METHODS = %w(POST PATCH PUT DELETE)
-      WHITELISTED = %w(api/v3/internal api/v3/geo/refresh_projects api/v3/geo/refresh_wikis api/v3/geo/refresh_key api/v3/geo/receive_events)
+      WHITELISTED = %w(api/v3/internal api/v3/geo/refresh_wikis api/v3/geo/receive_events)
       APPLICATION_JSON = 'application/json'
       def initialize(app)

spec/lib/gitlab/geo_spec.rb

@@ -68,15 +68,4 @@
       expect(described_class.geo_node?(host: 'inexistent', port: 1234)).to be_falsey
     end
   end
-
-  describe 'notify_project_update' do
-    let(:project) { FactoryGirl.build(:project) }
-
-    it 'delegates to NotifyService' do
-      expect(Geo::EnqueueProjectUpdateService).to receive(:new).with(project).and_call_original
-      expect_any_instance_of(Geo::EnqueueProjectUpdateService).to receive(:execute)
-
-      described_class.notify_project_update(project)
-    end
-  end
 end

spec/models/geo_node_spec.rb

@@ -87,6 +87,8 @@
         expect(node.system_hook.url).to be_present
         expect(node.system_hook.url).to eq(node.geo_events_url)
         expect(node.system_hook.token).to be_present
+        expect(node.system_hook.push_events).to be_truthy
+        expect(node.system_hook.tag_push_events).to be_truthy
       end
     end
   end

spec/requests/api/geo_spec.rb

@@ -5,30 +5,28 @@
   let(:admin) { create(:admin) }
   let(:user) { create(:user) }
   let(:geo_node) { build(:geo_node) }
-  describe 'POST /geo/refresh_projects' do
-    before(:each) { allow_any_instance_of(::Geo::ScheduleRepoUpdateService).to receive(:execute) }
-    it 'starts refresh process if admin and correct params' do
-      post api('/geo/refresh_projects', admin), projects: ['1', '2', '3']
-      expect(response.status).to eq 201
+  let(:geo_token_header) do
+    { 'X-Gitlab-Token' => geo_node.system_hook.token }
+  end
+  before(:each) do
+    allow(Gitlab::Geo).to receive(:current_node) { geo_node }
+  end
+  describe 'POST /geo/receive_events authentication' do
+    it 'denies access if token is not present' do
+      post api('/geo/receive_events')
+      expect(response.status).to eq 401
     end
-    it 'denies access if not admin' do
-      post api('/geo/refresh_projects', user)
-      expect(response.status).to eq 403
+    it 'denies access if token is invalid' do
+      post api('/geo/receive_events'), nil, { 'X-Gitlab-Token' => 'nothing' }
+      expect(response.status).to eq 401
     end
   end
-  describe 'POST /geo/receive_events' do
-    before(:each) do
-      allow_any_instance_of(::Geo::ScheduleKeyChangeService).to receive(:execute)
-      allow(Gitlab::Geo).to receive(:current_node) { geo_node }
-    end
-
-    let(:geo_token_header) do
-      { 'X-Gitlab-Token' => geo_node.system_hook.token }
-    end
+  describe 'POST /geo/receive_events key events' do
+    before(:each) { allow_any_instance_of(::Geo::ScheduleKeyChangeService).to receive(:execute) }
     let(:key_create_payload) do
       {
@@ -61,15 +59,43 @@
       post api('/geo/receive_events'), key_destroy_payload, geo_token_header
       expect(response.status).to eq 201
     end
-    it 'denies access if token is not present' do
-      post api('/geo/receive_events')
-      expect(response.status).to eq 401
+  end
+  describe 'POST /geo/receive_events push events' do
+    before(:each) { allow_any_instance_of(::Geo::ScheduleRepoUpdateService).to receive(:execute) }
+
+    let(:push_payload) do
+      {
+        'event_name' => 'push',
+        'project_id' => 1,
+        'project' => {
+          'git_ssh_url' => 'git@example.com:mike/diaspora.git'
+        }
+      }
     end
-    it 'denies access if token is invalid' do
-      post api('/geo/receive_events'), nil, { 'X-Gitlab-Token' => 'nothing' }
-      expect(response.status).to eq 401
+    it 'starts refresh process if admin and correct params' do
+      post api('/geo/receive_events'), push_payload, geo_token_header
+      expect(response.status).to eq 201
+    end
+  end
+
+  describe 'POST /geo/receive_events push_tag events' do
+    before(:each) { allow_any_instance_of(::Geo::ScheduleWikiRepoUpdateService).to receive(:execute) }
+
+    let(:tag_push_payload) do
+      {
+        'event_name' => 'tag_push',
+        'project_id' => 1,
+        'project' => {
+          'git_ssh_url' => 'git@example.com:mike/diaspora.git'
+        }
+      }
+    end
+
+    it 'starts refresh process if admin and correct params' do
+      post api('/geo/receive_events'), tag_push_payload, geo_token_header
+      expect(response.status).to eq 201
     end
   end
 end

spec/services/geo/enqueue_project_update_service_spec.rb

-describe Geo::EnqueueProjectUpdateService, services: true do
-  subject { Geo::EnqueueProjectUpdateService.new(project) }
-  let(:project) { double(:project) }
-  let(:fake_url) { 'git@localhost:repo/path.git' }
-  let(:fake_id) { 999 }
-  let(:queue) { subject.instance_variable_get(:@queue) }
-
-  before(:each) do
-    queue.empty!
-    expect(project).to receive(:url_to_repo) { fake_url }
-    expect(project).to receive(:id) { fake_id }
-  end
-
-  describe '#execute' do
-    let(:stored_data) { queue.first }
-    before(:each) { subject.execute }
-
-    it 'persists id and clone_url to redis queue' do
-      expect(stored_data).to have_key('id')
-      expect(stored_data).to have_key('clone_url')
-    end
-
-    it 'persisted id is equal to original' do
-      expect(stored_data['id']).to eq(fake_id)
-    end
-
-    it 'persisted clone_url is equal to original' do
-      expect(stored_data['clone_url']).to eq(fake_url)
-    end
-  end
-end