Add Product Analytics collector

Rack application within Rails to record product analytics events.

Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>

config/initializers/rack_attack.rb

@@ -68,6 +68,15 @@ class Rack::Attack
     end
   end
+  # Product analytics feature is in experimental stage.
+  # At this point we want to limit amount of events registered
+  # per application (aid stands for application id).
+  throttle('throttle_pa_collector', limit: 100, period: 60) do |req|
+    if req.pa_collector_request?
+      req.params['aid']
+    end
+  end
+
   throttle('throttle_authenticated_web', Gitlab::Throttle.authenticated_web_options) do |req|
     if req.web_request? &&
         Gitlab::Throttle.settings.throttle_authenticated_web_enabled
@@ -128,6 +137,10 @@ class Rack::Attack
       path =~ %r{^/-/(health|liveness|readiness)}
     end
+    def pa_collector_request?
+      path.start_with?('/-/collector/i')
+    end
+
     def should_be_skipped?
       api_internal_request? || health_check_request?
     end

config/routes.rb

 require 'sidekiq/web'
 require 'sidekiq/cron/web'
+require 'product_analytics/collector_app'
 Rails.application.routes.draw do
   concern :access_requestable do
@@ -174,6 +175,9 @@ Rails.application.routes.draw do
     # Used by third parties to verify CI_JOB_JWT, placeholder route
     # in case we decide to move away from doorkeeper-openid_connect
     get 'jwks' => 'doorkeeper/openid_connect/discovery#keys'
+
+    # Product analytics collector
+    match '/collector/i', to: ProductAnalytics::CollectorApp.new, via: :all
   end
   # End of the /-/ scope.

lib/product_analytics/collector_app.rb

+# frozen_string_literal: true
+
+module ProductAnalytics
+  class CollectorApp
+    def call(env)
+      request = Rack::Request.new(env)
+      params = request.params
+
+      return not_found unless EventParams.has_required_params?(params)
+
+      event_params = EventParams.parse_event_params(params)
+
+      if ProductAnalyticsEvent.create(event_params)
+        ok
+      else
+        not_found
+      end
+    end
+
+    def ok
+      [200, {}, ['OK']]
+    end
+
+    def not_found
+      [404, {}, ['']]
+    end
+  end
+end

lib/product_analytics/event_params.rb

+# frozen_string_literal: true
+
+module ProductAnalytics
+  # Converts params from Snowplow tracker to one
+  # compatible with GitLab ProductAnalyticsEvent model
+  class EventParams
+    def self.parse_event_params(params)
+      {
+        project_id:               params['aid'],
+        platform:                 params['p'],
+        collector_tstamp:         Time.zone.now,
+        event_id:                 params['eid'],
+        v_tracker:                params['tv'],
+        v_collector:              Gitlab::VERSION,
+        v_etl:                    Gitlab::VERSION,
+        os_timezone:              params['tz'],
+        name_tracker:             params['tna'],
+        br_lang:                  params['lang'],
+        doc_charset:              params['cs'],
+        br_features_pdf:          Gitlab::Utils.to_boolean(params['f_pdf']),
+        br_features_flash:        Gitlab::Utils.to_boolean(params['f_fla']),
+        br_features_java:         Gitlab::Utils.to_boolean(params['f_java']),
+        br_features_director:     Gitlab::Utils.to_boolean(params['f_dir']),
+        br_features_quicktime:    Gitlab::Utils.to_boolean(params['f_qt']),
+        br_features_realplayer:   Gitlab::Utils.to_boolean(params['f_realp']),
+        br_features_windowsmedia: Gitlab::Utils.to_boolean(params['f_wma']),
+        br_features_gears:        Gitlab::Utils.to_boolean(params['f_gears']),
+        br_features_silverlight:  Gitlab::Utils.to_boolean(params['f_ag']),
+        br_colordepth:            params['cd'],
+        br_cookies:               Gitlab::Utils.to_boolean(params['cookie']),
+        dvce_created_tstamp:      params['dtm'],
+        br_viewheight:            params['vp'],
+        domain_sessionidx:        params['vid'],
+        domain_sessionid:         params['sid'],
+        domain_userid:            params['duid'],
+        user_fingerprint:         params['fp'],
+        page_referrer:            params['refr'],
+        page_url:                 params['url']
+      }
+    end
+
+    def self.has_required_params?(params)
+      params['aid'].present? && params['eid'].present?
+    end
+  end
+end

spec/fixtures/product_analytics/event.json

+{
+ "aid":1,
+ "p":"web",
+ "tna":"sp",
+ "tv":"js-2.14.0",
+ "eid":"fbf14096-74ee-47e4-883c-8a0d6cb72e37",
+ "duid":"79543c31-cfc3-4479-a737-fafb9333c8ba",
+ "sid":"54f6d3f3-f4f9-4fdc-87e0-a2c775234c1b",
+ "vid":4,
+ "url":"http://example.com/products/1",
+ "refr":"http://example.com/products/1",
+ "lang":"en-US",
+ "cookie":true,
+ "tz":"America/Los_Angeles",
+ "cs":"UTF-8"
+}
+

spec/lib/product_analytics/event_params_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe ProductAnalytics::EventParams do
+  describe '.parse_event_params' do
+    subject { described_class.parse_event_params(raw_event) }
+
+    let(:raw_event) { Gitlab::Json.parse(fixture_file('product_analytics/event.json')) }
+
+    it 'extracts all params from raw event' do
+      expected_params = {
+        project_id: 1,
+        platform: 'web',
+        name_tracker: 'sp',
+        v_tracker: 'js-2.14.0',
+        event_id: 'fbf14096-74ee-47e4-883c-8a0d6cb72e37',
+        domain_userid: '79543c31-cfc3-4479-a737-fafb9333c8ba',
+        domain_sessionid: '54f6d3f3-f4f9-4fdc-87e0-a2c775234c1b',
+        domain_sessionidx: 4,
+        page_url: 'http://example.com/products/1',
+        page_referrer: 'http://example.com/products/1',
+        br_lang: 'en-US',
+        br_cookies: true,
+        os_timezone: 'America/Los_Angeles',
+        doc_charset: 'UTF-8'
+      }
+
+      expect(subject).to include(expected_params)
+    end
+  end
+
+  describe '.has_required_params?' do
+    subject { described_class.has_required_params?(params) }
+
+    context 'aid and eid are present' do
+      let(:params) { { 'aid' => 1, 'eid' => 2 } }
+
+      it { expect(subject).to be_truthy }
+    end
+
+    context 'aid and eid are missing' do
+      let(:params) { { } }
+
+      it { expect(subject).to be_falsey }
+    end
+
+    context 'eid is missing' do
+      let(:params) { { 'aid' => 1 } }
+
+      it { expect(subject).to be_falsey }
+    end
+  end
+end

spec/requests/product_analytics/collector_app_attack_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe 'ProductAnalytics::CollectorApp throttle' do
+  include RackAttackSpecHelpers
+
+  let(:request_limit) { 100 }
+
+  include_context 'rack attack cache store'
+
+  before do
+    allow(ProductAnalyticsEvent).to receive(:create).and_return(true)
+  end
+
+  context 'per ip address' do
+    let(:params) do
+      {
+        aid: rand(99),
+        eid: SecureRandom.uuid
+      }
+    end
+
+    it 'throttles the endpoint' do
+      # Allow requests under the rate limit.
+      request_limit.times do
+        expect_ok { get '/-/collector/i', params: params }
+      end
+
+      # Reject request over the limit
+      expect_rejection { get '/-/collector/i', params: params }
+
+      # Allow request from different IP
+      random_next_ip
+
+      expect_ok { get '/-/collector/i', params: params }
+    end
+  end
+
+  context 'per application id' do
+    let(:params) do
+      {
+        aid: 101,
+        eid: SecureRandom.uuid,
+      }
+    end
+
+    it 'throttles the endpoint' do
+      # Allow requests under the rate limit.
+      request_limit.times do
+        expect_ok { get '/-/collector/i', params: params }
+      end
+
+      # Ensure its not related to ip address
+      random_next_ip
+
+      # Reject request over the limit
+      expect_rejection { get '/-/collector/i', params: params }
+
+      # But allows request for different aid
+      expect_ok { get '/-/collector/i', params: params.merge(aid: 102) }
+    end
+  end
+
+  def random_next_ip
+    expect_next_instance_of(Rack::Attack::Request) do |instance|
+      expect(instance).to receive(:ip).at_least(:once).and_return(FFaker::Internet.ip_v4_address)
+    end
+  end
+end

spec/requests/product_analytics/collector_app_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe 'ProductAnalytics::CollectorApp' do
+  let_it_be(:project) { create(:project) }
+  let(:params) { {} }
+
+  subject { get '/-/collector/i', params: params }
+
+  context 'correct event params' do
+    let(:params) do
+      {
+        aid: project.id,
+        p: 'web',
+        tna: 'sp',
+        tv: 'js-2.14.0',
+        eid: SecureRandom.uuid,
+        duid: SecureRandom.uuid,
+        sid: SecureRandom.uuid,
+        vid: 4,
+        url: 'http://example.com/products/1',
+        refr: 'http://example.com/products/1',
+        lang: 'en-US',
+        cookie: true,
+        tz: 'America/Los_Angeles',
+        cs: 'UTF-8'
+      }
+    end
+
+    it 'repond with 200' do
+      expect { subject }.to change { ProductAnalyticsEvent.count }.by(1)
+
+      expect(response).to have_gitlab_http_status(:ok)
+    end
+  end
+
+  context 'empty event params' do
+    it 'responds with 404' do
+      expect { subject }.not_to change { ProductAnalyticsEvent.count }
+
+      expect(response).to have_gitlab_http_status(:not_found)
+    end
+  end
+end

spec/support/helpers/rack_attack_spec_helpers.rb

@@ -30,4 +30,10 @@ module RackAttackSpecHelpers
     expect(response).to have_gitlab_http_status(:too_many_requests)
   end
+
+  def expect_ok(&block)
+    yield
+
+    expect(response).to have_gitlab_http_status(:ok)
+  end
 end