diff --git a/Gemfile b/Gemfile
index 71d2eb1557cdda0f860492d1999c75905177017a..323827ed01b5641e9796731cb606aa46e9a0b858 100644
--- a/Gemfile
+++ b/Gemfile
@@ -285,6 +285,7 @@ group :metrics do
# Prometheus
gem 'prometheus-client-mmap', '~>0.7.0.beta5'
+ gem 'raindrops', '~> 0.18'
end
group :development do
diff --git a/Gemfile.lock b/Gemfile.lock
index f4ddd30da1b44a2d3a74a9c3ee7d33bf747a25cd..e6ea006356cb2cd4e8328125070ad87c0c288207 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -599,7 +599,7 @@ GEM
premailer-rails (1.9.7)
actionmailer (>= 3, < 6)
premailer (~> 1.7, >= 1.7.9)
- prometheus-client-mmap (0.7.0.beta5)
+ prometheus-client-mmap (0.7.0.beta7)
mmap2 (~> 2.2.6)
pry (0.10.4)
coderay (~> 1.1.0)
@@ -658,7 +658,7 @@ GEM
thor (>= 0.18.1, < 2.0)
rainbow (2.2.2)
rake
- raindrops (0.17.0)
+ raindrops (0.18.0)
rake (10.5.0)
rblineprof (0.3.6)
debugger-ruby_core_source (~> 1.3)
@@ -1062,6 +1062,7 @@ DEPENDENCIES
rails-deprecated_sanitizer (~> 1.0.3)
rails-i18n (~> 4.0.9)
rainbow (~> 2.2)
+ raindrops (~> 0.18)
rblineprof (~> 0.3.6)
rdoc (~> 4.2)
recaptcha (~> 3.0)
diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example
index 43a8c0078cae6b6ab525872eb443d9bbc8f94dc7..4b81fd90f593eaa1de82a42cc9499754c3034103 100644
--- a/config/gitlab.yml.example
+++ b/config/gitlab.yml.example
@@ -543,6 +543,10 @@ production: &base
# enabled: true
# host: localhost
# port: 3808
+ prometheus:
+ # Time between sampling of unicorn socket metrics, in seconds
+ # unicorn_sampler_interval: 10
+
#
# 5. Extra customization
diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb
index 8ddf8e4d2e42745579c55c68baaca52a52e20e94..cb11d2c34f4877cf9b2b76c51806f5cd0374d68f 100644
--- a/config/initializers/1_settings.rb
+++ b/config/initializers/1_settings.rb
@@ -494,6 +494,12 @@ Settings.webpack.dev_server['enabled'] ||= false
Settings.webpack.dev_server['host'] ||= 'localhost'
Settings.webpack.dev_server['port'] ||= 3808
+#
+# Prometheus metrics settings
+#
+Settings['prometheus'] ||= Settingslogic.new({})
+Settings.prometheus['unicorn_sampler_interval'] ||= 10
+
#
# Testing settings
#
diff --git a/config/initializers/8_metrics.rb b/config/initializers/8_metrics.rb
index a0a63ddf8f05acf5e0ce5fe11ff6b9800b02d824..d56fd7a6cfa282db864a19f387ff8f9fa0e2e11c 100644
--- a/config/initializers/8_metrics.rb
+++ b/config/initializers/8_metrics.rb
@@ -119,6 +119,13 @@ def instrument_classes(instrumentation)
end
# rubocop:enable Metrics/AbcSize
+Gitlab::Metrics::UnicornSampler.initialize_instance(Settings.prometheus.unicorn_sampler_interval).start
+
+Gitlab::Application.configure do |config|
+ # 0 should be Sentry to catch errors in this middleware
+ config.middleware.insert(1, Gitlab::Metrics::ConnectionRackMiddleware)
+end
+
if Gitlab::Metrics.enabled?
require 'pathname'
require 'influxdb'
@@ -175,7 +182,7 @@ if Gitlab::Metrics.enabled?
GC::Profiler.enable
- Gitlab::Metrics::Sampler.new.start
+ Gitlab::Metrics::InfluxSampler.initialize_instance.start
module TrackNewRedisConnections
def connect(*args)
diff --git a/lib/gitlab/metrics/base_sampler.rb b/lib/gitlab/metrics/base_sampler.rb
new file mode 100644
index 0000000000000000000000000000000000000000..219accfc029459510b429f79ba772e42559192f8
--- /dev/null
+++ b/lib/gitlab/metrics/base_sampler.rb
@@ -0,0 +1,94 @@
+require 'logger'
+module Gitlab
+ module Metrics
+ class BaseSampler
+ def self.initialize_instance(*args)
+ raise "#{name} singleton instance already initialized" if @instance
+ @instance = new(*args)
+ at_exit(&@instance.method(:stop))
+ @instance
+ end
+
+ def self.instance
+ @instance
+ end
+
+ attr_reader :running
+
+ # interval - The sampling interval in seconds.
+ def initialize(interval)
+ interval_half = interval.to_f / 2
+
+ @interval = interval
+ @interval_steps = (-interval_half..interval_half).step(0.1).to_a
+
+ @mutex = Mutex.new
+ end
+
+ def enabled?
+ true
+ end
+
+ def start
+ return unless enabled?
+
+ @mutex.synchronize do
+ return if running
+ @running = true
+
+ @thread = Thread.new do
+ sleep(sleep_interval)
+
+ while running
+ safe_sample
+
+ sleep(sleep_interval)
+ end
+ end
+ end
+ end
+
+ def stop
+ @mutex.synchronize do
+ return unless running
+
+ @running = false
+
+ if @thread
+ @thread.wakeup if @thread.alive?
+ @thread.join
+ @thread = nil
+ end
+ end
+ end
+
+ def safe_sample
+ sample
+ rescue => e
+ Rails.logger.warn("#{self.class}: #{e}, stopping")
+ stop
+ end
+
+ def sample
+ raise NotImplementedError
+ end
+
+ # Returns the sleep interval with a random adjustment.
+ #
+ # The random adjustment is put in place to ensure we:
+ #
+ # 1. Don't generate samples at the exact same interval every time (thus
+ # potentially missing anything that happens in between samples).
+ # 2. Don't sample data at the same interval two times in a row.
+ def sleep_interval
+ while step = @interval_steps.sample
+ if step != @last_step
+ @last_step = step
+
+ return @interval + @last_step
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/metrics/connection_rack_middleware.rb b/lib/gitlab/metrics/connection_rack_middleware.rb
new file mode 100644
index 0000000000000000000000000000000000000000..b3da360be8f3500aaa3fb28bf6fff5e98ba92baa
--- /dev/null
+++ b/lib/gitlab/metrics/connection_rack_middleware.rb
@@ -0,0 +1,45 @@
+module Gitlab
+ module Metrics
+ class ConnectionRackMiddleware
+ def initialize(app)
+ @app = app
+ end
+
+ def self.rack_request_count
+ @rack_request_count ||= Gitlab::Metrics.counter(:rack_request, 'Rack request count')
+ end
+
+ def self.rack_response_count
+ @rack_response_count ||= Gitlab::Metrics.counter(:rack_response, 'Rack response count')
+ end
+
+ def self.rack_uncaught_errors_count
+ @rack_uncaught_errors_count ||= Gitlab::Metrics.counter(:rack_uncaught_errors, 'Rack connections handling uncaught errors count')
+ end
+
+ def self.rack_execution_time
+ @rack_execution_time ||= Gitlab::Metrics.histogram(:rack_execution_time, 'Rack connection handling execution time',
+ {}, [0.05, 0.1, 0.25, 0.5, 0.7, 1, 1.5, 2, 2.5, 3, 5, 7, 10])
+ end
+
+ def call(env)
+ method = env['REQUEST_METHOD'].downcase
+ started = Time.now.to_f
+ begin
+ ConnectionRackMiddleware.rack_request_count.increment(method: method)
+
+ status, headers, body = @app.call(env)
+
+ ConnectionRackMiddleware.rack_response_count.increment(method: method, status: status)
+ [status, headers, body]
+ rescue
+ ConnectionRackMiddleware.rack_uncaught_errors_count.increment
+ raise
+ ensure
+ elapsed = Time.now.to_f - started
+ ConnectionRackMiddleware.rack_execution_time.observe({}, elapsed)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/metrics/sampler.rb b/lib/gitlab/metrics/influx_sampler.rb
similarity index 71%
rename from lib/gitlab/metrics/sampler.rb
rename to lib/gitlab/metrics/influx_sampler.rb
index 0000450d9bb6f7ba7d47ef6a54ca67834de13729..6db1dd755b7376e5d9b788c67e28d46d9858ea7e 100644
--- a/lib/gitlab/metrics/sampler.rb
+++ b/lib/gitlab/metrics/influx_sampler.rb
@@ -5,14 +5,11 @@ module Gitlab
# This class is used to gather statistics that can't be directly associated
# with a transaction such as system memory usage, garbage collection
# statistics, etc.
- class Sampler
+ class InfluxSampler < BaseSampler
# interval - The sampling interval in seconds.
def initialize(interval = Metrics.settings[:sample_interval])
- interval_half = interval.to_f / 2
-
- @interval = interval
- @interval_steps = (-interval_half..interval_half).step(0.1).to_a
- @last_step = nil
+ super(interval)
+ @last_step = nil
@metrics = []
@@ -26,18 +23,6 @@ module Gitlab
end
end
- def start
- Thread.new do
- Thread.current.abort_on_exception = true
-
- loop do
- sleep(sleep_interval)
-
- sample
- end
- end
- end
-
def sample
sample_memory_usage
sample_file_descriptors
@@ -86,7 +71,7 @@ module Gitlab
end
def sample_gc
- time = GC::Profiler.total_time * 1000.0
+ time = GC::Profiler.total_time * 1000.0
stats = GC.stat.merge(total_time: time)
# We want the difference of GC runs compared to the last sample, not the
@@ -111,23 +96,6 @@ module Gitlab
def sidekiq?
Sidekiq.server?
end
-
- # Returns the sleep interval with a random adjustment.
- #
- # The random adjustment is put in place to ensure we:
- #
- # 1. Don't generate samples at the exact same interval every time (thus
- # potentially missing anything that happens in between samples).
- # 2. Don't sample data at the same interval two times in a row.
- def sleep_interval
- while step = @interval_steps.sample
- if step != @last_step
- @last_step = step
-
- return @interval + @last_step
- end
- end
- end
end
end
end
diff --git a/lib/gitlab/metrics/prometheus.rb b/lib/gitlab/metrics/prometheus.rb
index 9d314a56e5890db90b3188dd268a9881fdf20a40..fb7bbc7cfc7f458d7c6af7aac64a96f853b8e469 100644
--- a/lib/gitlab/metrics/prometheus.rb
+++ b/lib/gitlab/metrics/prometheus.rb
@@ -29,8 +29,8 @@ module Gitlab
provide_metric(name) || registry.summary(name, docstring, base_labels)
end
- def gauge(name, docstring, base_labels = {})
- provide_metric(name) || registry.gauge(name, docstring, base_labels)
+ def gauge(name, docstring, base_labels = {}, multiprocess_mode = :all)
+ provide_metric(name) || registry.gauge(name, docstring, base_labels, multiprocess_mode)
end
def histogram(name, docstring, base_labels = {}, buckets = ::Prometheus::Client::Histogram::DEFAULT_BUCKETS)
diff --git a/lib/gitlab/metrics/unicorn_sampler.rb b/lib/gitlab/metrics/unicorn_sampler.rb
new file mode 100644
index 0000000000000000000000000000000000000000..f69872520398a1ad8b160b271d816642231f1217
--- /dev/null
+++ b/lib/gitlab/metrics/unicorn_sampler.rb
@@ -0,0 +1,48 @@
+module Gitlab
+ module Metrics
+ class UnicornSampler < BaseSampler
+ def initialize(interval)
+ super(interval)
+ end
+
+ def unicorn_active_connections
+ @unicorn_active_connections ||= Gitlab::Metrics.gauge(:unicorn_active_connections, 'Unicorn active connections', {}, :max)
+ end
+
+ def unicorn_queued_connections
+ @unicorn_queued_connections ||= Gitlab::Metrics.gauge(:unicorn_queued_connections, 'Unicorn queued connections', {}, :max)
+ end
+
+ def enabled?
+ # Raindrops::Linux.tcp_listener_stats is only present on Linux
+ unicorn_with_listeners? && Raindrops::Linux.respond_to?(:tcp_listener_stats)
+ end
+
+ def sample
+ Raindrops::Linux.tcp_listener_stats(tcp_listeners).each do |addr, stats|
+ unicorn_active_connections.set({ type: 'tcp', address: addr }, stats.active)
+ unicorn_queued_connections.set({ type: 'tcp', address: addr }, stats.queued)
+ end
+
+ Raindrops::Linux.unix_listener_stats(unix_listeners).each do |addr, stats|
+ unicorn_active_connections.set({ type: 'unix', address: addr }, stats.active)
+ unicorn_queued_connections.set({ type: 'unix', address: addr }, stats.queued)
+ end
+ end
+
+ private
+
+ def tcp_listeners
+ @tcp_listeners ||= Unicorn.listener_names.grep(%r{\A[^/]+:\d+\z})
+ end
+
+ def unix_listeners
+ @unix_listeners ||= Unicorn.listener_names - tcp_listeners
+ end
+
+ def unicorn_with_listeners?
+ defined?(Unicorn) && Unicorn.listener_names.any?
+ end
+ end
+ end
+end
diff --git a/spec/initializers/8_metrics_spec.rb b/spec/initializers/8_metrics_spec.rb
index a507d7f7f2b376988b18afd2e00c4c8d7650189d..d4189f902fd71f477c18c25f233500addb85f7f0 100644
--- a/spec/initializers/8_metrics_spec.rb
+++ b/spec/initializers/8_metrics_spec.rb
@@ -1,17 +1,25 @@
require 'spec_helper'
-require_relative '../../config/initializers/8_metrics'
describe 'instrument_classes', lib: true do
let(:config) { double(:config) }
+ let(:unicorn_sampler) { double(:unicorn_sampler) }
+ let(:influx_sampler) { double(:influx_sampler) }
+
before do
allow(config).to receive(:instrument_method)
allow(config).to receive(:instrument_methods)
allow(config).to receive(:instrument_instance_method)
allow(config).to receive(:instrument_instance_methods)
+ allow(Gitlab::Metrics::UnicornSampler).to receive(:initialize_instance).and_return(unicorn_sampler)
+ allow(Gitlab::Metrics::InfluxSampler).to receive(:initialize_instance).and_return(influx_sampler)
+ allow(unicorn_sampler).to receive(:start)
+ allow(influx_sampler).to receive(:start)
+ allow(Gitlab::Application).to receive(:configure)
end
it 'can autoload and instrument all files' do
+ require_relative '../../config/initializers/8_metrics'
expect { instrument_classes(config) }.not_to raise_error
end
end
diff --git a/spec/lib/gitlab/metrics/connection_rack_middleware_spec.rb b/spec/lib/gitlab/metrics/connection_rack_middleware_spec.rb
new file mode 100644
index 0000000000000000000000000000000000000000..94251af305f6f940893ffbd97d9f393847cb0b0e
--- /dev/null
+++ b/spec/lib/gitlab/metrics/connection_rack_middleware_spec.rb
@@ -0,0 +1,88 @@
+require 'spec_helper'
+
+describe Gitlab::Metrics::ConnectionRackMiddleware do
+ let(:app) { double('app') }
+ subject { described_class.new(app) }
+
+ around do |example|
+ Timecop.freeze { example.run }
+ end
+
+ describe '#call' do
+ let(:status) { 100 }
+ let(:env) { { 'REQUEST_METHOD' => 'GET' } }
+ let(:stack_result) { [status, {}, 'body'] }
+
+ before do
+ allow(app).to receive(:call).and_return(stack_result)
+ end
+
+ context '@app.call succeeds with 200' do
+ before do
+ allow(app).to receive(:call).and_return([200, nil, nil])
+ end
+
+ it 'increments response count with status label' do
+ expect(described_class).to receive_message_chain(:rack_response_count, :increment).with(include(status: 200, method: 'get'))
+
+ subject.call(env)
+ end
+
+ it 'increments requests count' do
+ expect(described_class).to receive_message_chain(:rack_request_count, :increment).with(method: 'get')
+
+ subject.call(env)
+ end
+
+ it 'measures execution time' do
+ execution_time = 10
+ allow(app).to receive(:call) do |*args|
+ Timecop.freeze(execution_time.seconds)
+ end
+
+ expect(described_class).to receive_message_chain(:rack_execution_time, :observe).with({}, execution_time)
+
+ subject.call(env)
+ end
+ end
+
+ context '@app.call throws exception' do
+ let(:rack_response_count) { double('rack_response_count') }
+
+ before do
+ allow(app).to receive(:call).and_raise(StandardError)
+ allow(described_class).to receive(:rack_response_count).and_return(rack_response_count)
+ end
+
+ it 'increments exceptions count' do
+ expect(described_class).to receive_message_chain(:rack_uncaught_errors_count, :increment)
+
+ expect { subject.call(env) }.to raise_error(StandardError)
+ end
+
+ it 'increments requests count' do
+ expect(described_class).to receive_message_chain(:rack_request_count, :increment).with(method: 'get')
+
+ expect { subject.call(env) }.to raise_error(StandardError)
+ end
+
+ it "does't increment response count" do
+ expect(described_class.rack_response_count).not_to receive(:increment)
+
+ expect { subject.call(env) }.to raise_error(StandardError)
+ end
+
+ it 'measures execution time' do
+ execution_time = 10
+ allow(app).to receive(:call) do |*args|
+ Timecop.freeze(execution_time.seconds)
+ raise StandardError
+ end
+
+ expect(described_class).to receive_message_chain(:rack_execution_time, :observe).with({}, execution_time)
+
+ expect { subject.call(env) }.to raise_error(StandardError)
+ end
+ end
+ end
+end
diff --git a/spec/lib/gitlab/metrics/sampler_spec.rb b/spec/lib/gitlab/metrics/influx_sampler_spec.rb
similarity index 94%
rename from spec/lib/gitlab/metrics/sampler_spec.rb
rename to spec/lib/gitlab/metrics/influx_sampler_spec.rb
index d07ce6f81af2450eb6234bd7a65af5a75062991d..0bc68d642761b559c8b30f13d64e24f083ca5f2f 100644
--- a/spec/lib/gitlab/metrics/sampler_spec.rb
+++ b/spec/lib/gitlab/metrics/influx_sampler_spec.rb
@@ -1,6 +1,6 @@
require 'spec_helper'
-describe Gitlab::Metrics::Sampler do
+describe Gitlab::Metrics::InfluxSampler do
let(:sampler) { described_class.new(5) }
after do
@@ -8,10 +8,10 @@ describe Gitlab::Metrics::Sampler do
end
describe '#start' do
- it 'gathers a sample at a given interval' do
- expect(sampler).to receive(:sleep).with(a_kind_of(Numeric))
- expect(sampler).to receive(:sample)
- expect(sampler).to receive(:loop).and_yield
+ it 'runs once and gathers a sample at a given interval' do
+ expect(sampler).to receive(:sleep).with(a_kind_of(Numeric)).twice
+ expect(sampler).to receive(:sample).once
+ expect(sampler).to receive(:running).and_return(false, true, false)
sampler.start.join
end
diff --git a/spec/lib/gitlab/metrics/unicorn_sampler_spec.rb b/spec/lib/gitlab/metrics/unicorn_sampler_spec.rb
new file mode 100644
index 0000000000000000000000000000000000000000..dc0d1f2e940ad8c1cc49506b6fb3f5ee3a84d356
--- /dev/null
+++ b/spec/lib/gitlab/metrics/unicorn_sampler_spec.rb
@@ -0,0 +1,108 @@
+require 'spec_helper'
+
+describe Gitlab::Metrics::UnicornSampler do
+ subject { described_class.new(1.second) }
+
+ describe '#sample' do
+ let(:unicorn) { double('unicorn') }
+ let(:raindrops) { double('raindrops') }
+ let(:stats) { double('stats') }
+
+ before do
+ stub_const('Unicorn', unicorn)
+ stub_const('Raindrops::Linux', raindrops)
+ allow(raindrops).to receive(:unix_listener_stats).and_return({})
+ allow(raindrops).to receive(:tcp_listener_stats).and_return({})
+ end
+
+ context 'unicorn listens on unix sockets' do
+ let(:socket_address) { '/some/sock' }
+ let(:sockets) { [socket_address] }
+
+ before do
+ allow(unicorn).to receive(:listener_names).and_return(sockets)
+ end
+
+ it 'samples socket data' do
+ expect(raindrops).to receive(:unix_listener_stats).with(sockets)
+
+ subject.sample
+ end
+
+ context 'stats collected' do
+ before do
+ allow(stats).to receive(:active).and_return('active')
+ allow(stats).to receive(:queued).and_return('queued')
+ allow(raindrops).to receive(:unix_listener_stats).and_return({ socket_address => stats })
+ end
+
+ it 'updates metrics type unix and with addr' do
+ labels = { type: 'unix', address: socket_address }
+
+ expect(subject).to receive_message_chain(:unicorn_active_connections, :set).with(labels, 'active')
+ expect(subject).to receive_message_chain(:unicorn_queued_connections, :set).with(labels, 'queued')
+
+ subject.sample
+ end
+ end
+ end
+
+ context 'unicorn listens on tcp sockets' do
+ let(:tcp_socket_address) { '0.0.0.0:8080' }
+ let(:tcp_sockets) { [tcp_socket_address] }
+
+ before do
+ allow(unicorn).to receive(:listener_names).and_return(tcp_sockets)
+ end
+
+ it 'samples socket data' do
+ expect(raindrops).to receive(:tcp_listener_stats).with(tcp_sockets)
+
+ subject.sample
+ end
+
+ context 'stats collected' do
+ before do
+ allow(stats).to receive(:active).and_return('active')
+ allow(stats).to receive(:queued).and_return('queued')
+ allow(raindrops).to receive(:tcp_listener_stats).and_return({ tcp_socket_address => stats })
+ end
+
+ it 'updates metrics type unix and with addr' do
+ labels = { type: 'tcp', address: tcp_socket_address }
+
+ expect(subject).to receive_message_chain(:unicorn_active_connections, :set).with(labels, 'active')
+ expect(subject).to receive_message_chain(:unicorn_queued_connections, :set).with(labels, 'queued')
+
+ subject.sample
+ end
+ end
+ end
+ end
+
+ describe '#start' do
+ context 'when enabled' do
+ before do
+ allow(subject).to receive(:enabled?).and_return(true)
+ end
+
+ it 'creates new thread' do
+ expect(Thread).to receive(:new)
+
+ subject.start
+ end
+ end
+
+ context 'when disabled' do
+ before do
+ allow(subject).to receive(:enabled?).and_return(false)
+ end
+
+ it "doesn't create new thread" do
+ expect(Thread).not_to receive(:new)
+
+ subject.start
+ end
+ end
+ end
+end