diff --git a/CHANGELOG b/CHANGELOG
index 837e9e27aba9a7dc0d9588af0d332bff750e206a..4986ffd73516d300a7a37d35e3761d80edfb7ecf 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -16,6 +16,7 @@ v 8.11.0 (unreleased)
- API: List access requests, request access, approve, and deny access requests to a project or a group. !4833
- Use long options for curl examples in documentation !5703 (winniehell)
- Remove magic comments (`# encoding: UTF-8`) from Ruby files. !5456 (winniehell)
+ - GitLab Performance Monitoring can now track custom events such as the number of tags pushed to a repository
- Add support for relative links starting with ./ or / to RelativeLinkFilter (winniehell)
- Ignore URLs starting with // in Markdown links !5677 (winniehell)
- Fix CI status icon link underline (ClemMakesApps)
diff --git a/app/models/repository.rb b/app/models/repository.rb
index e56bac509a4ed1e774778c68b998565b8b1218b9..916843dabdd9427f20514a549ed382cb6913cd3e 100644
--- a/app/models/repository.rb
+++ b/app/models/repository.rb
@@ -391,6 +391,8 @@ class Repository
expire_exists_cache
expire_root_ref_cache
expire_emptiness_caches
+
+ repository_event(:create_repository)
end
# Runs code just before a repository is deleted.
@@ -407,6 +409,8 @@ class Repository
expire_root_ref_cache
expire_emptiness_caches
expire_exists_cache
+
+ repository_event(:remove_repository)
end
# Runs code just before the HEAD of a repository is changed.
@@ -414,6 +418,8 @@ class Repository
# Cached divergent commit counts are based on repository head
expire_branch_cache
expire_root_ref_cache
+
+ repository_event(:change_default_branch)
end
# Runs code before pushing (= creating or removing) a tag.
@@ -421,12 +427,16 @@ class Repository
expire_cache
expire_tags_cache
expire_tag_count_cache
+
+ repository_event(:push_tag)
end
# Runs code before removing a tag.
def before_remove_tag
expire_tags_cache
expire_tag_count_cache
+
+ repository_event(:remove_tag)
end
def before_import
@@ -443,6 +453,8 @@ class Repository
# Runs code after a new commit has been pushed.
def after_push_commit(branch_name, revision)
expire_cache(branch_name, revision)
+
+ repository_event(:push_commit, branch: branch_name)
end
# Runs code after a new branch has been created.
@@ -450,11 +462,15 @@ class Repository
expire_branches_cache
expire_has_visible_content_cache
expire_branch_count_cache
+
+ repository_event(:push_branch)
end
# Runs code before removing an existing branch.
def before_remove_branch
expire_branches_cache
+
+ repository_event(:remove_branch)
end
# Runs code after an existing branch has been removed.
@@ -1059,4 +1075,8 @@ class Repository
def keep_around_ref_name(sha)
"refs/keep-around/#{sha}"
end
+
+ def repository_event(event, tags = {})
+ Gitlab::Metrics.add_event(event, { path: path_with_namespace }.merge(tags))
+ end
end
diff --git a/app/workers/repository_fork_worker.rb b/app/workers/repository_fork_worker.rb
index d69d6037053d3da556ab1b80fde4a75a529706b9..61ed1c38ac427f20cbb3a71bdbe49e768c6687c8 100644
--- a/app/workers/repository_fork_worker.rb
+++ b/app/workers/repository_fork_worker.rb
@@ -5,6 +5,10 @@ class RepositoryForkWorker
sidekiq_options queue: :gitlab_shell
def perform(project_id, forked_from_repository_storage_path, source_path, target_path)
+ Gitlab::Metrics.add_event(:fork_repository,
+ source_path: source_path,
+ target_path: target_path)
+
project = Project.find_by_id(project_id)
unless project.present?
diff --git a/app/workers/repository_import_worker.rb b/app/workers/repository_import_worker.rb
index 7d819fe78f83fc24699af26b4806b46b4dda19b0..e6701078f71f86b972cb02d268127e5ab7a40f8e 100644
--- a/app/workers/repository_import_worker.rb
+++ b/app/workers/repository_import_worker.rb
@@ -10,6 +10,10 @@ class RepositoryImportWorker
@project = Project.find(project_id)
@current_user = @project.creator
+ Gitlab::Metrics.add_event(:import_repository,
+ import_url: @project.import_url,
+ path: @project.path_with_namespace)
+
result = Projects::ImportService.new(project, current_user).execute
if result[:status] == :error
diff --git a/doc/monitoring/performance/influxdb_schema.md b/doc/monitoring/performance/influxdb_schema.md
index 41861860b6d2d23ed9a5ec54fdefdcce18c6fcc0..eff0e29f58d5e856d762d6cc815b73b09b5806c3 100644
--- a/doc/monitoring/performance/influxdb_schema.md
+++ b/doc/monitoring/performance/influxdb_schema.md
@@ -9,6 +9,7 @@ The following measurements are currently stored in InfluxDB:
- `PROCESS_object_counts`
- `PROCESS_transactions`
- `PROCESS_views`
+- `events`
Here, `PROCESS` is replaced with either `rails` or `sidekiq` depending on the
process type. In all series, any form of duration is stored in milliseconds.
@@ -78,6 +79,14 @@ following value fields are available:
The `action` tag contains the action name of the transaction that rendered the
view.
+## events
+
+This measurement is used to store generic events such as the number of Git
+pushes, Emails sent, etc. Each point in this measurement has a single value
+field called `count`. The value of this field is simply set to `1`. Each point
+also has at least one tag: `event`. This tag's value is set to the event name.
+Depending on the event type additional tags may be available as well.
+
---
Read more on:
diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb
index 260ac81f5fab8ea822a250a664d8426f2ab8b7a0..9f3b582a263c0a9d5aa35f1570bf787778742184 100644
--- a/lib/ci/api/builds.rb
+++ b/lib/ci/api/builds.rb
@@ -20,8 +20,13 @@ module Ci
build = Ci::RegisterBuildService.new.execute(current_runner)
if build
+ Gitlab::Metrics.add_event(:build_found,
+ project: build.project.path_with_namespace)
+
present build, with: Entities::BuildDetails
else
+ Gitlab::Metrics.add_event(:build_not_found)
+
not_found!
end
end
@@ -42,6 +47,9 @@ module Ci
build.update_attributes(trace: params[:trace]) if params[:trace]
+ Gitlab::Metrics.add_event(:update_build,
+ project: build.project.path_with_namespace)
+
case params[:state].to_s
when 'success'
build.success
diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb
index 41fcd971c228c1341ba807c24e389d86d2a2c3c9..3d1ba33ec68d69378450449758b211fe1b2a7509 100644
--- a/lib/gitlab/metrics.rb
+++ b/lib/gitlab/metrics.rb
@@ -124,6 +124,15 @@ module Gitlab
trans.action = action if trans
end
+ # Tracks an event.
+ #
+ # See `Gitlab::Metrics::Transaction#add_event` for more details.
+ def self.add_event(*args)
+ trans = current_transaction
+
+ trans.add_event(*args) if trans
+ end
+
# Returns the prefix to use for the name of a series.
def self.series_prefix
@series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_'
diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb
index f23d67e1e3849f4ab512c2d390110605009988d9..bd0afe53c5166da5abc2515a57f0a8d82d64fb8d 100644
--- a/lib/gitlab/metrics/metric.rb
+++ b/lib/gitlab/metrics/metric.rb
@@ -4,15 +4,20 @@ module Gitlab
class Metric
JITTER_RANGE = 0.000001..0.001
- attr_reader :series, :values, :tags
+ attr_reader :series, :values, :tags, :type
# series - The name of the series (as a String) to store the metric in.
# values - A Hash containing the values to store.
# tags - A Hash containing extra tags to add to the metrics.
- def initialize(series, values, tags = {})
+ def initialize(series, values, tags = {}, type = :metric)
@values = values
@series = series
@tags = tags
+ @type = type
+ end
+
+ def event?
+ type == :event
end
# Returns a Hash in a format that can be directly written to InfluxDB.
diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb
index e61670f491cfd5d8b58f1366fa834b3680aefe99..b4493bf44d28eacde9d5828ab30db6598aa48ed7 100644
--- a/lib/gitlab/metrics/rack_middleware.rb
+++ b/lib/gitlab/metrics/rack_middleware.rb
@@ -17,6 +17,10 @@ module Gitlab
begin
retval = trans.run { @app.call(env) }
+ rescue Exception => error # rubocop: disable Lint/RescueException
+ trans.add_event(:rails_exception)
+
+ raise error
# Even in the event of an error we want to submit any metrics we
# might've gathered up to this point.
ensure
diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb
index a1240fd33eee739d2e62990f72e161f5cb6a0c13..f9dd8e419128e1330bed49c974779cb29775fb4c 100644
--- a/lib/gitlab/metrics/sidekiq_middleware.rb
+++ b/lib/gitlab/metrics/sidekiq_middleware.rb
@@ -11,6 +11,10 @@ module Gitlab
# Old gitlad-shell messages don't provide enqueued_at/created_at attributes
trans.set(:sidekiq_queue_duration, Time.now.to_f - (message['enqueued_at'] || message['created_at'] || 0))
trans.run { yield }
+ rescue Exception => error # rubocop: disable Lint/RescueException
+ trans.add_event(:sidekiq_exception)
+
+ raise error
ensure
trans.finish
end
diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb
index 968f32189505326c02c3c32a5a08498b83e46875..7bc16181be624cac25ed051d8b27b94a20a04440 100644
--- a/lib/gitlab/metrics/transaction.rb
+++ b/lib/gitlab/metrics/transaction.rb
@@ -4,7 +4,10 @@ module Gitlab
class Transaction
THREAD_KEY = :_gitlab_metrics_transaction
- attr_reader :tags, :values, :methods
+ # The series to store events (e.g. Git pushes) in.
+ EVENT_SERIES = 'events'
+
+ attr_reader :tags, :values, :method, :metrics
attr_accessor :action
@@ -55,6 +58,20 @@ module Gitlab
@metrics << Metric.new("#{Metrics.series_prefix}#{series}", values, tags)
end
+ # Tracks a business level event
+ #
+ # Business level events including events such as Git pushes, Emails being
+ # sent, etc.
+ #
+ # event_name - The name of the event (e.g. "git_push").
+ # tags - A set of tags to attach to the event.
+ def add_event(event_name, tags = {})
+ @metrics << Metric.new(EVENT_SERIES,
+ { count: 1 },
+ { event: event_name }.merge(tags),
+ :event)
+ end
+
# Returns a MethodCall object for the given name.
def method_call_for(name)
unless method = @methods[name]
@@ -101,7 +118,7 @@ module Gitlab
submit_hashes = submit.map do |metric|
hash = metric.to_hash
- hash[:tags][:action] ||= @action if @action
+ hash[:tags][:action] ||= @action if @action && !metric.event?
hash
end
diff --git a/spec/lib/gitlab/metrics/metric_spec.rb b/spec/lib/gitlab/metrics/metric_spec.rb
index f718d536130a35e09f2948129461067c1f9b570c..f26fca52c5067a77d0d7679fc6fabc802ee462a0 100644
--- a/spec/lib/gitlab/metrics/metric_spec.rb
+++ b/spec/lib/gitlab/metrics/metric_spec.rb
@@ -23,6 +23,24 @@ describe Gitlab::Metrics::Metric do
it { is_expected.to eq({ host: 'localtoast' }) }
end
+ describe '#type' do
+ subject { metric.type }
+
+ it { is_expected.to eq(:metric) }
+ end
+
+ describe '#event?' do
+ it 'returns false for a regular metric' do
+ expect(metric.event?).to eq(false)
+ end
+
+ it 'returns true for an event metric' do
+ expect(metric).to receive(:type).and_return(:event)
+
+ expect(metric.event?).to eq(true)
+ end
+ end
+
describe '#to_hash' do
it 'returns a Hash' do
expect(metric.to_hash).to be_an_instance_of(Hash)
diff --git a/spec/lib/gitlab/metrics/rack_middleware_spec.rb b/spec/lib/gitlab/metrics/rack_middleware_spec.rb
index f264ed64029a532763611948e0f0b03f7e3e7392..a30cb2a5e38220c41e8683062781542507cd15de 100644
--- a/spec/lib/gitlab/metrics/rack_middleware_spec.rb
+++ b/spec/lib/gitlab/metrics/rack_middleware_spec.rb
@@ -45,6 +45,15 @@ describe Gitlab::Metrics::RackMiddleware do
middleware.call(env)
end
+
+ it 'tracks any raised exceptions' do
+ expect(app).to receive(:call).with(env).and_raise(RuntimeError)
+
+ expect_any_instance_of(Gitlab::Metrics::Transaction).
+ to receive(:add_event).with(:rails_exception)
+
+ expect { middleware.call(env) }.to raise_error(RuntimeError)
+ end
end
describe '#transaction_from_env' do
diff --git a/spec/lib/gitlab/metrics/sidekiq_middleware_spec.rb b/spec/lib/gitlab/metrics/sidekiq_middleware_spec.rb
index 4d2aa03e722cbac5d5d2c8eef9cc39980ccf28a1..acaba785606e13e9b43348a3fa4247eb3ac56d91 100644
--- a/spec/lib/gitlab/metrics/sidekiq_middleware_spec.rb
+++ b/spec/lib/gitlab/metrics/sidekiq_middleware_spec.rb
@@ -12,7 +12,9 @@ describe Gitlab::Metrics::SidekiqMiddleware do
with('TestWorker#perform').
and_call_original
- expect_any_instance_of(Gitlab::Metrics::Transaction).to receive(:set).with(:sidekiq_queue_duration, instance_of(Float))
+ expect_any_instance_of(Gitlab::Metrics::Transaction).to receive(:set).
+ with(:sidekiq_queue_duration, instance_of(Float))
+
expect_any_instance_of(Gitlab::Metrics::Transaction).to receive(:finish)
middleware.call(worker, message, :test) { nil }
@@ -25,10 +27,28 @@ describe Gitlab::Metrics::SidekiqMiddleware do
with('TestWorker#perform').
and_call_original
- expect_any_instance_of(Gitlab::Metrics::Transaction).to receive(:set).with(:sidekiq_queue_duration, instance_of(Float))
+ expect_any_instance_of(Gitlab::Metrics::Transaction).to receive(:set).
+ with(:sidekiq_queue_duration, instance_of(Float))
+
expect_any_instance_of(Gitlab::Metrics::Transaction).to receive(:finish)
middleware.call(worker, {}, :test) { nil }
end
+
+ it 'tracks any raised exceptions' do
+ worker = double(:worker, class: double(:class, name: 'TestWorker'))
+
+ expect_any_instance_of(Gitlab::Metrics::Transaction).
+ to receive(:run).and_raise(RuntimeError)
+
+ expect_any_instance_of(Gitlab::Metrics::Transaction).
+ to receive(:add_event).with(:sidekiq_exception)
+
+ expect_any_instance_of(Gitlab::Metrics::Transaction).
+ to receive(:finish)
+
+ expect { middleware.call(worker, message, :test) }.
+ to raise_error(RuntimeError)
+ end
end
end
diff --git a/spec/lib/gitlab/metrics/transaction_spec.rb b/spec/lib/gitlab/metrics/transaction_spec.rb
index f1a191d94100889466524e3a8a81e9e052a6e590..3887c04c83214b19f531523afd6e646e0c3104f1 100644
--- a/spec/lib/gitlab/metrics/transaction_spec.rb
+++ b/spec/lib/gitlab/metrics/transaction_spec.rb
@@ -142,5 +142,62 @@ describe Gitlab::Metrics::Transaction do
transaction.submit
end
+
+ it 'does not add an action tag for events' do
+ transaction.action = 'Foo#bar'
+ transaction.add_event(:meow)
+
+ hash = {
+ series: 'events',
+ tags: { event: :meow },
+ values: { count: 1 },
+ timestamp: an_instance_of(Fixnum)
+ }
+
+ expect(Gitlab::Metrics).to receive(:submit_metrics).
+ with([hash])
+
+ transaction.submit
+ end
+ end
+
+ describe '#add_event' do
+ it 'adds a metric' do
+ transaction.add_event(:meow)
+
+ expect(transaction.metrics[0]).to be_an_instance_of(Gitlab::Metrics::Metric)
+ end
+
+ it "does not prefix the metric's series name" do
+ transaction.add_event(:meow)
+
+ metric = transaction.metrics[0]
+
+ expect(metric.series).to eq(described_class::EVENT_SERIES)
+ end
+
+ it 'tracks a counter for every event' do
+ transaction.add_event(:meow)
+
+ metric = transaction.metrics[0]
+
+ expect(metric.values).to eq(count: 1)
+ end
+
+ it 'tracks the event name' do
+ transaction.add_event(:meow)
+
+ metric = transaction.metrics[0]
+
+ expect(metric.tags).to eq(event: :meow)
+ end
+
+ it 'allows tracking of custom tags' do
+ transaction.add_event(:meow, animal: 'cat')
+
+ metric = transaction.metrics[0]
+
+ expect(metric.tags).to eq(event: :meow, animal: 'cat')
+ end
end
end
diff --git a/spec/lib/gitlab/metrics_spec.rb b/spec/lib/gitlab/metrics_spec.rb
index 84f9475a0f85fbe66d58c8adce93d0eb7edd04f7..ab6e311b1e80b4034febc97db91c69f4393a8bfd 100644
--- a/spec/lib/gitlab/metrics_spec.rb
+++ b/spec/lib/gitlab/metrics_spec.rb
@@ -153,4 +153,28 @@ describe Gitlab::Metrics do
expect(described_class.series_prefix).to be_an_instance_of(String)
end
end
+
+ describe '.add_event' do
+ context 'without a transaction' do
+ it 'does nothing' do
+ expect_any_instance_of(Gitlab::Metrics::Transaction).
+ not_to receive(:add_event)
+
+ Gitlab::Metrics.add_event(:meow)
+ end
+ end
+
+ context 'with a transaction' do
+ it 'adds an event' do
+ transaction = Gitlab::Metrics::Transaction.new
+
+ expect(transaction).to receive(:add_event).with(:meow)
+
+ expect(Gitlab::Metrics).to receive(:current_transaction).
+ and_return(transaction)
+
+ Gitlab::Metrics.add_event(:meow)
+ end
+ end
+ end
end