Initial commit

.env.example

+# The name of the user to connect to the database with.
+INFLUX_USER='gitlab'
+
+# The password of the above user
+INFLUX_PASSWORD=''
+
+# The hostname of the server running the database
+INFLUX_HOST='performance.example.com'
+
+# The name of the database.
+INFLUX_DATABASE='gitlab'
+
+# The port number on which InfluxDB is running
+INFLUX_PORT='8086'

.gitignore

+.env

Gemfile

+source 'https://rubygems.org'
+
+gem 'influxdb'
+gem 'dotenv'
+gem 'rake'

Gemfile.lock

+GEM
+  remote: https://rubygems.org/
+  specs:
+    cause (0.1)
+    dotenv (2.1.1)
+    influxdb (0.3.5)
+      cause
+      json
+    json (1.8.3)
+    rake (11.2.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  dotenv
+  influxdb
+  rake
+
+BUNDLED WITH
+   1.12.5

README.md

+# InfluxDB Management
+
+Manually managing an InfluxDB database is a bit annoying. For example, whenever
+the list of continuous queries is updated one has to copy-paste the whole list
+into an InfluxDB console and run every query.
+
+This repository contains some scripts to more easily keep your database up to
+date so it can be used for storing GitLab performance metrics.
+
+## Requirements
+
+* Ruby 2.x
+* Bundler
+
+## Installation
+
+First make sure you have Bundler installed:
+
+    gem install bundler --no-ri --no-rdoc
+
+Once Bundler is installed you can install the dependencies of this project:
+
+    bundle install
+
+Next, make sure you have the following details of your InfluxDB database:
+
+* A username
+* A password
+* The hostname of the server running the database
+* The database name (usually "gitlab")
+
+These details are needed to connect to the InfluxDB database. Note that the user
+must be able to execute the `CREATE CONTINUOUS QUERY` and `DROP CONTINUOUS
+QUERY` commands. Once you have these details you'll need to set up a
+configuration file. To do so, run the following command:
+
+    cp .env.example .env
+
+Now edit `.env` using your favourite editor and make sure that the various
+variables in this file contain the correct values.
+
+## Usage
+
+Once configured you can set up the continuous queries by running the following
+command:
+
+    bundle exec rake queries
+
+## Structure
+
+Each query resides in its own file in a sub directory (per category) in the
+`queries/` directory. The names of the files should match the name of the
+continuous query, the extension should be `.txt`.

Rakefile

+require 'dotenv'
+require 'influxdb'
+
+Dotenv.load('.env')
+
+CLIENT = InfluxDB::Client.new(
+  ENV.fetch('INFLUX_DATABASE'),
+  host: ENV.fetch('INFLUX_HOST'),
+  username: ENV.fetch('INFLUX_USER'),
+  password: ENV.fetch('INFLUX_PASSWORD'),
+  port: ENV.fetch('INFLUX_PORT').to_i
+)
+
+QUERIES = File.expand_path('../queries', __FILE__)
+
+desc 'Updates the continuous queries in the database'
+task :queries do
+  db = ENV.fetch('INFLUX_DATABASE')
+
+  Dir[File.join(QUERIES, '**', '*.txt')].each do |file|
+    name = File.basename(file, File.extname(file))
+    query = File.read(file).strip
+
+    if query.include?('$DATABASE')
+      query.gsub!('$DATABASE', db)
+    else
+      raise "The query in #{file} must specify the database using `ON $DATABASE`"
+    end
+
+    puts "Creating continuous query #{name}"
+
+    CLIENT.query("DROP CONTINUOUS QUERY #{name} ON #{db}")
+    CLIENT.query(query)
+  end
+end

queries/grape/grape_git_timings_per_action.txt

+CREATE CONTINUOUS QUERY grape_git_timings_per_action ON $DATABASE
+BEGIN
+    SELECT mean("duration") AS duration_mean,
+        percentile("duration", 95) AS duration_95th,
+        percentile("duration", 99) AS duration_99th
+    INTO downsampled.grape_git_timings_per_action
+    FROM "default".rails_method_calls
+    WHERE (action !~ /.+/ OR action =~ /^Grape#/)
+    AND method =~ /^(Rugged|Gitlab::Git)/
+    GROUP BY time(1m), action
+END

queries/grape/grape_markdown_render_timings_overall.txt

+CREATE CONTINUOUS QUERY grape_markdown_render_timings_overall ON $DATABASE
+BEGIN
+    SELECT mean(banzai_cached_render_real_time) AS cached_real_mean,
+        percentile(banzai_cached_render_real_time, 95) AS cached_real_95th,
+        percentile(banzai_cached_render_real_time, 99) AS cached_real_99th,
+        mean(banzai_cached_render_cpu_time) AS cached_cpu_mean,
+        percentile(banzai_cached_render_cpu_time, 95) AS cached_cpu_95th,
+        percentile(banzai_cached_render_cpu_time, 99) AS cached_cpu_99th,
+        sum(banzai_cached_render_call_count) AS cached_call_count,
+        mean(banzai_cacheless_render_real_time) AS cacheless_real_mean,
+        percentile(banzai_cacheless_render_real_time, 95) AS cacheless_real_95th,
+        percentile(banzai_cacheless_render_real_time, 99) AS cacheless_real_99th,
+        mean(banzai_cacheless_render_cpu_time) AS cacheless_cpu_mean,
+        percentile(banzai_cacheless_render_cpu_time, 95) AS cacheless_cpu_95th,
+        percentile(banzai_cacheless_render_cpu_time, 99) AS cacheless_cpu_99th,
+        sum(banzai_cacheless_render_call_count) AS cacheless_call_count
+    INTO downsampled.grape_markdown_render_timings_overall
+    FROM "default".rails_transactions
+    WHERE (action !~ /.+/ OR action =~ /^Grape#/)
+    AND (banzai_cached_render_call_count > 0 OR banzai_cacheless_render_call_count > 0)
+    GROUP BY time(1m)
+END

queries/grape/grape_markdown_render_timings_per_action.txt

+CREATE CONTINUOUS QUERY grape_markdown_render_timings_per_action ON $DATABASE
+BEGIN
+    SELECT mean(banzai_cached_render_real_time) AS cached_real_mean,
+        percentile(banzai_cached_render_real_time, 95) AS cached_real_95th,
+        percentile(banzai_cached_render_real_time, 99) AS cached_real_99th,
+        mean(banzai_cached_render_cpu_time) AS cached_cpu_mean,
+        percentile(banzai_cached_render_cpu_time, 95) AS cached_cpu_95th,
+        percentile(banzai_cached_render_cpu_time, 99) AS cached_cpu_99th,
+        sum(banzai_cached_render_call_count) AS cached_call_count,
+        mean(banzai_cacheless_render_real_time) AS cacheless_real_mean,
+        percentile(banzai_cacheless_render_real_time, 95) AS cacheless_real_95th,
+        percentile(banzai_cacheless_render_real_time, 99) AS cacheless_real_99th,
+        mean(banzai_cacheless_render_cpu_time) AS cacheless_cpu_mean,
+        percentile(banzai_cacheless_render_cpu_time, 95) AS cacheless_cpu_95th,
+        percentile(banzai_cacheless_render_cpu_time, 99) AS cacheless_cpu_99th,
+        sum(banzai_cacheless_render_call_count) AS cacheless_call_count
+    INTO downsampled.grape_markdown_render_timings_per_action
+    FROM "default".rails_transactions
+    WHERE (action !~ /.+/ OR action =~ /^Grape#/)
+    AND (banzai_cached_render_call_count > 0 OR banzai_cacheless_render_call_count > 0)
+    GROUP BY time(1m), action
+END

queries/grape/grape_markdown_timings_overall.txt

+CREATE CONTINUOUS QUERY grape_markdown_timings_overall ON $DATABASE
+BEGIN
+    SELECT mean("duration") AS duration_mean,
+        percentile("duration", 95) AS duration_95th,
+        percentile("duration", 99) AS duration_99th
+    INTO downsampled.grape_markdown_timings_overall
+    FROM "default".rails_method_calls
+    WHERE (action !~ /.+/ OR action =~ /^Grape#/)
+    AND method =~ /^Banzai/
+    GROUP BY time(1m)
+END

queries/grape/grape_method_call_timings_per_action_and_method.txt

+CREATE CONTINUOUS QUERY grape_method_call_timings_per_action_and_method ON $DATABASE
+BEGIN
+    SELECT mean("duration") AS duration_mean,
+        percentile("duration", 95) AS duration_95th,
+        percentile("duration", 99) AS duration_99th,
+        mean(cpu_duration) AS cpu_duration_mean,
+        percentile(cpu_duration, 95) AS cpu_duration_95th,
+        percentile(cpu_duration, 99) AS cpu_duration_99th,
+        mean(call_count) AS call_count_mean,
+        percentile(call_count, 95) AS call_count_95th,
+        percentile(call_count, 99) AS call_count_99th
+    INTO downsampled.grape_method_call_timings_per_action_and_method
+    FROM "default".rails_method_calls
+    WHERE action !~ /.+/ OR action =~ /^Grape#/
+    GROUP BY time(1m), method, action
+END

queries/grape/grape_method_call_timings_per_method.txt

+CREATE CONTINUOUS QUERY grape_method_call_timings_per_method ON $DATABASE
+BEGIN
+    SELECT mean("duration") AS duration_mean,
+        percentile("duration", 95) AS duration_95th,
+        percentile("duration", 99) AS duration_99th,
+        mean(cpu_duration) AS cpu_duration_mean,
+        percentile(cpu_duration, 95) AS cpu_duration_95th,
+        percentile(cpu_duration, 99) AS cpu_duration_99th,
+        mean(call_count) AS call_count_mean,
+        percentile(call_count, 95) AS call_count_95th,
+        percentile(call_count, 99) AS call_count_99th
+    INTO downsampled.grape_method_call_timings_per_method
+    FROM "default".rails_method_calls
+    WHERE action !~ /.+/ OR action =~ /^Grape#/
+    GROUP BY time(1m), method
+END

queries/grape/grape_transaction_counts_overall.txt

+CREATE CONTINUOUS QUERY grape_transaction_counts_overall ON $DATABASE
+BEGIN
+    SELECT count("duration") AS count
+    INTO downsampled.grape_transaction_counts_overall
+    FROM "default".rails_transactions
+    WHERE action !~ /.+/
+    OR action =~ /^Grape#/
+    GROUP BY time(1m)
+END

queries/grape/grape_transaction_counts_per_action.txt

+CREATE CONTINUOUS QUERY grape_transaction_counts_per_action ON $DATABASE
+BEGIN
+    SELECT count("duration") AS count
+    INTO downsampled.grape_transaction_counts_per_action
+    FROM "default".rails_transactions
+    WHERE action !~ /.+/
+    OR action =~ /^Grape#/
+    GROUP BY time(1m), action
+END

queries/grape/grape_transaction_new_redis_connection_counts_overall.txt

+CREATE CONTINUOUS QUERY grape_transaction_new_redis_connection_counts_overall ON $DATABASE
+BEGIN
+    SELECT max(new_redis_connections) AS new_redis_connections_max,
+        mean(new_redis_connections) AS new_redis_connections_mean,
+        percentile(new_redis_connections, 95) AS new_redis_connections_95th,
+        percentile(new_redis_connections, 99) AS new_redis_connections_99th
+    INTO downsampled.grape_transaction_new_redis_connection_counts_overall
+    FROM "default".rails_transactions
+    WHERE action !~ /.+/ OR action =~ /^Grape#/
+    GROUP BY time(1m)
+END

queries/grape/grape_transaction_new_redis_connection_counts_per_action.txt

+CREATE CONTINUOUS QUERY grape_transaction_new_redis_connection_counts_per_action ON $DATABASE
+BEGIN
+    SELECT max(new_redis_connections) AS new_redis_connections_max,
+        mean(new_redis_connections) AS new_redis_connections_mean,
+        percentile(new_redis_connections, 95) AS new_redis_connections_95th,
+        percentile(new_redis_connections, 99) AS new_redis_connections_99th
+    INTO downsampled.grape_transaction_new_redis_connection_counts_per_action
+    FROM "default".rails_transactions
+    WHERE action !~ /.+/
+    OR action =~ /^Grape#/
+    GROUP BY time(1m), action
+END

queries/grape/grape_transaction_sql_counts_overall.txt

+CREATE CONTINUOUS QUERY grape_transaction_sql_counts_overall ON $DATABASE
+BEGIN
+    SELECT max(sql_count) AS sql_count_max,
+        mean(sql_count) AS sql_count_mean,
+        percentile(sql_count, 95) AS sql_count_95th,
+        percentile(sql_count, 99) AS sql_count_99th
+    INTO downsampled.grape_transaction_sql_counts_overall
+    FROM "default".rails_transactions
+    WHERE action !~ /.+/ OR action =~ /^Grape#/
+    GROUP BY time(1m)
+END

queries/grape/grape_transaction_sql_counts_per_action.txt

+CREATE CONTINUOUS QUERY grape_transaction_sql_counts_per_action ON $DATABASE
+BEGIN
+    SELECT max(sql_count) AS sql_count_max,
+        mean(sql_count) AS sql_count_mean,
+        percentile(sql_count, 95) AS sql_count_95th,
+        percentile(sql_count, 99) AS sql_count_99th
+    INTO downsampled.grape_transaction_sql_counts_per_action
+    FROM "default".rails_transactions
+    WHERE action !~ /.+/ OR action =~ /^Grape#/
+    GROUP BY time(1m), action
+END

queries/grape/grape_transaction_timings_overall.txt

+CREATE CONTINUOUS QUERY grape_transaction_timings_overall ON $DATABASE
+BEGIN
+    SELECT mean("duration") AS duration_mean,
+        percentile("duration", 95) AS duration_95th,
+        percentile("duration", 99) AS duration_99th,
+        mean(sql_duration) AS sql_duration_mean,
+        percentile(sql_duration, 95) AS sql_duration_95th,
+        percentile(sql_duration, 99) AS sql_duration_99th,
+        max(sql_duration) AS sql_duration_max,
+        mean(view_duration) AS view_duration_mean,
+        percentile(view_duration, 95) AS view_duration_95th,
+        percentile(view_duration, 99) AS view_duration_99th,
+        mean(cache_read_duration) AS cache_read_duration_mean,
+        percentile(cache_read_duration, 99) AS cache_read_duration_99th,
+        percentile(cache_read_duration, 95) AS cache_read_duration_95th,
+        mean(cache_write_duration) AS cache_write_duration_mean,
+        percentile(cache_write_duration, 99) AS cache_write_duration_99th,
+        percentile(cache_write_duration, 95) AS cache_write_duration_95th,
+        mean(cache_delete_duration) AS cache_delete_duration_mean,
+        percentile(cache_delete_duration, 99) AS cache_delete_duration_99th,
+        percentile(cache_delete_duration, 95) AS cache_delete_duration_95th,
+        mean(cache_exists_duration) AS cache_exists_duration_mean,
+        percentile(cache_exists_duration, 99) AS cache_exists_duration_99th,
+        percentile(cache_exists_duration, 95) AS cache_exists_duration_95th,
+        mean(cache_duration) AS cache_duration_mean,
+        percentile(cache_duration, 99) AS cache_duration_99th,
+        percentile(cache_duration, 95) AS cache_duration_95th,
+        mean(rails_queue_duration) AS rails_queue_duration_mean,
+        percentile(rails_queue_duration, 95) AS rails_queue_duration_95th,
+        percentile(rails_queue_duration, 99) AS rails_queue_duration_99th
+    INTO downsampled.grape_transaction_timings_overall
+    FROM "default".rails_transactions
+    WHERE action !~ /.+/ OR action =~ /^Grape#/
+    GROUP BY time(1m)
+END

queries/grape/grape_transaction_timings_per_action.txt

+CREATE CONTINUOUS QUERY grape_transaction_timings_per_action ON $DATABASE
+BEGIN
+    SELECT mean("duration") AS duration_mean,
+        percentile("duration", 95) AS duration_95th,
+        percentile("duration", 99) AS duration_99th,
+        mean(sql_duration) AS sql_duration_mean,
+        percentile(sql_duration, 95) AS sql_duration_95th,
+        percentile(sql_duration, 99) AS sql_duration_99th,
+        max(sql_duration) AS sql_duration_max,
+        mean(view_duration) AS view_duration_mean,
+        percentile(view_duration, 95) AS view_duration_95th,
+        percentile(view_duration, 99) AS view_duration_99th,
+        mean(cache_read_duration) AS cache_read_duration_mean,
+        percentile(cache_read_duration, 99) AS cache_read_duration_99th,
+        percentile(cache_read_duration, 95) AS cache_read_duration_95th,
+        mean(cache_write_duration) AS cache_write_duration_mean,
+        percentile(cache_write_duration, 99) AS cache_write_duration_99th,
+        percentile(cache_write_duration, 95) AS cache_write_duration_95th,
+        mean(cache_delete_duration) AS cache_delete_duration_mean,
+        percentile(cache_delete_duration, 99) AS cache_delete_duration_99th,
+        percentile(cache_delete_duration, 95) AS cache_delete_duration_95th,
+        mean(cache_exists_duration) AS cache_exists_duration_mean,
+        percentile(cache_exists_duration, 99) AS cache_exists_duration_99th,
+        percentile(cache_exists_duration, 95) AS cache_exists_duration_95th,
+        mean(cache_duration) AS cache_duration_mean,
+        percentile(cache_duration, 99) AS cache_duration_99th,
+        percentile(cache_duration, 95) AS cache_duration_95th,
+        mean(rails_queue_duration) AS rails_queue_duration_mean,
+        percentile(rails_queue_duration, 95) AS rails_queue_duration_95th,
+        percentile(rails_queue_duration, 99) AS rails_queue_duration_99th
+    INTO downsampled.grape_transaction_timings_per_action
+    FROM "default".rails_transactions
+    WHERE action !~ /.+/ OR action =~ /^Grape#/
+    GROUP BY time(1m), action
+END