Update postgres_exporter metrics

Reduce volume of metrics produced by the postgres_exporter.
* Disable `pg_stat_user_tables` metrics by default.
* Update `pg_stat_user_tables` query to match GitLab.com production[0]

Saves about 9500 metrics on a default omnibus install. With the
per-table stats enabled, saves about 2800 metrics.

Overall, this saves about 35% of the overall metrics load (700MiB).

https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5654

[0]: https://gitlab.com/gitlab-cookbooks/gitlab-exporters/-/merge_requests/194



Signed-off-by: Ben Kochie <bjk@gitlab.com>

changelogs/unreleased/bjk-pg_stat_cleanup.yml

+---
+title: Update postgres_exporter metrics
+merge_request: 4586
+author:
+type: performance

files/gitlab-config-template/gitlab.rb.template

@@ -1839,6 +1839,7 @@ external_url 'GENERATED_EXTERNAL_URL'
 #   'SSL_CERT_DIR' => "/opt/gitlab/embedded/ssl/certs/"
 # }
 # postgres_exporter['sslmode'] = nil
+# postgres_exporter['per_table_stats'] = false
 ################################################################################
 ## Prometheus PgBouncer exporter (EE only)

files/gitlab-cookbooks/monitoring/attributes/default.rb

@@ -79,6 +79,7 @@ default['monitoring']['postgres-exporter']['env'] = {
   'SSL_CERT_DIR' => "#{node['package']['install-dir']}/embedded/ssl/certs/"
 }
 default['monitoring']['postgres-exporter']['sslmode'] = nil
+default['monitoring']['postgres-exporter']['per_table_stats'] = false
 ####
 # Gitlab exporter

files/gitlab-cookbooks/monitoring/recipes/postgres-exporter.rb

@@ -65,7 +65,7 @@ runit_service 'postgres-exporter' do
 end
 template File.join(postgres_exporter_dir, 'queries.yaml') do
-  source 'postgres-queries.yaml'
+  source 'postgres-queries.yaml.erb'
   owner postgresql_user
   mode '0644'
   notifies :restart, 'runit_service[postgres-exporter]'

files/gitlab-cookbooks/monitoring/templates/postgres-queries.yaml → files/gitlab-cookbooks/monitoring/templates/postgres-queries.yaml.erb

@@ -15,8 +15,29 @@ pg_postmaster:
         usage: "GAUGE"
         description: "Time at which postmaster started"
+<% if node['monitoring']['postgres-exporter']['per_table_stats'] -%>
 pg_stat_user_tables:
-  query: "SELECT current_database() datname, schemaname, relname, seq_scan, seq_tup_read, idx_scan, idx_tup_fetch, n_tup_ins, n_tup_upd, n_tup_del, n_tup_hot_upd, n_live_tup, n_dead_tup, n_mod_since_analyze, COALESCE(last_vacuum, '1970-01-01Z'), COALESCE(last_vacuum, '1970-01-01Z') as last_vacuum, COALESCE(last_autovacuum, '1970-01-01Z') as last_autovacuum, COALESCE(last_analyze, '1970-01-01Z') as last_analyze, COALESCE(last_autoanalyze, '1970-01-01Z') as last_autoanalyze, vacuum_count, autovacuum_count, analyze_count, autoanalyze_count FROM pg_stat_user_tables"
+  query: |
+    SELECT
+      current_database() datname,
+      schemaname,
+      relname,
+      seq_scan,
+      seq_tup_read,
+      idx_scan,
+      idx_tup_fetch,
+      n_tup_ins,
+      n_tup_upd,
+      n_tup_del,
+      n_tup_hot_upd,
+      n_live_tup,
+      n_dead_tup,
+      GREATEST(last_autovacuum, last_vacuum, '1970-01-01Z') as last_vacuum,
+      GREATEST(last_autoanalyze, last_analyze, '1970-01-01Z') as last_analyze,
+      (vacuum_count + autovacuum_count) as vacuum_count,
+      (analyze_count + autoanalyze_count) as analyze_count
+    FROM
+      pg_stat_user_tables
   metrics:
     - datname:
         usage: "LABEL"
@@ -57,34 +78,20 @@ pg_stat_user_tables:
     - n_dead_tup:
         usage: "GAUGE"
         description: "Estimated number of dead rows"
-    - n_mod_since_analyze:
-        usage: "GAUGE"
-        description: "Estimated number of rows changed since last analyze"
     - last_vacuum:
         usage: "GAUGE"
-        description: "Last time at which this table was manually vacuumed (not counting VACUUM FULL)"
-    - last_autovacuum:
-        usage: "GAUGE"
-        description: "Last time at which this table was vacuumed by the autovacuum daemon"
+        description: "Last time at which this table was vacuumed (not counting VACUUM FULL)"
     - last_analyze:
         usage: "GAUGE"
-        description: "Last time at which this table was manually analyzed"
-    - last_autoanalyze:
-        usage: "GAUGE"
-        description: "Last time at which this table was analyzed by the autovacuum daemon"
+        description: "Last time at which this table was analyzed"
     - vacuum_count:
         usage: "COUNTER"
-        description: "Number of times this table has been manually vacuumed (not counting VACUUM FULL)"
-    - autovacuum_count:
-        usage: "COUNTER"
-        description: "Number of times this table has been vacuumed by the autovacuum daemon"
+        description: "Number of times this table has been vacuumed"
     - analyze_count:
         usage: "COUNTER"
-        description: "Number of times this table has been manually analyzed"
-    - autoanalyze_count:
-        usage: "COUNTER"
-        description: "Number of times this table has been analyzed by the autovacuum daemon"
+        description: "Number of times this table has been analyzed"
+<% end -%>
 pg_total_relation_size:
   query: |
     SELECT relnamespace::regnamespace as schemaname,

spec/chef/monitoring/postgres_exporter_spec.rb

@@ -97,7 +97,10 @@ RSpec.describe 'monitoring::postgres-exporter' do
     it 'creates the queries.yaml file' do
       expect(chef_run).to render_file('/var/opt/gitlab/postgres-exporter/queries.yaml')
-        .with_content(/pg_replication:/)
+        .with_content { |content|
+          expect(content).to match(/pg_replication:/)
+          expect(content).not_to match(/pg_stat_user_table:/)
+        }
     end
     it 'creates default set of directories' do
@@ -153,6 +156,7 @@ RSpec.describe 'monitoring::postgres-exporter' do
           },
           listen_address: 'localhost:9700',
           enable: true,
+          per_table_stats: true,
           sslmode: 'require',
           env: {
             'USER_SETTING' => 'asdf1234'
@@ -168,6 +172,11 @@ RSpec.describe 'monitoring::postgres-exporter' do
         .with_content(/some.flag=foo/)
     end
+    it 'creates the queries.yaml file' do
+      expect(chef_run).to render_file('/var/opt/gitlab/postgres-exporter/queries.yaml')
+        .with_content(/pg_stat_user_tables:/)
+    end
+
     it 'creates necessary env variable files' do
       expect(chef_run).to create_env_dir('/opt/gitlab/etc/postgres-exporter/env').with_variables(
         default_vars.merge(