Add database reindexing cronjob

Relates to https://gitlab.com/gitlab-org/gitlab/-/issues/249662

changelogs/unreleased/ab-reindexing-tasks.yml

+---
+title: Add database reindexing cronjob
+merge_request: 4602
+author:
+type: added

doc/settings/database.md

@@ -840,6 +840,39 @@ or not specifying the setting at all disables the timeout.
 NOTE: **Note:**
 After changing timeout settings, please run `gitlab-ctl reconfigure` to update the configuration.
+## Automatic database reindexing
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/249662) in GitLab 13.5..
+
+Recreates database indexes in the background (called "reindexing"). This can
+be used to remove bloated space that has accumulated in indexes and helps to maintain healthy and
+efficient indexes.
+
+NOTE: **Note:**
+This is an experimental feature that is not enabled by default yet.
+
+The reindexing task can be started regularly through a cronjob. In order to configure the cronjob,
+`gitlab_rails['database_reindexing']['enable']` should be set to `true`.
+
+NOTE: **Note:**
+In a multi-node environment, this feature should only be enabled on an application host.
+The reindexing process cannot go through PgBouncer, it has to have a direct database connection.
+
+By default, this starts the cronjob every hour during weekends (likely a low-traffic time) only.
+
+Note you can change the schedule by refining the following settings:
+
+```shell
+gitlab_rails['database_reindexing']['hour'] = '*'
+gitlab_rails['database_reindexing']['minute'] = 0
+gitlab_rails['database_reindexing']['month'] = '*'
+gitlab_rails['database_reindexing']['day_of_month'] = '*'
+gitlab_rails['database_reindexing']['day_of_week'] = '0,6'
+```
+
+NOTE: **Note:**
+After changing these settings, please run `gitlab-ctl reconfigure` to update the configuration.
+
 ## Packaged PostgreSQL deployed in an HA/Geo Cluster
 ### Upgrading a GitLab HA cluster

files/gitlab-cookbooks/gitlab/attributes/default.rb

@@ -389,6 +389,15 @@ default['gitlab']['gitlab-rails']['db_statement_timeout'] = nil
 default['gitlab']['gitlab-rails']['db_fdw'] = nil
 default['gitlab']['gitlab-rails']['db_connect_timeout'] = nil
+# Automatic Database Reindexing
+# See https://docs.gitlab.com/omnibus/settings/database.html#automatic-database-reindexing
+default['gitlab']['gitlab-rails']['database_reindexing']['enable'] = false
+default['gitlab']['gitlab-rails']['database_reindexing']['hour'] = '*'
+default['gitlab']['gitlab-rails']['database_reindexing']['minute'] = 0
+default['gitlab']['gitlab-rails']['database_reindexing']['month'] = '*'
+default['gitlab']['gitlab-rails']['database_reindexing']['day_of_month'] = '*'
+default['gitlab']['gitlab-rails']['database_reindexing']['day_of_week'] = '0,6'
+
 default['gitlab']['gitlab-rails']['redis_host'] = "127.0.0.1"
 default['gitlab']['gitlab-rails']['redis_port'] = nil
 default['gitlab']['gitlab-rails']['redis_ssl'] = false

files/gitlab-cookbooks/gitlab/recipes/database_reindexing_disable.rb

+#
+# Copyright:: Copyright (c) 2020 GitLab B.V.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+crond_job 'database-reindexing' do
+  action :delete
+end

files/gitlab-cookbooks/gitlab/recipes/database_reindexing_enable.rb

+#
+# Copyright:: Copyright (c) 2020 GitLab B.V.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+reindexing = node['gitlab']['gitlab-rails']['database_reindexing']
+
+include_recipe "crond::enable"
+
+crond_job 'database-reindexing' do
+  user "root"
+  hour reindexing['hour']
+  minute reindexing['minute']
+  month reindexing['month']
+  day_of_month reindexing['day_of_month']
+  day_of_week reindexing['day_of_week']
+
+  command "/opt/gitlab/bin/gitlab-rake gitlab:db:reindex"
+end

files/gitlab-cookbooks/gitlab/recipes/default.rb

@@ -184,6 +184,12 @@ else
   include_recipe 'letsencrypt::disable'
 end
+if node['gitlab']['gitlab-rails']['database_reindexing']['enable']
+  include_recipe 'gitlab::database_reindexing_enable'
+else
+  include_recipe 'gitlab::database_reindexing_disable'
+end
+
 OmnibusHelper.is_deprecated_os?
 # Report on any deprecations we encountered at the end of the run

files/gitlab-cookbooks/letsencrypt/recipes/enable.rb

@@ -50,8 +50,6 @@ else
   crond_job 'letsencrypt-renew' do
     action :delete
   end
-
-  include_recipe "crond::disable"
 end
 ruby_block 'display_le_message' do

spec/chef/recipes/database_reindexing_disable_spec.rb

+require 'chef_helper'
+
+RSpec.describe 'gitlab::database-reindexing' do
+  let(:chef_run) { converge_config('gitlab::database_reindexing_disable') }
+
+  before do
+    allow(Gitlab).to receive(:[]).and_call_original
+  end
+
+  it 'removes the database-reindexing cronjob' do
+    expect(chef_run).to delete_crond_job('database-reindexing')
+  end
+end

spec/chef/recipes/database_reindexing_enable_spec.rb

+require 'chef_helper'
+
+RSpec.describe 'gitlab::database-reindexing' do
+  let(:chef_run) { converge_config('gitlab::database_reindexing_enable') }
+
+  before do
+    allow(Gitlab).to receive(:[]).and_call_original
+  end
+
+  context 'with defaults' do
+    before do
+      stub_gitlab_rb(gitlab_rails: { database_reindexing: { enable: true } })
+    end
+
+    it 'enables crond' do
+      expect(chef_run).to include_recipe('crond::enable')
+    end
+
+    it 'adds a crond_job with default schedule' do
+      expect(chef_run).to create_crond_job('database-reindexing').with(
+        user: "root",
+        hour: '*',
+        minute: 0,
+        month: '*',
+        day_of_month: '*',
+        day_of_week: '0,6',
+        command: "/opt/gitlab/bin/gitlab-rake gitlab:db:reindex"
+      )
+    end
+  end
+
+  context 'with specific schedule' do
+    let(:config) do
+      {
+        enable: true,
+        hour: 10,
+        minute: 5,
+        month: 3,
+        day_of_month: 2,
+        day_of_week: 1
+      }
+    end
+
+    it 'adds a crond_job with the configured schedule' do
+      stub_gitlab_rb(gitlab_rails: { database_reindexing: config })
+
+      expect(chef_run).to create_crond_job('database-reindexing').with(
+        user: "root",
+        hour: 10,
+        minute: 5,
+        month: 3,
+        day_of_week: 1,
+        command: "/opt/gitlab/bin/gitlab-rake gitlab:db:reindex"
+      )
+    end
+  end
+end

spec/chef/recipes/letsencrypt_spec.rb

@@ -151,11 +151,6 @@ server {
           allow_any_instance_of(PgHelper).to receive(:is_standby?).and_return false
         end
-        it 'disables crond' do
-          expect(chef_run).to include_recipe('crond::disable')
-          expect(chef_run).not_to include_recipe('crond::enable')
-        end
-
         it 'removes the letsencrypt-renew cronjob' do
           expect(chef_run).to delete_crond_job('letsencrypt-renew')
         end