Merge branch 'repopulate_site_statistics' into 'master'

Fix SiteStatistics wikis_count

See merge request gitlab-org/gitlab-ce!21030

app/models/site_statistic.rb

@@ -49,7 +49,7 @@ class SiteStatistic < ActiveRecord::Base
   #
   # @return [SiteStatistic] record with tracked information
   def self.fetch
-    SiteStatistic.transaction(requires_new: true) do
+    transaction(requires_new: true) do
       SiteStatistic.first_or_create!
     end
   rescue ActiveRecord::RecordNotUnique

changelogs/unreleased/repopulate_site_statistics.yml

+---
+title: Migrate NULL wiki_access_level to correct number so we count active wikis correctly
+merge_request: 21030
+author:
+type: changed

db/post_migrate/20180809195358_migrate_null_wiki_access_levels.rb

+# frozen_string_literal: true
+
+class MigrateNullWikiAccessLevels < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  class ProjectFeature < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'project_features'
+  end
+
+  def up
+    ProjectFeature.where(wiki_access_level: nil).each_batch do |relation|
+      relation.update_all(wiki_access_level: 20)
+    end
+
+    # We need to re-count wikis as previous attempt was not considering the NULLs.
+    transaction do
+      execute('SET LOCAL statement_timeout TO 0') if Gitlab::Database.postgresql? # see https://gitlab.com/gitlab-org/gitlab-ce/issues/48967
+
+      execute("UPDATE site_statistics SET wikis_count = (SELECT COUNT(*) FROM project_features WHERE wiki_access_level != 0)")
+    end
+  end
+
+  def down
+    # there is no way to rollback this change, there are no downsides in keeping migrated data.
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema.define(version: 20180808162000) do
+ActiveRecord::Schema.define(version: 20180809195358) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"

lib/tasks/gitlab/site_statistics.rake

+namespace :gitlab do
+  desc "GitLab | Refresh Site Statistics counters"
+  task refresh_site_statistics: :environment do
+    puts 'Updating Site Statistics counters: '
+
+    print '* Repositories... '
+    SiteStatistic.transaction do
+      # see https://gitlab.com/gitlab-org/gitlab-ce/issues/48967
+      ActiveRecord::Base.connection.execute('SET LOCAL statement_timeout TO 0') if Gitlab::Database.postgresql?
+      SiteStatistic.update_all('repositories_count = (SELECT COUNT(*) FROM projects)')
+    end
+    puts 'OK!'.color(:green)
+
+    print '* Wikis... '
+    SiteStatistic.transaction do
+      # see https://gitlab.com/gitlab-org/gitlab-ce/issues/48967
+      ActiveRecord::Base.connection.execute('SET LOCAL statement_timeout TO 0') if Gitlab::Database.postgresql?
+      SiteStatistic.update_all('wikis_count = (SELECT COUNT(*) FROM project_features WHERE wiki_access_level != 0)')
+    end
+    puts 'OK!'.color(:green)
+    puts
+  end
+end

spec/lib/gitlab/bare_repository_import/importer_spec.rb

@@ -10,9 +10,6 @@ describe Gitlab::BareRepositoryImport::Importer, :seed_helper do
   subject(:importer) { described_class.new(admin, bare_repository) }
   before do
-    @rainbow = Rainbow.enabled
-    Rainbow.enabled = false
-
     allow(described_class).to receive(:log)
   end
@@ -20,7 +17,6 @@ describe Gitlab::BareRepositoryImport::Importer, :seed_helper do
     FileUtils.rm_rf(base_dir)
     TestEnv.clean_test_path
     ensure_seeds
-    Rainbow.enabled = @rainbow
   end
   shared_examples 'importing a repository' do

spec/lib/system_check/simple_executor_spec.rb

@@ -98,15 +98,6 @@ describe SystemCheck::SimpleExecutor do
     end
   end
-  before do
-    @rainbow = Rainbow.enabled
-    Rainbow.enabled = false
-  end
-
-  after do
-    Rainbow.enabled = @rainbow
-  end
-
   describe '#component' do
     it 'returns stored component name' do
       expect(subject.component).to eq('Test')

spec/migrations/migrate_null_wiki_access_levels_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20180809195358_migrate_null_wiki_access_levels.rb')
+
+describe MigrateNullWikiAccessLevels, :migration do
+  let(:namespaces) { table('namespaces') }
+  let(:projects) { table(:projects) }
+  let(:project_features) { table(:project_features) }
+  let(:migration) { described_class.new }
+
+  before do
+    namespace = namespaces.create(name: 'foo', path: 'foo')
+
+    projects.create!(id: 1, name: 'gitlab1', path: 'gitlab1', namespace_id: namespace.id)
+    projects.create!(id: 2, name: 'gitlab2', path: 'gitlab2', namespace_id: namespace.id)
+    projects.create!(id: 3, name: 'gitlab3', path: 'gitlab3', namespace_id: namespace.id)
+
+    project_features.create!(id: 1, project_id: 1, wiki_access_level: nil)
+    project_features.create!(id: 2, project_id: 2, wiki_access_level: 10)
+    project_features.create!(id: 3, project_id: 3, wiki_access_level: 20)
+  end
+
+  describe '#up' do
+    it 'migrates existing project_features with wiki_access_level NULL to 20' do
+      expect { migration.up }.to change { project_features.where(wiki_access_level: 20).count }.by(1)
+    end
+  end
+end

spec/spec_helper.rb

@@ -29,6 +29,7 @@ end
 # require rainbow gem String monkeypatch, so we can test SystemChecks
 require 'rainbow/ext/string'
+Rainbow.enabled = false
 # Requires supporting ruby files with custom matchers and macros, etc,
 # in spec/support/ and its subdirectories.

spec/tasks/gitlab/site_statistics_rake_spec.rb

+# frozen_string_literal: true
+require 'rake_helper'
+
+describe 'rake gitlab:refresh_site_statistics' do
+  before do
+    Rake.application.rake_require 'tasks/gitlab/site_statistics'
+
+    create(:project)
+    SiteStatistic.fetch.update(repositories_count: 0, wikis_count: 0)
+  end
+
+  let(:task) { 'gitlab:refresh_site_statistics' }
+
+  it 'recalculates existing counters' do
+    run_rake_task(task)
+
+    expect(SiteStatistic.fetch.repositories_count).to eq(1)
+    expect(SiteStatistic.fetch.wikis_count).to eq(1)
+  end
+
+  it 'displays message listing counters' do
+    expect { run_rake_task(task) }.to output(/Updating Site Statistics counters:.* Repositories\.\.\. OK!.* Wikis\.\.\. OK!/m).to_stdout
+  end
+end