Add more storage statistics

This adds counters for build artifacts and LFS objects, and moves
the preexisting repository_size and commit_count from the projects
table into a new project_statistics table.

The counters are displayed in the administration area for projects
and groups, and also available through the API for admins (on */all)
and normal users (on */owned)

The statistics are updated through ProjectCacheWorker, which can now
do more granular updates with the new :statistics argument.

spec/models/ci/build_spec.rb

@@ -85,4 +85,30 @@ describe Ci::Build, models: true do
       it { expect(build.trace_file_path).to eq(build.old_path_to_trace) }
     end
   end
+
+  describe '#update_project_statistics' do
+    let!(:build) { create(:ci_build, artifacts_size: 23) }
+
+    it 'updates project statistics when the artifact size changes' do
+      expect(ProjectCacheWorker).to receive(:perform_async)
+        .with(build.project_id, [], [:build_artifacts_size])
+
+      build.artifacts_size = 42
+      build.save!
+    end
+
+    it 'does not update project statistics when the artifact size stays the same' do
+      expect(ProjectCacheWorker).not_to receive(:perform_async)
+
+      build.name = 'changed'
+      build.save!
+    end
+
+    it 'updates project statistics when the build is destroyed' do
+      expect(ProjectCacheWorker).to receive(:perform_async)
+        .with(build.project_id, [], [:build_artifacts_size])
+
+      build.destroy
+    end
+  end
 end

spec/models/lfs_objects_project_spec.rb

+require 'spec_helper'
+
+describe LfsObjectsProject, models: true do
+  subject { create(:lfs_objects_project, project: project) }
+  let(:project) { create(:empty_project) }
+
+  describe 'associations' do
+    it { is_expected.to belong_to(:project) }
+    it { is_expected.to belong_to(:lfs_object) }
+  end
+
+  describe 'validation' do
+    it { is_expected.to validate_presence_of(:lfs_object_id) }
+    it { is_expected.to validate_uniqueness_of(:lfs_object_id).scoped_to(:project_id).with_message("already exists in project") }
+
+    it { is_expected.to validate_presence_of(:project_id) }
+  end
+
+  describe '#update_project_statistics' do
+    it 'updates project statistics when the object is added' do
+      expect(ProjectCacheWorker).to receive(:perform_async)
+        .with(project.id, [], [:lfs_objects_size])
+
+      subject.save!
+    end
+
+    it 'updates project statistics when the object is removed' do
+      subject.save!
+
+      expect(ProjectCacheWorker).to receive(:perform_async)
+        .with(project.id, [], [:lfs_objects_size])
+
+      subject.destroy
+    end
+  end
+end

spec/models/namespace_spec.rb

@@ -4,6 +4,7 @@ describe Namespace, models: true do
   let!(:namespace) { create(:namespace) }
   it { is_expected.to have_many :projects }
+  it { is_expected.to have_many :project_statistics }
   it { is_expected.to validate_presence_of(:name) }
   it { is_expected.to validate_uniqueness_of(:name).scoped_to(:parent_id) }
@@ -57,6 +58,50 @@ describe Namespace, models: true do
     end
   end
+  describe '.with_statistics' do
+    let(:namespace) { create :namespace }
+
+    let(:project1) do
+      create(:empty_project,
+             namespace: namespace,
+             statistics: build(:project_statistics,
+                               storage_size:         606,
+                               repository_size:      101,
+                               lfs_objects_size:     202,
+                               build_artifacts_size: 303))
+    end
+
+    let(:project2) do
+      create(:empty_project,
+             namespace: namespace,
+             statistics: build(:project_statistics,
+                               storage_size:         60,
+                               repository_size:      10,
+                               lfs_objects_size:     20,
+                               build_artifacts_size: 30))
+    end
+
+    it "sums all project storage counters in the namespace" do
+      project1
+      project2
+      statistics = Namespace.with_statistics.find(namespace.id)
+
+      expect(statistics.storage_size).to eq 666
+      expect(statistics.repository_size).to eq 111
+      expect(statistics.lfs_objects_size).to eq 222
+      expect(statistics.build_artifacts_size).to eq 333
+    end
+
+    it "correctly handles namespaces without projects" do
+      statistics = Namespace.with_statistics.find(namespace.id)
+
+      expect(statistics.storage_size).to eq 0
+      expect(statistics.repository_size).to eq 0
+      expect(statistics.lfs_objects_size).to eq 0
+      expect(statistics.build_artifacts_size).to eq 0
+    end
+  end
+
   describe '#move_dir' do
     before do
       @namespace = create :namespace

spec/models/project_spec.rb

@@ -49,6 +49,7 @@ describe Project, models: true do
     it { is_expected.to have_one(:gitlab_issue_tracker_service).dependent(:destroy) }
     it { is_expected.to have_one(:external_wiki_service).dependent(:destroy) }
     it { is_expected.to have_one(:project_feature).dependent(:destroy) }
+    it { is_expected.to have_one(:statistics).class_name('ProjectStatistics').dependent(:delete) }
     it { is_expected.to have_one(:import_data).class_name('ProjectImportData').dependent(:destroy) }
     it { is_expected.to have_one(:last_event).class_name('Event') }
     it { is_expected.to have_one(:forked_from_project).through(:forked_project_link) }
@@ -1729,6 +1730,26 @@ describe Project, models: true do
     end
   end
+  describe '#update_project_statistics' do
+    let(:project) { create(:empty_project) }
+
+    it "is called after creation" do
+      expect(project.statistics).to be_a ProjectStatistics
+      expect(project.statistics).to be_persisted
+    end
+
+    it "copies the namespace_id" do
+      expect(project.statistics.namespace_id).to eq project.namespace_id
+    end
+
+    it "updates the namespace_id when changed" do
+      namespace = create(:namespace)
+      project.update(namespace: namespace)
+
+      expect(project.statistics.namespace_id).to eq namespace.id
+    end
+  end
+
   def enable_lfs
     allow(Gitlab.config.lfs).to receive(:enabled).and_return(true)
   end

spec/models/project_statistics_spec.rb

+require 'rails_helper'
+
+describe ProjectStatistics, models: true do
+  let(:project) { create :empty_project }
+  let(:statistics) { project.statistics }
+
+  describe 'constants' do
+    describe 'STORAGE_COLUMNS' do
+      it 'is an array of symbols' do
+        expect(described_class::STORAGE_COLUMNS).to be_kind_of Array
+        expect(described_class::STORAGE_COLUMNS.map(&:class).uniq).to eq [Symbol]
+      end
+    end
+
+    describe 'STATISTICS_COLUMNS' do
+      it 'is an array of symbols' do
+        expect(described_class::STATISTICS_COLUMNS).to be_kind_of Array
+        expect(described_class::STATISTICS_COLUMNS.map(&:class).uniq).to eq [Symbol]
+      end
+
+      it 'includes all storage columns' do
+        expect(described_class::STATISTICS_COLUMNS & described_class::STORAGE_COLUMNS).to eq described_class::STORAGE_COLUMNS
+      end
+    end
+  end
+
+  describe 'associations' do
+    it { is_expected.to belong_to(:project) }
+    it { is_expected.to belong_to(:namespace) }
+  end
+
+  describe 'statistics columns' do
+    it "support values up to 8 exabytes" do
+      statistics.update!(
+        commit_count: 8.exabytes - 1,
+        repository_size: 2.exabytes,
+        lfs_objects_size: 2.exabytes,
+        build_artifacts_size: 4.exabytes - 1,
+      )
+
+      statistics.reload
+
+      expect(statistics.commit_count).to eq(8.exabytes - 1)
+      expect(statistics.repository_size).to eq(2.exabytes)
+      expect(statistics.lfs_objects_size).to eq(2.exabytes)
+      expect(statistics.build_artifacts_size).to eq(4.exabytes - 1)
+      expect(statistics.storage_size).to eq(8.exabytes - 1)
+    end
+  end
+
+  describe '#total_repository_size' do
+    it "sums repository and LFS object size" do
+      statistics.repository_size = 2
+      statistics.lfs_objects_size = 3
+      statistics.build_artifacts_size = 4
+
+      expect(statistics.total_repository_size).to eq 5
+    end
+  end
+
+  describe '#refresh!' do
+    before do
+      allow(statistics).to receive(:update_commit_count)
+      allow(statistics).to receive(:update_repository_size)
+      allow(statistics).to receive(:update_lfs_objects_size)
+      allow(statistics).to receive(:update_build_artifacts_size)
+      allow(statistics).to receive(:update_storage_size)
+    end
+
+    context "without arguments" do
+      before do
+        statistics.refresh!
+      end
+
+      it "sums all counters" do
+        expect(statistics).to have_received(:update_commit_count)
+        expect(statistics).to have_received(:update_repository_size)
+        expect(statistics).to have_received(:update_lfs_objects_size)
+        expect(statistics).to have_received(:update_build_artifacts_size)
+      end
+    end
+
+    context "when passing an only: argument" do
+      before do
+        statistics.refresh! only: [:lfs_objects_size]
+      end
+
+      it "only updates the given columns" do
+        expect(statistics).to have_received(:update_lfs_objects_size)
+        expect(statistics).not_to have_received(:update_commit_count)
+        expect(statistics).not_to have_received(:update_repository_size)
+        expect(statistics).not_to have_received(:update_build_artifacts_size)
+      end
+    end
+  end
+
+  describe '#update_commit_count' do
+    before do
+      allow(project.repository).to receive(:commit_count).and_return(23)
+      statistics.update_commit_count
+    end
+
+    it "stores the number of commits in the repository" do
+      expect(statistics.commit_count).to eq 23
+    end
+  end
+
+  describe '#update_repository_size' do
+    before do
+      allow(project.repository).to receive(:size).and_return(12.megabytes)
+      statistics.update_repository_size
+    end
+
+    it "stores the size of the repository" do
+      expect(statistics.repository_size).to eq 12.megabytes
+    end
+  end
+
+  describe '#update_lfs_objects_size' do
+    let!(:lfs_object1) { create(:lfs_object, size: 23.megabytes) }
+    let!(:lfs_object2) { create(:lfs_object, size: 34.megabytes) }
+    let!(:lfs_objects_project1) { create(:lfs_objects_project, project: project, lfs_object: lfs_object1) }
+    let!(:lfs_objects_project2) { create(:lfs_objects_project, project: project, lfs_object: lfs_object2) }
+
+    before do
+      statistics.update_lfs_objects_size
+    end
+
+    it "stores the size of related LFS objects" do
+      expect(statistics.lfs_objects_size).to eq 57.megabytes
+    end
+  end
+
+  describe '#update_build_artifacts_size' do
+    let!(:pipeline) { create(:ci_pipeline, project: project) }
+    let!(:build1) { create(:ci_build, pipeline: pipeline, artifacts_size: 45.megabytes) }
+    let!(:build2) { create(:ci_build, pipeline: pipeline, artifacts_size: 56.megabytes) }
+
+    before do
+      statistics.update_build_artifacts_size
+    end
+
+    it "stores the size of related build artifacts" do
+      expect(statistics.build_artifacts_size).to eq 101.megabytes
+    end
+  end
+
+  describe '#update_storage_size' do
+    it "sums all storage counters" do
+      statistics.update!(
+        repository_size: 2,
+        lfs_objects_size: 3,
+      )
+
+      statistics.reload
+
+      expect(statistics.storage_size).to eq 5
+    end
+  end
+end

spec/requests/api/groups_spec.rb

@@ -35,6 +35,14 @@ describe API::Groups, api: true  do
         expect(json_response.length).to eq(1)
         expect(json_response.first['name']).to eq(group1.name)
       end
+
+      it "does not include statistics" do
+        get api("/groups", user1), statistics: true
+
+        expect(response).to have_http_status(200)
+        expect(json_response).to be_an Array
+        expect(json_response.first).not_to include 'statistics'
+      end
     end
     context "when authenticated as admin" do
@@ -44,6 +52,31 @@ describe API::Groups, api: true  do
         expect(json_response).to be_an Array
         expect(json_response.length).to eq(2)
       end
+
+      it "does not include statistics by default" do
+        get api("/groups", admin)
+
+        expect(response).to have_http_status(200)
+        expect(json_response).to be_an Array
+        expect(json_response.first).not_to include('statistics')
+      end
+
+      it "includes statistics if requested" do
+        attributes = {
+          storage_size: 702,
+          repository_size: 123,
+          lfs_objects_size: 234,
+          build_artifacts_size: 345,
+        }
+
+        project1.statistics.update!(attributes)
+
+        get api("/groups", admin), statistics: true
+
+        expect(response).to have_http_status(200)
+        expect(json_response).to be_an Array
+        expect(json_response.first['statistics']).to eq attributes.stringify_keys
+      end
     end
     context "when using skip_groups in request" do

spec/requests/api/projects_spec.rb

@@ -49,7 +49,7 @@ describe API::Projects, api: true  do
       end
     end
-    context 'when authenticated' do
+    context 'when authenticated as regular user' do
       it 'returns an array of projects' do
         get api('/projects', user)
         expect(response).to have_http_status(200)
@@ -172,6 +172,22 @@ describe API::Projects, api: true  do
           end
         end
       end
+
+      it "does not include statistics by default" do
+        get api('/projects/all', admin)
+
+        expect(response).to have_http_status(200)
+        expect(json_response).to be_an Array
+        expect(json_response.first).not_to include('statistics')
+      end
+
+      it "includes statistics if requested" do
+        get api('/projects/all', admin), statistics: true
+
+        expect(response).to have_http_status(200)
+        expect(json_response).to be_an Array
+        expect(json_response.first).to include 'statistics'
+      end
     end
   end
@@ -196,6 +212,32 @@ describe API::Projects, api: true  do
         expect(json_response.first['name']).to eq(project4.name)
         expect(json_response.first['owner']['username']).to eq(user4.username)
       end
+
+      it "does not include statistics by default" do
+        get api('/projects/owned', user4)
+
+        expect(response).to have_http_status(200)
+        expect(json_response).to be_an Array
+        expect(json_response.first).not_to include('statistics')
+      end
+
+      it "includes statistics if requested" do
+        attributes = {
+          commit_count: 23,
+          storage_size: 702,
+          repository_size: 123,
+          lfs_objects_size: 234,
+          build_artifacts_size: 345,
+        }
+
+        project4.statistics.update!(attributes)
+
+        get api('/projects/owned', user4), statistics: true
+
+        expect(response).to have_http_status(200)
+        expect(json_response).to be_an Array
+        expect(json_response.first['statistics']).to eq attributes.stringify_keys
+      end
     end
   end

spec/services/git_push_service_spec.rb

@@ -583,7 +583,7 @@ describe GitPushService, services: true do
         service.push_commits = [commit]
         expect(ProjectCacheWorker).to receive(:perform_async).
-          with(project.id, %i(readme))
+          with(project.id, %i(readme), %i(commit_count repository_size))
         service.update_caches
       end
@@ -596,7 +596,7 @@ describe GitPushService, services: true do
       it 'does not flush any conditional caches' do
         expect(ProjectCacheWorker).to receive(:perform_async).
-          with(project.id, []).
+          with(project.id, [], %i(commit_count repository_size)).
           and_call_original
         service.update_caches

spec/workers/project_cache_worker_spec.rb

 require 'spec_helper'
 describe ProjectCacheWorker do
-  let(:project) { create(:project) }
   let(:worker) { described_class.new }
+  let(:project) { create(:project) }
+  let(:statistics) { project.statistics }
   describe '#perform' do
     before do
@@ -12,7 +13,7 @@ describe ProjectCacheWorker do
     context 'with a non-existing project' do
       it 'does nothing' do
-        expect(worker).not_to receive(:update_repository_size)
+        expect(worker).not_to receive(:update_statistics)
         worker.perform(-1)
       end
@@ -22,24 +23,19 @@ describe ProjectCacheWorker do
       it 'does nothing' do
         allow_any_instance_of(Repository).to receive(:exists?).and_return(false)
-        expect(worker).not_to receive(:update_repository_size)
+        expect(worker).not_to receive(:update_statistics)
         worker.perform(project.id)
       end
     end
     context 'with an existing project' do
-      it 'updates the repository size' do
-        expect(worker).to receive(:update_repository_size).and_call_original
-
-        worker.perform(project.id)
-      end
-
-      it 'updates the commit count' do
-        expect_any_instance_of(Project).to receive(:update_commit_count).
-          and_call_original
-        worker.perform(project.id)
+      it 'updates the project statistics' do
+        expect(worker).to receive(:update_statistics)
+          .with(kind_of(Project), %i(repository_size))
+          .and_call_original
+        worker.perform(project.id, [], %w(repository_size))
       end
       it 'refreshes the method caches' do
@@ -47,33 +43,35 @@ describe ProjectCacheWorker do
           with(%i(readme)).
           and_call_original
-        worker.perform(project.id, %i(readme))
+        worker.perform(project.id, %w(readme))
       end
     end
   end
-  describe '#update_repository_size' do
+  describe '#update_statistics' do
     context 'when a lease could not be obtained' do
       it 'does not update the repository size' do
         allow(worker).to receive(:try_obtain_lease_for).
-          with(project.id, :update_repository_size).
+          with(project.id, :update_statistics).
           and_return(false)
-        expect(project).not_to receive(:update_repository_size)
-        worker.update_repository_size(project)
+        expect(statistics).not_to receive(:refresh!)
+        worker.update_statistics(project)
       end
     end
     context 'when a lease could be obtained' do
-      it 'updates the repository size' do
+      it 'updates the project statistics' do
         allow(worker).to receive(:try_obtain_lease_for).
-          with(project.id, :update_repository_size).
+          with(project.id, :update_statistics).
           and_return(true)
-        expect(project).to receive(:update_repository_size).and_call_original
-        worker.update_repository_size(project)
+        expect(statistics).to receive(:refresh!)
+          .with(only: %i(repository_size))
+          .and_call_original
+        worker.update_statistics(project, %i(repository_size))
       end
     end
   end