Add more storage statistics

This adds counters for build artifacts and LFS objects, and moves
the preexisting repository_size and commit_count from the projects
table into a new project_statistics table.

The counters are displayed in the administration area for projects
and groups, and also available through the API for admins (on */all)
and normal users (on */owned)

The statistics are updated through ProjectCacheWorker, which can now
do more granular updates with the new :statistics argument.

app/views/projects/show.html.haml

@@ -17,10 +17,10 @@
     %ul.nav
       %li
         = link_to project_files_path(@project) do
-          Files (#{repository_size})
+          Files (#{storage_counter(@project.statistics.total_repository_size)})
       %li
         = link_to namespace_project_commits_path(@project.namespace, @project, current_ref) do
-          #{'Commit'.pluralize(@project.commit_count)} (#{number_with_delimiter(@project.commit_count)})
+          #{'Commit'.pluralize(@project.statistics.commit_count)} (#{number_with_delimiter(@project.statistics.commit_count)})
       %li
         = link_to namespace_project_branches_path(@project.namespace, @project) do
           #{'Branch'.pluralize(@repository.branch_count)} (#{number_with_delimiter(@repository.branch_count)})

app/workers/project_cache_worker.rb

@@ -6,26 +6,27 @@ class ProjectCacheWorker
   LEASE_TIMEOUT = 15.minutes.to_i
   # project_id - The ID of the project for which to flush the cache.
-  # refresh - An Array containing extra types of data to refresh such as
-  #           `:readme` to flush the README and `:changelog` to flush the
-  #           CHANGELOG.
-  def perform(project_id, refresh = [])
+  # files - An Array containing extra types of files to refresh such as
+  #         `:readme` to flush the README and `:changelog` to flush the
+  #         CHANGELOG.
+  # statistics - An Array containing columns from ProjectStatistics to
+  #              refresh, if empty all columns will be refreshed
+  def perform(project_id, files = [], statistics = [])
     project = Project.find_by(id: project_id)
     return unless project && project.repository.exists?
-    update_repository_size(project)
-    project.update_commit_count
-    project.repository.refresh_method_caches(refresh.map(&:to_sym))
+    update_statistics(project, statistics.map(&:to_sym))
+    project.repository.refresh_method_caches(files.map(&:to_sym))
   end
-  def update_repository_size(project)
-    return unless try_obtain_lease_for(project.id, :update_repository_size)
-    Rails.logger.info("Updating repository size for project #{project.id}")
-    project.update_repository_size
+  def update_statistics(project, statistics = [])
+    return unless try_obtain_lease_for(project.id, :update_statistics)
+    Rails.logger.info("Updating statistics for project #{project.id}")
+    project.statistics.refresh!(only: statistics)
   end
   private

changelogs/unreleased/feature-more-storage-statistics.yml

+---
+title: Add more storage statistics
+merge_request: 7754
+author: Markus Koller

config/initializers/inflections.rb

@@ -10,5 +10,5 @@
 # end
 #
 ActiveSupport::Inflector.inflections do |inflect|
-  inflect.uncountable %w(award_emoji)
+  inflect.uncountable %w(award_emoji project_statistics)
 end

db/migrate/20161201155511_create_project_statistics.rb

+class CreateProjectStatistics < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    # use bigint columns to support values >2GB
+    counter_column = { limit: 8, null: false, default: 0 }
+
+    create_table :project_statistics do |t|
+      t.references :project, null: false, index: { unique: true }, foreign_key: { on_delete: :cascade }
+      t.references :namespace, null: false, index: true
+      t.integer :commit_count, counter_column
+      t.integer :storage_size, counter_column
+      t.integer :repository_size, counter_column
+      t.integer :lfs_objects_size, counter_column
+      t.integer :build_artifacts_size, counter_column
+    end
+  end
+end

db/migrate/20161201160452_migrate_project_statistics.rb

+class MigrateProjectStatistics < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = true
+  DOWNTIME_REASON = 'Removes two columns from the projects table'
+
+  def up
+    # convert repository_size in float (megabytes) to integer (bytes),
+    # initialize total storage_size with repository_size
+    execute <<-EOF
+      INSERT INTO project_statistics (project_id, namespace_id, commit_count, storage_size, repository_size)
+        SELECT id, namespace_id, commit_count, (repository_size * 1024 * 1024), (repository_size * 1024 * 1024) FROM projects
+    EOF
+
+    remove_column :projects, :repository_size
+    remove_column :projects, :commit_count
+  end
+
+  def down
+    add_column_with_default :projects, :repository_size, :float, default: 0.0
+    add_column_with_default :projects, :commit_count, :integer, default: 0
+  end
+end

db/schema.rb

@@ -901,6 +901,19 @@ ActiveRecord::Schema.define(version: 20161220141214) do
   add_index "project_import_data", ["project_id"], name: "index_project_import_data_on_project_id", using: :btree
+  create_table "project_statistics", force: :cascade do |t|
+    t.integer "project_id", null: false
+    t.integer "namespace_id", null: false
+    t.integer "commit_count", limit: 8, default: 0, null: false
+    t.integer "storage_size", limit: 8, default: 0, null: false
+    t.integer "repository_size", limit: 8, default: 0, null: false
+    t.integer "lfs_objects_size", limit: 8, default: 0, null: false
+    t.integer "build_artifacts_size", limit: 8, default: 0, null: false
+  end
+
+  add_index "project_statistics", ["namespace_id"], name: "index_project_statistics_on_namespace_id", using: :btree
+  add_index "project_statistics", ["project_id"], name: "index_project_statistics_on_project_id", unique: true, using: :btree
+
   create_table "projects", force: :cascade do |t|
     t.string "name"
     t.string "path"
@@ -915,11 +928,9 @@ ActiveRecord::Schema.define(version: 20161220141214) do
     t.boolean "archived", default: false, null: false
     t.string "avatar"
     t.string "import_status"
-    t.float "repository_size", default: 0.0
     t.integer "star_count", default: 0, null: false
     t.string "import_type"
     t.string "import_source"
-    t.integer "commit_count", default: 0
     t.text "import_error"
     t.integer "ci_id"
     t.boolean "shared_runners_enabled", default: true, null: false
@@ -1288,6 +1299,7 @@ ActiveRecord::Schema.define(version: 20161220141214) do
   add_foreign_key "personal_access_tokens", "users"
   add_foreign_key "project_authorizations", "projects", on_delete: :cascade
   add_foreign_key "project_authorizations", "users", on_delete: :cascade
+  add_foreign_key "project_statistics", "projects", on_delete: :cascade
   add_foreign_key "protected_branch_merge_access_levels", "protected_branches"
   add_foreign_key "protected_branch_push_access_levels", "protected_branches"
   add_foreign_key "subscriptions", "projects", on_delete: :cascade

doc/administration/build_artifacts.md

@@ -88,3 +88,9 @@ artifacts through the [Admin area settings](../user/admin_area/settings/continuo
 [reconfigure gitlab]: restart_gitlab.md "How to restart GitLab"
 [restart gitlab]: restart_gitlab.md "How to restart GitLab"
+
+## Storage statistics
+
+You can see the total storage used for build artifacts on groups and projects
+in the administration area, as well as through the [groups](../api/groups.md)
+and [projects APIs](../api/projects.md).

doc/api/groups.md

@@ -13,6 +13,7 @@ Parameters:
 | `search` | string | no | Return list of authorized groups matching the search criteria |
 | `order_by` | string | no | Order groups by `name` or `path`. Default is `name` |
 | `sort` | string | no | Order groups in `asc` or `desc` order. Default is `asc` |
+| `statistics` | boolean | no | Include group statistics (admins only) |
 ```
 GET /groups
@@ -31,7 +32,6 @@ GET /groups
 You can search for groups by name or path, see below.
-=======
 ## List owned groups
 Get a list of groups which are owned by the authenticated user.
@@ -40,6 +40,12 @@ Get a list of groups which are owned by the authenticated user.
 GET /groups/owned
 ```
+Parameters:
+
+| Attribute | Type | Required | Description |
+| --------- | ---- | -------- | ----------- |
+| `statistics` | boolean | no | Include group statistics |
+
 ## List a group's projects
 Get a list of projects in this group.

doc/api/projects.md

@@ -307,6 +307,8 @@ Parameters:
 | `order_by` | string | no | Return projects ordered by `id`, `name`, `path`, `created_at`, `updated_at`, or `last_activity_at` fields. Default is `created_at` |
 | `sort` | string | no | Return projects sorted in `asc` or `desc` order. Default is `desc` |
 | `search` | string | no | Return list of authorized projects matching the search criteria |
+| `simple` | boolean | no | Return only the ID, URL, name, and path of each project |
+| `statistics` | boolean | no | Include project statistics |
 ### List starred projects
@@ -325,6 +327,7 @@ Parameters:
 | `order_by` | string | no | Return projects ordered by `id`, `name`, `path`, `created_at`, `updated_at`, or `last_activity_at` fields. Default is `created_at` |
 | `sort` | string | no | Return projects sorted in `asc` or `desc` order. Default is `desc` |
 | `search` | string | no | Return list of authorized projects matching the search criteria |
+| `simple` | boolean | no | Return only the ID, URL, name, and path of each project |
 ### List ALL projects
@@ -343,6 +346,7 @@ Parameters:
 | `order_by` | string | no | Return projects ordered by `id`, `name`, `path`, `created_at`, `updated_at`, or `last_activity_at` fields. Default is `created_at` |
 | `sort` | string | no | Return projects sorted in `asc` or `desc` order. Default is `desc` |
 | `search` | string | no | Return list of authorized projects matching the search criteria |
+| `statistics` | boolean | no | Include project statistics |
 ### Get single project

doc/workflow/lfs/lfs_administration.md

@@ -40,6 +40,12 @@ In `config/gitlab.yml`:
     storage_path: /mnt/storage/lfs-objects
 ```
+## Storage statistics
+
+You can see the total storage used for LFS objects on groups and projects
+in the administration area, as well as through the [groups](../api/groups.md)
+and [projects APIs](../api/projects.md).
+
 ## Known limitations
 * Currently, storing GitLab Git LFS objects on a non-local storage (like S3 buckets)
@@ -47,3 +53,5 @@ In `config/gitlab.yml`:
 * Currently, removing LFS objects from GitLab Git LFS storage is not supported
 * LFS authentications via SSH was added with GitLab 8.12
 * Only compatible with the GitLFS client versions 1.1.0 and up, or 1.0.2.
+* The storage statistics currently count each LFS object multiple times for
+  every project linking to it

lib/api/entities.rb

@@ -101,6 +101,16 @@ module API
       expose :only_allow_merge_if_build_succeeds
       expose :request_access_enabled
       expose :only_allow_merge_if_all_discussions_are_resolved
+
+      expose :statistics, using: 'API::Entities::ProjectStatistics', if: :statistics
+    end
+
+    class ProjectStatistics < Grape::Entity
+      expose :commit_count
+      expose :storage_size
+      expose :repository_size
+      expose :lfs_objects_size
+      expose :build_artifacts_size
     end
     class Member < UserBasic
@@ -127,6 +137,15 @@ module API
       expose :avatar_url
       expose :web_url
       expose :request_access_enabled
+
+      expose :statistics, if: :statistics do
+        with_options format_with: -> (value) { value.to_i } do
+          expose :storage_size
+          expose :repository_size
+          expose :lfs_objects_size
+          expose :build_artifacts_size
+        end
+      end
     end
     class GroupDetail < Group

lib/api/groups.rb

@@ -11,6 +11,20 @@ module API
         optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group'
         optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access'
       end
+
+      params :statistics_params do
+        optional :statistics, type: Boolean, default: false, desc: 'Include project statistics'
+      end
+
+      def present_groups(groups, options = {})
+        options = options.reverse_merge(
+          with: Entities::Group,
+          current_user: current_user,
+        )
+
+        groups = groups.with_statistics if options[:statistics]
+        present paginate(groups), options
+      end
     end
     resource :groups do
@@ -18,6 +32,7 @@ module API
         success Entities::Group
       end
       params do
+        use :statistics_params
         optional :skip_groups, type: Array[Integer], desc: 'Array of group ids to exclude from list'
         optional :all_available, type: Boolean, desc: 'Show all group that you have access to'
         optional :search, type: String, desc: 'Search for a specific group'
@@ -38,7 +53,7 @@ module API
         groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present?
         groups = groups.reorder(params[:order_by] => params[:sort])
-        present paginate(groups), with: Entities::Group, current_user: current_user
+        present_groups groups, statistics: params[:statistics] && current_user.is_admin?
       end
       desc 'Get list of owned groups for authenticated user' do
@@ -46,10 +61,10 @@ module API
       end
       params do
         use :pagination
+        use :statistics_params
       end
       get '/owned' do
-        groups = current_user.owned_groups
-        present paginate(groups), with: Entities::Group, current_user: current_user
+        present_groups current_user.owned_groups, statistics: params[:statistics]
       end
       desc 'Create a group. Available only for users who can create groups.' do

lib/api/helpers.rb

@@ -248,7 +248,7 @@ module API
       rack_response({ 'message' => '500 Internal Server Error' }.to_json, 500)
     end
-    # Projects helpers
+    # project helpers
     def filter_projects(projects)
       if params[:search].present?

lib/api/projects.rb

@@ -40,6 +40,15 @@ module API
     resource :projects do
       helpers do
+        params :collection_params do
+          use :sort_params
+          use :filter_params
+          use :pagination
+
+          optional :simple, type: Boolean, default: false,
+                            desc: 'Return only the ID, URL, name, and path of each project'
+        end
+
         params :sort_params do
           optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at],
                               default: 'created_at', desc: 'Return projects ordered by field'
@@ -52,97 +61,94 @@ module API
           optional :visibility, type: String, values: %w[public internal private],
                                 desc: 'Limit by visibility'
           optional :search, type: String, desc: 'Return list of authorized projects matching the search criteria'
-          use :sort_params
+        end
+
+        params :statistics_params do
+          optional :statistics, type: Boolean, default: false, desc: 'Include project statistics'
         end
         params :create_params do
           optional :namespace_id, type: Integer, desc: 'Namespace ID for the new project. Default to the user namespace.'
           optional :import_url, type: String, desc: 'URL from which the project is imported'
         end
+
+        def present_projects(projects, options = {})
+          options = options.reverse_merge(
+            with: Entities::Project,
+            current_user: current_user,
+            simple: params[:simple],
+          )
+
+          projects = filter_projects(projects)
+          projects = projects.with_statistics if options[:statistics]
+          options[:with] = Entities::BasicProjectDetails if options[:simple]
+
+          present paginate(projects), options
+        end
       end
       desc 'Get a list of visible projects for authenticated user' do
         success Entities::BasicProjectDetails
       end
       params do
-        optional :simple, type: Boolean, default: false,
-                          desc: 'Return only the ID, URL, name, and path of each project'
-        use :filter_params
-        use :pagination
+        use :collection_params
       end
       get '/visible' do
-        projects = ProjectsFinder.new.execute(current_user)
-        projects = filter_projects(projects)
-        entity = params[:simple] || !current_user ? Entities::BasicProjectDetails : Entities::ProjectWithAccess
-
-        present paginate(projects), with: entity, current_user: current_user
+        entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails
+        present_projects ProjectsFinder.new.execute(current_user), with: entity
       end
       desc 'Get a projects list for authenticated user' do
         success Entities::BasicProjectDetails
       end
       params do
-        optional :simple, type: Boolean, default: false,
-                          desc: 'Return only the ID, URL, name, and path of each project'
-        use :filter_params
-        use :pagination
+        use :collection_params
       end
       get do
         authenticate!
-        projects = current_user.authorized_projects
-        projects = filter_projects(projects)
-        entity = params[:simple] ? Entities::BasicProjectDetails : Entities::ProjectWithAccess
-
-        present paginate(projects), with: entity, current_user: current_user
+        present_projects current_user.authorized_projects,
+          with: Entities::ProjectWithAccess
       end
       desc 'Get an owned projects list for authenticated user' do
         success Entities::BasicProjectDetails
       end
       params do
-        use :filter_params
-        use :pagination
+        use :collection_params
+        use :statistics_params
       end
       get '/owned' do
         authenticate!
-        projects = current_user.owned_projects
-        projects = filter_projects(projects)
-
-        present paginate(projects), with: Entities::ProjectWithAccess, current_user: current_user
+        present_projects current_user.owned_projects,
+          with: Entities::ProjectWithAccess,
+          statistics: params[:statistics]
       end
       desc 'Gets starred project for the authenticated user' do
         success Entities::BasicProjectDetails
       end
       params do
-        use :filter_params
-        use :pagination
+        use :collection_params
       end
       get '/starred' do
         authenticate!
-        projects = current_user.viewable_starred_projects
-        projects = filter_projects(projects)
-
-        present paginate(projects), with: Entities::Project, current_user: current_user
+        present_projects current_user.viewable_starred_projects
       end
       desc 'Get all projects for admin user' do
         success Entities::BasicProjectDetails
       end
       params do
-        use :filter_params
-        use :pagination
+        use :collection_params
+        use :statistics_params
       end
       get '/all' do
         authenticated_as_admin!
-        projects = Project.all
-        projects = filter_projects(projects)
-
-        present paginate(projects), with: Entities::ProjectWithAccess, current_user: current_user
+        present_projects Project.all, with: Entities::ProjectWithAccess, statistics: params[:statistics]
       end
       desc 'Search for projects the current user has access to' do

lib/tasks/gitlab/import.rake

@@ -63,8 +63,7 @@ namespace :gitlab do
             if project.persisted?
               puts " * Created #{project.name} (#{repo_path})".color(:green)
-              project.update_repository_size
-              project.update_commit_count
+              ProjectCacheWorker.perform(project.id)
             else
               puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red)
               puts "   Errors: #{project.errors.messages}".color(:red)

spec/factories/lfs_objects.rb

@@ -2,7 +2,7 @@ include ActionDispatch::TestProcess
 FactoryGirl.define do
   factory :lfs_object do
-    oid "b68143e6463773b1b6c6fd009a76c32aeec041faff32ba2ed42fd7f708a17f80"
+    sequence(:oid) { |n| "b68143e6463773b1b6c6fd009a76c32aeec041faff32ba2ed42fd7f708a%05x" % n }
     size 499013
   end

spec/factories/project_statistics.rb

+FactoryGirl.define do
+  factory :project_statistics do
+    project { create :project }
+    namespace { project.namespace }
+  end
+end

spec/helpers/storage_helper_spec.rb

+require 'spec_helper'
+
+describe StorageHelper do
+  describe '#storage_counter' do
+    it 'formats bytes to one decimal place' do
+      expect(helper.storage_counter(1.23.megabytes)).to eq '1.2 MB'
+    end
+
+    it 'does not add decimals for sizes < 1 MB' do
+      expect(helper.storage_counter(23.5.kilobytes)).to eq '24 KB'
+    end
+
+    it 'does not add decimals for zeroes' do
+      expect(helper.storage_counter(2.megabytes)).to eq '2 MB'
+    end
+
+    it 'uses commas as thousands separator' do
+      expect(helper.storage_counter(100_000_000_000_000_000)).to eq '90,949.5 TB'
+    end
+  end
+end

spec/lib/gitlab/import_export/all_models.yml

@@ -192,6 +192,7 @@ project:
 - authorized_users
 - project_authorizations
 - route
+- statistics
 award_emoji:
 - awardable
 - user