diff --git a/app/controllers/admin/groups_controller.rb b/app/controllers/admin/groups_controller.rb
index 1e3d194e9f9b5846fc63ee46bfbfe90740ddcb24..61a3a03182a14e95731a97e2d86858e459ca91d8 100644
--- a/app/controllers/admin/groups_controller.rb
+++ b/app/controllers/admin/groups_controller.rb
@@ -1,17 +1,18 @@
class Admin::GroupsController < Admin::ApplicationController
- before_action :group, only: [:edit, :show, :update, :destroy, :project_update, :members_update]
+ before_action :group, only: [:edit, :update, :destroy, :project_update, :members_update]
def index
- @groups = Group.all
+ @groups = Group.with_statistics
@groups = @groups.sort(@sort = params[:sort])
@groups = @groups.search(params[:name]) if params[:name].present?
@groups = @groups.page(params[:page])
end
def show
+ @group = Group.with_statistics.find_by_full_path(params[:id])
@members = @group.members.order("access_level DESC").page(params[:members_page])
@requesters = AccessRequestsFinder.new(@group).execute(current_user)
- @projects = @group.projects.page(params[:projects_page])
+ @projects = @group.projects.with_statistics.page(params[:projects_page])
end
def new
diff --git a/app/controllers/admin/projects_controller.rb b/app/controllers/admin/projects_controller.rb
index 1d963bdf7d58df28917edef167ac5141a5e52458..b09ae4230969720c14698b5e21316162d880ae1f 100644
--- a/app/controllers/admin/projects_controller.rb
+++ b/app/controllers/admin/projects_controller.rb
@@ -3,7 +3,7 @@ class Admin::ProjectsController < Admin::ApplicationController
before_action :group, only: [:show, :transfer]
def index
- @projects = Project.all
+ @projects = Project.with_statistics
@projects = @projects.in_namespace(params[:namespace_id]) if params[:namespace_id].present?
@projects = @projects.where(visibility_level: params[:visibility_level]) if params[:visibility_level].present?
@projects = @projects.with_push if params[:with_push].present?
diff --git a/app/controllers/groups_controller.rb b/app/controllers/groups_controller.rb
index efe9c001bcf40d0f65011e3974ca02f3482c191a..01c8fa2739f363240e96f772a3e3662d92263c89 100644
--- a/app/controllers/groups_controller.rb
+++ b/app/controllers/groups_controller.rb
@@ -75,7 +75,7 @@ class GroupsController < Groups::ApplicationController
end
def projects
- @projects = @group.projects.page(params[:page])
+ @projects = @group.projects.with_statistics.page(params[:page])
end
def update
diff --git a/app/helpers/projects_helper.rb b/app/helpers/projects_helper.rb
index 7445f3c113cbbaf713e5fa3e790ac0c9a147fc75..0f17f6d8b7e35dcff679338ae2f8e093f8f2a394 100644
--- a/app/helpers/projects_helper.rb
+++ b/app/helpers/projects_helper.rb
@@ -246,11 +246,6 @@ module ProjectsHelper
end
end
- def repository_size(project = @project)
- size_in_bytes = project.repository_size * 1.megabyte
- number_to_human_size(size_in_bytes, delimiter: ',', precision: 2)
- end
-
def default_url_to_repo(project = @project)
case default_clone_protocol
when 'ssh'
@@ -398,20 +393,6 @@ module ProjectsHelper
[@project.path_with_namespace, sha, "readme"].join('-')
end
- def round_commit_count(project)
- count = project.commit_count
-
- if count > 10000
- '10000+'
- elsif count > 5000
- '5000+'
- elsif count > 1000
- '1000+'
- else
- count
- end
- end
-
def current_ref
@ref || @repository.try(:root_ref)
end
diff --git a/app/helpers/sorting_helper.rb b/app/helpers/sorting_helper.rb
index f03c46270503659a2b5549a860a9e3e83fb7bf89..ff787fb4131480c8b958a39b82acc1d54b6c0bc6 100644
--- a/app/helpers/sorting_helper.rb
+++ b/app/helpers/sorting_helper.rb
@@ -11,6 +11,7 @@ module SortingHelper
sort_value_due_date_soon => sort_title_due_date_soon,
sort_value_due_date_later => sort_title_due_date_later,
sort_value_largest_repo => sort_title_largest_repo,
+ sort_value_largest_group => sort_title_largest_group,
sort_value_recently_signin => sort_title_recently_signin,
sort_value_oldest_signin => sort_title_oldest_signin,
sort_value_downvotes => sort_title_downvotes,
@@ -92,6 +93,10 @@ module SortingHelper
'Largest repository'
end
+ def sort_title_largest_group
+ 'Largest group'
+ end
+
def sort_title_recently_signin
'Recent sign in'
end
@@ -193,7 +198,11 @@ module SortingHelper
end
def sort_value_largest_repo
- 'repository_size_desc'
+ 'storage_size_desc'
+ end
+
+ def sort_value_largest_group
+ 'storage_size_desc'
end
def sort_value_recently_signin
diff --git a/app/helpers/storage_helper.rb b/app/helpers/storage_helper.rb
new file mode 100644
index 0000000000000000000000000000000000000000..e19c67a37ca0c46c38b076b2a05d1d648cdca618
--- /dev/null
+++ b/app/helpers/storage_helper.rb
@@ -0,0 +1,7 @@
+module StorageHelper
+ def storage_counter(size_in_bytes)
+ precision = size_in_bytes < 1.megabyte ? 0 : 1
+
+ number_to_human_size(size_in_bytes, delimiter: ',', precision: precision, significant: false)
+ end
+end
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index cb76cdf5981421e7ec9072bd89191e7283d1590d..270427987418d7e1fecbadd4fe93aa58ef7331f2 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -43,6 +43,8 @@ module Ci
before_destroy { project }
after_create :execute_hooks
+ after_save :update_project_statistics, if: :artifacts_size_changed?
+ after_destroy :update_project_statistics
class << self
def first_pending
@@ -584,5 +586,9 @@ module Ci
Ci::MaskSecret.mask!(trace, token)
trace
end
+
+ def update_project_statistics
+ ProjectCacheWorker.perform_async(project_id, [], [:build_artifacts_size])
+ end
end
end
diff --git a/app/models/group.rb b/app/models/group.rb
index ac8a82c8c1e253b27f3ec42ec1d0119e54063720..85696ad97476ef4a637987677af12c732f029857 100644
--- a/app/models/group.rb
+++ b/app/models/group.rb
@@ -48,7 +48,13 @@ class Group < Namespace
end
def sort(method)
- order_by(method)
+ if method == 'storage_size_desc'
+ # storage_size is a virtual column so we need to
+ # pass a string to avoid AR adding the table name
+ reorder('storage_size DESC, namespaces.id DESC')
+ else
+ order_by(method)
+ end
end
def reference_prefix
diff --git a/app/models/lfs_objects_project.rb b/app/models/lfs_objects_project.rb
index 0fd5f089db9076d03d7c17fcea5518964d688809..007eed5600a0b47e146f477948731b5f30f480ba 100644
--- a/app/models/lfs_objects_project.rb
+++ b/app/models/lfs_objects_project.rb
@@ -5,4 +5,13 @@ class LfsObjectsProject < ActiveRecord::Base
validates :lfs_object_id, presence: true
validates :lfs_object_id, uniqueness: { scope: [:project_id], message: "already exists in project" }
validates :project_id, presence: true
+
+ after_create :update_project_statistics
+ after_destroy :update_project_statistics
+
+ private
+
+ def update_project_statistics
+ ProjectCacheWorker.perform_async(project_id, [], [:lfs_objects_size])
+ end
end
diff --git a/app/models/namespace.rb b/app/models/namespace.rb
index b52f08c70813f94f5cdd46694429b5125e3fefac..d41833de66f2cd0325092fc5f51227baf703a77f 100644
--- a/app/models/namespace.rb
+++ b/app/models/namespace.rb
@@ -9,6 +9,7 @@ class Namespace < ActiveRecord::Base
cache_markdown_field :description, pipeline: :description
has_many :projects, dependent: :destroy
+ has_many :project_statistics
belongs_to :owner, class_name: "User"
belongs_to :parent, class_name: "Namespace"
@@ -38,6 +39,18 @@ class Namespace < ActiveRecord::Base
scope :root, -> { where('type IS NULL') }
+ scope :with_statistics, -> do
+ joins('LEFT JOIN project_statistics ps ON ps.namespace_id = namespaces.id')
+ .group('namespaces.id')
+ .select(
+ 'namespaces.*',
+ 'COALESCE(SUM(ps.storage_size), 0) AS storage_size',
+ 'COALESCE(SUM(ps.repository_size), 0) AS repository_size',
+ 'COALESCE(SUM(ps.lfs_objects_size), 0) AS lfs_objects_size',
+ 'COALESCE(SUM(ps.build_artifacts_size), 0) AS build_artifacts_size',
+ )
+ end
+
class << self
def by_path(path)
find_by('lower(path) = :value', value: path.downcase)
diff --git a/app/models/project.rb b/app/models/project.rb
index 26fa20f856de195c6641f4e86eadb72eb80731cd..19bbe65b01d6b6281eae3dc5b770e5393e463aea 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -44,6 +44,7 @@ class Project < ActiveRecord::Base
after_create :ensure_dir_exist
after_create :create_project_feature, unless: :project_feature
after_save :ensure_dir_exist, if: :namespace_id_changed?
+ after_save :update_project_statistics, if: :namespace_id_changed?
# set last_activity_at to the same as created_at
after_create :set_last_activity_at
@@ -151,6 +152,7 @@ class Project < ActiveRecord::Base
has_one :import_data, dependent: :destroy, class_name: "ProjectImportData"
has_one :project_feature, dependent: :destroy
+ has_one :statistics, class_name: 'ProjectStatistics', dependent: :delete
has_many :commit_statuses, dependent: :destroy, foreign_key: :gl_project_id
has_many :pipelines, dependent: :destroy, class_name: 'Ci::Pipeline', foreign_key: :gl_project_id
@@ -220,6 +222,7 @@ class Project < ActiveRecord::Base
scope :with_push, -> { joins(:events).where('events.action = ?', Event::PUSHED) }
scope :with_project_feature, -> { joins('LEFT JOIN project_features ON projects.id = project_features.project_id') }
+ scope :with_statistics, -> { includes(:statistics) }
# "enabled" here means "not disabled". It includes private features!
scope :with_feature_enabled, ->(feature) {
@@ -332,8 +335,10 @@ class Project < ActiveRecord::Base
end
def sort(method)
- if method == 'repository_size_desc'
- reorder(repository_size: :desc, id: :desc)
+ if method == 'storage_size_desc'
+ # storage_size is a joined column so we need to
+ # pass a string to avoid AR adding the table name
+ reorder('project_statistics.storage_size DESC, projects.id DESC')
else
order_by(method)
end
@@ -1036,14 +1041,6 @@ class Project < ActiveRecord::Base
forked? && project == forked_from_project
end
- def update_repository_size
- update_attribute(:repository_size, repository.size)
- end
-
- def update_commit_count
- update_attribute(:commit_count, repository.commit_count)
- end
-
def forks_count
forks.count
end
@@ -1322,4 +1319,9 @@ class Project < ActiveRecord::Base
def full_path_changed?
path_changed? || namespace_id_changed?
end
+
+ def update_project_statistics
+ stats = statistics || build_statistics
+ stats.update(namespace_id: namespace_id)
+ end
end
diff --git a/app/models/project_statistics.rb b/app/models/project_statistics.rb
new file mode 100644
index 0000000000000000000000000000000000000000..2270ac750716afcbf25c17ee484cfa5bbee0d953
--- /dev/null
+++ b/app/models/project_statistics.rb
@@ -0,0 +1,43 @@
+class ProjectStatistics < ActiveRecord::Base
+ belongs_to :project
+ belongs_to :namespace
+
+ before_save :update_storage_size
+
+ STORAGE_COLUMNS = [:repository_size, :lfs_objects_size, :build_artifacts_size]
+ STATISTICS_COLUMNS = [:commit_count] + STORAGE_COLUMNS
+
+ def total_repository_size
+ repository_size + lfs_objects_size
+ end
+
+ def refresh!(only: nil)
+ STATISTICS_COLUMNS.each do |column, generator|
+ if only.blank? || only.include?(column)
+ public_send("update_#{column}")
+ end
+ end
+
+ save!
+ end
+
+ def update_commit_count
+ self.commit_count = project.repository.commit_count
+ end
+
+ def update_repository_size
+ self.repository_size = project.repository.size
+ end
+
+ def update_lfs_objects_size
+ self.lfs_objects_size = project.lfs_objects.sum(:size)
+ end
+
+ def update_build_artifacts_size
+ self.build_artifacts_size = project.builds.sum(:artifacts_size)
+ end
+
+ def update_storage_size
+ self.storage_size = STORAGE_COLUMNS.sum(&method(:read_attribute))
+ end
+end
diff --git a/app/services/git_push_service.rb b/app/services/git_push_service.rb
index 6bbc3a9d9ff7afea1fc42d2af1b8642bef6798f1..dbe2fda27b5192e31165c71a23c56e2267e21e51 100644
--- a/app/services/git_push_service.rb
+++ b/app/services/git_push_service.rb
@@ -77,7 +77,7 @@ class GitPushService < BaseService
types = []
end
- ProjectCacheWorker.perform_async(@project.id, types)
+ ProjectCacheWorker.perform_async(@project.id, types, [:commit_count, :repository_size])
end
# Schedules processing of commit messages.
diff --git a/app/services/git_tag_push_service.rb b/app/services/git_tag_push_service.rb
index 20a4445bddf2334b54be1388b4aec5a1b44a4ec3..96432837481e5e3a577851395eeba0b95b595989 100644
--- a/app/services/git_tag_push_service.rb
+++ b/app/services/git_tag_push_service.rb
@@ -12,7 +12,7 @@ class GitTagPushService < BaseService
project.execute_hooks(@push_data.dup, :tag_push_hooks)
project.execute_services(@push_data.dup, :tag_push_hooks)
Ci::CreatePipelineService.new(project, current_user, @push_data).execute
- ProjectCacheWorker.perform_async(project.id)
+ ProjectCacheWorker.perform_async(project.id, [], [:commit_count, :repository_size])
true
end
diff --git a/app/views/admin/groups/_group.html.haml b/app/views/admin/groups/_group.html.haml
index cf28f92853e145f5ade7982d2122fffa6e7122b2..6fc212119c47443331138d3867e998ae7067b732 100644
--- a/app/views/admin/groups/_group.html.haml
+++ b/app/views/admin/groups/_group.html.haml
@@ -5,6 +5,9 @@
= link_to 'Edit', admin_group_edit_path(group), id: "edit_#{dom_id(group)}", class: 'btn'
= link_to 'Delete', [:admin, group], data: { confirm: "Are you sure you want to remove #{group.name}?" }, method: :delete, class: 'btn btn-remove'
.stats
+ %span.badge
+ = storage_counter(group.storage_size)
+
%span
= icon('bookmark')
= number_with_delimiter(group.projects.count)
diff --git a/app/views/admin/groups/index.html.haml b/app/views/admin/groups/index.html.haml
index 794f910a61feac37454506141713714c61d18191..07775247cfd00be2608d9e7e0162ac4270351071 100644
--- a/app/views/admin/groups/index.html.haml
+++ b/app/views/admin/groups/index.html.haml
@@ -27,6 +27,8 @@
= sort_title_recently_updated
= link_to admin_groups_path(sort: sort_value_oldest_updated, name: project_name) do
= sort_title_oldest_updated
+ = link_to admin_groups_path(sort: sort_value_largest_group, name: project_name) do
+ = sort_title_largest_group
= link_to new_admin_group_path, class: "btn btn-new" do
New Group
%ul.content-list
diff --git a/app/views/admin/groups/show.html.haml b/app/views/admin/groups/show.html.haml
index 7b0175af2140e0a6e87f156cd02be58e8840aa3d..ab9c79f6addc8d488174984bd49e8510da0dfe9b 100644
--- a/app/views/admin/groups/show.html.haml
+++ b/app/views/admin/groups/show.html.haml
@@ -38,6 +38,18 @@
%strong
= @group.created_at.to_s(:medium)
+ %li
+ %span.light Storage:
+ %strong= storage_counter(@group.storage_size)
+ (
+ = storage_counter(@group.repository_size)
+ repositories,
+ = storage_counter(@group.build_artifacts_size)
+ build artifacts,
+ = storage_counter(@group.lfs_objects_size)
+ LFS
+ )
+
%li
%span.light Group Git LFS status:
%strong
@@ -55,8 +67,8 @@
%li
%strong
= link_to project.name_with_namespace, [:admin, project.namespace.becomes(Namespace), project]
- %span.label.label-gray
- = repository_size(project)
+ %span.badge
+ = storage_counter(project.statistics.storage_size)
%span.pull-right.light
%span.monospace= project.path_with_namespace + ".git"
.panel-footer
@@ -73,8 +85,8 @@
%li
%strong
= link_to project.name_with_namespace, [:admin, project.namespace.becomes(Namespace), project]
- %span.label.label-gray
- = repository_size(project)
+ %span.badge
+ = storage_counter(project.statistics.storage_size)
%span.pull-right.light
%span.monospace= project.path_with_namespace + ".git"
diff --git a/app/views/admin/projects/index.html.haml b/app/views/admin/projects/index.html.haml
index 8bc7dc7dd51e6805f910758f9574e16c5e6fe525..2e6f03fcde001219214a48ceb21349c5469536b1 100644
--- a/app/views/admin/projects/index.html.haml
+++ b/app/views/admin/projects/index.html.haml
@@ -69,8 +69,8 @@
.controls
- if project.archived
%span.label.label-warning archived
- %span.label.label-gray
- = repository_size(project)
+ %span.badge
+ = storage_counter(project.statistics.storage_size)
= link_to 'Edit', edit_namespace_project_path(project.namespace, project), id: "edit_#{dom_id(project)}", class: "btn"
= link_to 'Delete', [project.namespace.becomes(Namespace), project], data: { confirm: remove_project_message(project) }, method: :delete, class: "btn btn-remove"
.title
diff --git a/app/views/admin/projects/show.html.haml b/app/views/admin/projects/show.html.haml
index 6c7c3c48604aa7a26baf05d8d9cc0d079ee2bacf..2967da6e692f565022fbd5861236742e9656f8ef 100644
--- a/app/views/admin/projects/show.html.haml
+++ b/app/views/admin/projects/show.html.haml
@@ -65,9 +65,16 @@
= @project.repository.path_to_repo
%li
- %span.light Size
- %strong
- = repository_size(@project)
+ %span.light Storage:
+ %strong= storage_counter(@project.statistics.storage_size)
+ (
+ = storage_counter(@project.statistics.repository_size)
+ repository,
+ = storage_counter(@project.statistics.build_artifacts_size)
+ build artifacts,
+ = storage_counter(@project.statistics.lfs_objects_size)
+ LFS
+ )
%li
%span.light last commit:
diff --git a/app/views/groups/projects.html.haml b/app/views/groups/projects.html.haml
index 33fee334d93f805e6f602ccbbe624be77a43b95b..2e7e5e5c30950386fb05194bafa2ba0320991dbd 100644
--- a/app/views/groups/projects.html.haml
+++ b/app/views/groups/projects.html.haml
@@ -18,8 +18,8 @@
.pull-right
- if project.archived
%span.label.label-warning archived
- %span.label.label-gray
- = repository_size(project)
+ %span.badge
+ = storage_counter(project.statistics.storage_size)
= link_to 'Members', namespace_project_project_members_path(project.namespace, project), id: "edit_#{dom_id(project)}", class: "btn btn-sm"
= link_to 'Edit', edit_namespace_project_path(project.namespace, project), id: "edit_#{dom_id(project)}", class: "btn btn-sm"
= link_to 'Remove', project, data: { confirm: remove_project_message(project)}, method: :delete, class: "btn btn-sm btn-remove"
diff --git a/app/views/projects/show.html.haml b/app/views/projects/show.html.haml
index a915c159cb413775797e3b1088005beeca16d0a7..e33182c357f5723c0fc91abccc691f172317b08f 100644
--- a/app/views/projects/show.html.haml
+++ b/app/views/projects/show.html.haml
@@ -17,10 +17,10 @@
%ul.nav
%li
= link_to project_files_path(@project) do
- Files (#{repository_size})
+ Files (#{storage_counter(@project.statistics.total_repository_size)})
%li
= link_to namespace_project_commits_path(@project.namespace, @project, current_ref) do
- #{'Commit'.pluralize(@project.commit_count)} (#{number_with_delimiter(@project.commit_count)})
+ #{'Commit'.pluralize(@project.statistics.commit_count)} (#{number_with_delimiter(@project.statistics.commit_count)})
%li
= link_to namespace_project_branches_path(@project.namespace, @project) do
#{'Branch'.pluralize(@repository.branch_count)} (#{number_with_delimiter(@repository.branch_count)})
diff --git a/app/workers/project_cache_worker.rb b/app/workers/project_cache_worker.rb
index 27d7e6527210cc503e4ce68a42b15ea4bb3ed91b..8ff9d07860fdf0ab9eaf7ca85cdca43a0e9b6749 100644
--- a/app/workers/project_cache_worker.rb
+++ b/app/workers/project_cache_worker.rb
@@ -6,26 +6,27 @@ class ProjectCacheWorker
LEASE_TIMEOUT = 15.minutes.to_i
# project_id - The ID of the project for which to flush the cache.
- # refresh - An Array containing extra types of data to refresh such as
- # `:readme` to flush the README and `:changelog` to flush the
- # CHANGELOG.
- def perform(project_id, refresh = [])
+ # files - An Array containing extra types of files to refresh such as
+ # `:readme` to flush the README and `:changelog` to flush the
+ # CHANGELOG.
+ # statistics - An Array containing columns from ProjectStatistics to
+ # refresh, if empty all columns will be refreshed
+ def perform(project_id, files = [], statistics = [])
project = Project.find_by(id: project_id)
return unless project && project.repository.exists?
- update_repository_size(project)
- project.update_commit_count
+ update_statistics(project, statistics.map(&:to_sym))
- project.repository.refresh_method_caches(refresh.map(&:to_sym))
+ project.repository.refresh_method_caches(files.map(&:to_sym))
end
- def update_repository_size(project)
- return unless try_obtain_lease_for(project.id, :update_repository_size)
+ def update_statistics(project, statistics = [])
+ return unless try_obtain_lease_for(project.id, :update_statistics)
- Rails.logger.info("Updating repository size for project #{project.id}")
+ Rails.logger.info("Updating statistics for project #{project.id}")
- project.update_repository_size
+ project.statistics.refresh!(only: statistics)
end
private
diff --git a/changelogs/unreleased/feature-more-storage-statistics.yml b/changelogs/unreleased/feature-more-storage-statistics.yml
new file mode 100644
index 0000000000000000000000000000000000000000..824fd36dc34dbb4ff0be0aa9a773728c6ca17105
--- /dev/null
+++ b/changelogs/unreleased/feature-more-storage-statistics.yml
@@ -0,0 +1,4 @@
+---
+title: Add more storage statistics
+merge_request: 7754
+author: Markus Koller
diff --git a/config/initializers/inflections.rb b/config/initializers/inflections.rb
index 3d1a41a46525d29327b2f7f0db903fb5a6794acf..d4197da3fa9b9a51c81bb915538ea56af2c36ef6 100644
--- a/config/initializers/inflections.rb
+++ b/config/initializers/inflections.rb
@@ -10,5 +10,5 @@
# end
#
ActiveSupport::Inflector.inflections do |inflect|
- inflect.uncountable %w(award_emoji)
+ inflect.uncountable %w(award_emoji project_statistics)
end
diff --git a/db/migrate/20161201155511_create_project_statistics.rb b/db/migrate/20161201155511_create_project_statistics.rb
new file mode 100644
index 0000000000000000000000000000000000000000..26e6d3623eb1317526fc2a5ae504010e411ba81a
--- /dev/null
+++ b/db/migrate/20161201155511_create_project_statistics.rb
@@ -0,0 +1,20 @@
+class CreateProjectStatistics < ActiveRecord::Migration
+ include Gitlab::Database::MigrationHelpers
+
+ DOWNTIME = false
+
+ def change
+ # use bigint columns to support values >2GB
+ counter_column = { limit: 8, null: false, default: 0 }
+
+ create_table :project_statistics do |t|
+ t.references :project, null: false, index: { unique: true }, foreign_key: { on_delete: :cascade }
+ t.references :namespace, null: false, index: true
+ t.integer :commit_count, counter_column
+ t.integer :storage_size, counter_column
+ t.integer :repository_size, counter_column
+ t.integer :lfs_objects_size, counter_column
+ t.integer :build_artifacts_size, counter_column
+ end
+ end
+end
diff --git a/db/migrate/20161201160452_migrate_project_statistics.rb b/db/migrate/20161201160452_migrate_project_statistics.rb
new file mode 100644
index 0000000000000000000000000000000000000000..3ae3f2c159b57c9c09e74ae6db2cbe5693650ce4
--- /dev/null
+++ b/db/migrate/20161201160452_migrate_project_statistics.rb
@@ -0,0 +1,23 @@
+class MigrateProjectStatistics < ActiveRecord::Migration
+ include Gitlab::Database::MigrationHelpers
+
+ DOWNTIME = true
+ DOWNTIME_REASON = 'Removes two columns from the projects table'
+
+ def up
+ # convert repository_size in float (megabytes) to integer (bytes),
+ # initialize total storage_size with repository_size
+ execute <<-EOF
+ INSERT INTO project_statistics (project_id, namespace_id, commit_count, storage_size, repository_size)
+ SELECT id, namespace_id, commit_count, (repository_size * 1024 * 1024), (repository_size * 1024 * 1024) FROM projects
+ EOF
+
+ remove_column :projects, :repository_size
+ remove_column :projects, :commit_count
+ end
+
+ def down
+ add_column_with_default :projects, :repository_size, :float, default: 0.0
+ add_column_with_default :projects, :commit_count, :integer, default: 0
+ end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 05b6c807660180e493eeb08ff8490f99db2d91b1..b70261e050f3c9e4d3187fbb359dbf45fdb1f31b 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -901,6 +901,19 @@ ActiveRecord::Schema.define(version: 20161221140236) do
add_index "project_import_data", ["project_id"], name: "index_project_import_data_on_project_id", using: :btree
+ create_table "project_statistics", force: :cascade do |t|
+ t.integer "project_id", null: false
+ t.integer "namespace_id", null: false
+ t.integer "commit_count", limit: 8, default: 0, null: false
+ t.integer "storage_size", limit: 8, default: 0, null: false
+ t.integer "repository_size", limit: 8, default: 0, null: false
+ t.integer "lfs_objects_size", limit: 8, default: 0, null: false
+ t.integer "build_artifacts_size", limit: 8, default: 0, null: false
+ end
+
+ add_index "project_statistics", ["namespace_id"], name: "index_project_statistics_on_namespace_id", using: :btree
+ add_index "project_statistics", ["project_id"], name: "index_project_statistics_on_project_id", unique: true, using: :btree
+
create_table "projects", force: :cascade do |t|
t.string "name"
t.string "path"
@@ -915,11 +928,9 @@ ActiveRecord::Schema.define(version: 20161221140236) do
t.boolean "archived", default: false, null: false
t.string "avatar"
t.string "import_status"
- t.float "repository_size", default: 0.0
t.integer "star_count", default: 0, null: false
t.string "import_type"
t.string "import_source"
- t.integer "commit_count", default: 0
t.text "import_error"
t.integer "ci_id"
t.boolean "shared_runners_enabled", default: true, null: false
@@ -1288,6 +1299,7 @@ ActiveRecord::Schema.define(version: 20161221140236) do
add_foreign_key "personal_access_tokens", "users"
add_foreign_key "project_authorizations", "projects", on_delete: :cascade
add_foreign_key "project_authorizations", "users", on_delete: :cascade
+ add_foreign_key "project_statistics", "projects", on_delete: :cascade
add_foreign_key "protected_branch_merge_access_levels", "protected_branches"
add_foreign_key "protected_branch_push_access_levels", "protected_branches"
add_foreign_key "subscriptions", "projects", on_delete: :cascade
diff --git a/doc/administration/build_artifacts.md b/doc/administration/build_artifacts.md
index 3ba8387c7f0e7c14fa87dc16cb0953f1147c928d..cca422892ec62840fedd1e1dd1f80670950a53e2 100644
--- a/doc/administration/build_artifacts.md
+++ b/doc/administration/build_artifacts.md
@@ -88,3 +88,9 @@ artifacts through the [Admin area settings](../user/admin_area/settings/continuo
[reconfigure gitlab]: restart_gitlab.md "How to restart GitLab"
[restart gitlab]: restart_gitlab.md "How to restart GitLab"
+
+## Storage statistics
+
+You can see the total storage used for build artifacts on groups and projects
+in the administration area, as well as through the [groups](../api/groups.md)
+and [projects APIs](../api/projects.md).
diff --git a/doc/api/groups.md b/doc/api/groups.md
index 134d7bda22f19702441cbdd0d72fd391537b6444..bc737bff8eec9fa2a660da4345b67dc761449d91 100644
--- a/doc/api/groups.md
+++ b/doc/api/groups.md
@@ -13,6 +13,7 @@ Parameters:
| `search` | string | no | Return list of authorized groups matching the search criteria |
| `order_by` | string | no | Order groups by `name` or `path`. Default is `name` |
| `sort` | string | no | Order groups in `asc` or `desc` order. Default is `asc` |
+| `statistics` | boolean | no | Include group statistics (admins only) |
```
GET /groups
@@ -31,7 +32,6 @@ GET /groups
You can search for groups by name or path, see below.
-=======
## List owned groups
Get a list of groups which are owned by the authenticated user.
@@ -40,6 +40,12 @@ Get a list of groups which are owned by the authenticated user.
GET /groups/owned
```
+Parameters:
+
+| Attribute | Type | Required | Description |
+| --------- | ---- | -------- | ----------- |
+| `statistics` | boolean | no | Include group statistics |
+
## List a group's projects
Get a list of projects in this group.
diff --git a/doc/api/projects.md b/doc/api/projects.md
index edffad555a584270f364cc9a8d9dfe6845cc2c64..122075bbd115eabe4ec7776158b30073c87f7929 100644
--- a/doc/api/projects.md
+++ b/doc/api/projects.md
@@ -307,6 +307,8 @@ Parameters:
| `order_by` | string | no | Return projects ordered by `id`, `name`, `path`, `created_at`, `updated_at`, or `last_activity_at` fields. Default is `created_at` |
| `sort` | string | no | Return projects sorted in `asc` or `desc` order. Default is `desc` |
| `search` | string | no | Return list of authorized projects matching the search criteria |
+| `simple` | boolean | no | Return only the ID, URL, name, and path of each project |
+| `statistics` | boolean | no | Include project statistics |
### List starred projects
@@ -325,6 +327,7 @@ Parameters:
| `order_by` | string | no | Return projects ordered by `id`, `name`, `path`, `created_at`, `updated_at`, or `last_activity_at` fields. Default is `created_at` |
| `sort` | string | no | Return projects sorted in `asc` or `desc` order. Default is `desc` |
| `search` | string | no | Return list of authorized projects matching the search criteria |
+| `simple` | boolean | no | Return only the ID, URL, name, and path of each project |
### List ALL projects
@@ -343,6 +346,7 @@ Parameters:
| `order_by` | string | no | Return projects ordered by `id`, `name`, `path`, `created_at`, `updated_at`, or `last_activity_at` fields. Default is `created_at` |
| `sort` | string | no | Return projects sorted in `asc` or `desc` order. Default is `desc` |
| `search` | string | no | Return list of authorized projects matching the search criteria |
+| `statistics` | boolean | no | Include project statistics |
### Get single project
diff --git a/doc/workflow/lfs/lfs_administration.md b/doc/workflow/lfs/lfs_administration.md
index b3c73e947f03510e47c8a82e29b388d3d5edfb1e..5f6a718135d898bd051f896881663bb75af6629d 100644
--- a/doc/workflow/lfs/lfs_administration.md
+++ b/doc/workflow/lfs/lfs_administration.md
@@ -40,6 +40,12 @@ In `config/gitlab.yml`:
storage_path: /mnt/storage/lfs-objects
```
+## Storage statistics
+
+You can see the total storage used for LFS objects on groups and projects
+in the administration area, as well as through the [groups](../api/groups.md)
+and [projects APIs](../api/projects.md).
+
## Known limitations
* Currently, storing GitLab Git LFS objects on a non-local storage (like S3 buckets)
@@ -47,3 +53,5 @@ In `config/gitlab.yml`:
* Currently, removing LFS objects from GitLab Git LFS storage is not supported
* LFS authentications via SSH was added with GitLab 8.12
* Only compatible with the GitLFS client versions 1.1.0 and up, or 1.0.2.
+* The storage statistics currently count each LFS object multiple times for
+ every project linking to it
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index dfbb3ab86dd23c00f55c84a9e978290862068808..9f15c08f472730ff3a62c91f13bdcde5ec5e5f8b 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -78,21 +78,21 @@ module API
expose :container_registry_enabled
# Expose old field names with the new permissions methods to keep API compatible
- expose(:issues_enabled) { |project, options| project.feature_available?(:issues, options[:user]) }
- expose(:merge_requests_enabled) { |project, options| project.feature_available?(:merge_requests, options[:user]) }
- expose(:wiki_enabled) { |project, options| project.feature_available?(:wiki, options[:user]) }
- expose(:builds_enabled) { |project, options| project.feature_available?(:builds, options[:user]) }
- expose(:snippets_enabled) { |project, options| project.feature_available?(:snippets, options[:user]) }
+ expose(:issues_enabled) { |project, options| project.feature_available?(:issues, options[:current_user]) }
+ expose(:merge_requests_enabled) { |project, options| project.feature_available?(:merge_requests, options[:current_user]) }
+ expose(:wiki_enabled) { |project, options| project.feature_available?(:wiki, options[:current_user]) }
+ expose(:builds_enabled) { |project, options| project.feature_available?(:builds, options[:current_user]) }
+ expose(:snippets_enabled) { |project, options| project.feature_available?(:snippets, options[:current_user]) }
expose :created_at, :last_activity_at
expose :shared_runners_enabled
expose :lfs_enabled?, as: :lfs_enabled
expose :creator_id
- expose :namespace
+ expose :namespace, using: 'API::Entities::Namespace'
expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda{ |project, options| project.forked? }
expose :avatar_url
expose :star_count, :forks_count
- expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:user]) && project.default_issues_tracker? }
+ expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) && project.default_issues_tracker? }
expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] }
expose :public_builds
expose :shared_with_groups do |project, options|
@@ -101,6 +101,16 @@ module API
expose :only_allow_merge_if_build_succeeds
expose :request_access_enabled
expose :only_allow_merge_if_all_discussions_are_resolved
+
+ expose :statistics, using: 'API::Entities::ProjectStatistics', if: :statistics
+ end
+
+ class ProjectStatistics < Grape::Entity
+ expose :commit_count
+ expose :storage_size
+ expose :repository_size
+ expose :lfs_objects_size
+ expose :build_artifacts_size
end
class Member < UserBasic
@@ -127,6 +137,15 @@ module API
expose :avatar_url
expose :web_url
expose :request_access_enabled
+
+ expose :statistics, if: :statistics do
+ with_options format_with: -> (value) { value.to_i } do
+ expose :storage_size
+ expose :repository_size
+ expose :lfs_objects_size
+ expose :build_artifacts_size
+ end
+ end
end
class GroupDetail < Group
@@ -391,7 +410,7 @@ module API
end
class Namespace < Grape::Entity
- expose :id, :path, :kind
+ expose :id, :name, :path, :kind
end
class MemberAccess < Grape::Entity
@@ -440,12 +459,12 @@ module API
class ProjectWithAccess < Project
expose :permissions do
expose :project_access, using: Entities::ProjectAccess do |project, options|
- project.project_members.find_by(user_id: options[:user].id)
+ project.project_members.find_by(user_id: options[:current_user].id)
end
expose :group_access, using: Entities::GroupAccess do |project, options|
if project.group
- project.group.group_members.find_by(user_id: options[:user].id)
+ project.group.group_members.find_by(user_id: options[:current_user].id)
end
end
end
diff --git a/lib/api/groups.rb b/lib/api/groups.rb
index 9b9d3df74359a97aee4dceda1588b92d0a116077..e04d2e40fb6f6ed2886d84fe962f2261939f530e 100644
--- a/lib/api/groups.rb
+++ b/lib/api/groups.rb
@@ -11,6 +11,20 @@ module API
optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group'
optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access'
end
+
+ params :statistics_params do
+ optional :statistics, type: Boolean, default: false, desc: 'Include project statistics'
+ end
+
+ def present_groups(groups, options = {})
+ options = options.reverse_merge(
+ with: Entities::Group,
+ current_user: current_user,
+ )
+
+ groups = groups.with_statistics if options[:statistics]
+ present paginate(groups), options
+ end
end
resource :groups do
@@ -18,6 +32,7 @@ module API
success Entities::Group
end
params do
+ use :statistics_params
optional :skip_groups, type: Array[Integer], desc: 'Array of group ids to exclude from list'
optional :all_available, type: Boolean, desc: 'Show all group that you have access to'
optional :search, type: String, desc: 'Search for a specific group'
@@ -38,7 +53,7 @@ module API
groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present?
groups = groups.reorder(params[:order_by] => params[:sort])
- present paginate(groups), with: Entities::Group
+ present_groups groups, statistics: params[:statistics] && current_user.is_admin?
end
desc 'Get list of owned groups for authenticated user' do
@@ -46,10 +61,10 @@ module API
end
params do
use :pagination
+ use :statistics_params
end
get '/owned' do
- groups = current_user.owned_groups
- present paginate(groups), with: Entities::Group, user: current_user
+ present_groups current_user.owned_groups, statistics: params[:statistics]
end
desc 'Create a group. Available only for users who can create groups.' do
@@ -66,7 +81,7 @@ module API
group = ::Groups::CreateService.new(current_user, declared_params(include_missing: false)).execute
if group.persisted?
- present group, with: Entities::Group
+ present group, with: Entities::Group, current_user: current_user
else
render_api_error!("Failed to save group #{group.errors.messages}", 400)
end
@@ -92,7 +107,7 @@ module API
authorize! :admin_group, group
if ::Groups::UpdateService.new(group, current_user, declared_params(include_missing: false)).execute
- present group, with: Entities::GroupDetail
+ present group, with: Entities::GroupDetail, current_user: current_user
else
render_validation_error!(group)
end
@@ -103,7 +118,7 @@ module API
end
get ":id" do
group = find_group!(params[:id])
- present group, with: Entities::GroupDetail
+ present group, with: Entities::GroupDetail, current_user: current_user
end
desc 'Remove a group.'
@@ -134,7 +149,7 @@ module API
projects = GroupProjectsFinder.new(group).execute(current_user)
projects = filter_projects(projects)
entity = params[:simple] ? Entities::BasicProjectDetails : Entities::Project
- present paginate(projects), with: entity, user: current_user
+ present paginate(projects), with: entity, current_user: current_user
end
desc 'Transfer a project to the group namespace. Available only for admin.' do
@@ -150,7 +165,7 @@ module API
result = ::Projects::TransferService.new(project, current_user).execute(group)
if result
- present group, with: Entities::GroupDetail
+ present group, with: Entities::GroupDetail, current_user: current_user
else
render_api_error!("Failed to transfer project #{project.errors.messages}", 400)
end
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 4be659fc20bd50df9a9ba4ff8b071b2e7bfafd25..fe00c83bff3d71a633e714009e4d40191ea6c05a 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -248,7 +248,7 @@ module API
rack_response({ 'message' => '500 Internal Server Error' }.to_json, 500)
end
- # Projects helpers
+ # project helpers
def filter_projects(projects)
if params[:search].present?
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index 2929d2157dc72b16fac0b4fdb5c22b189f3e20cd..3be14e8eb763cb6d95614c1f4ba11fb2ff7973bc 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -40,6 +40,15 @@ module API
resource :projects do
helpers do
+ params :collection_params do
+ use :sort_params
+ use :filter_params
+ use :pagination
+
+ optional :simple, type: Boolean, default: false,
+ desc: 'Return only the ID, URL, name, and path of each project'
+ end
+
params :sort_params do
optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at],
default: 'created_at', desc: 'Return projects ordered by field'
@@ -52,97 +61,94 @@ module API
optional :visibility, type: String, values: %w[public internal private],
desc: 'Limit by visibility'
optional :search, type: String, desc: 'Return list of authorized projects matching the search criteria'
- use :sort_params
+ end
+
+ params :statistics_params do
+ optional :statistics, type: Boolean, default: false, desc: 'Include project statistics'
end
params :create_params do
optional :namespace_id, type: Integer, desc: 'Namespace ID for the new project. Default to the user namespace.'
optional :import_url, type: String, desc: 'URL from which the project is imported'
end
+
+ def present_projects(projects, options = {})
+ options = options.reverse_merge(
+ with: Entities::Project,
+ current_user: current_user,
+ simple: params[:simple],
+ )
+
+ projects = filter_projects(projects)
+ projects = projects.with_statistics if options[:statistics]
+ options[:with] = Entities::BasicProjectDetails if options[:simple]
+
+ present paginate(projects), options
+ end
end
desc 'Get a list of visible projects for authenticated user' do
success Entities::BasicProjectDetails
end
params do
- optional :simple, type: Boolean, default: false,
- desc: 'Return only the ID, URL, name, and path of each project'
- use :filter_params
- use :pagination
+ use :collection_params
end
get '/visible' do
- projects = ProjectsFinder.new.execute(current_user)
- projects = filter_projects(projects)
- entity = params[:simple] || !current_user ? Entities::BasicProjectDetails : Entities::ProjectWithAccess
-
- present paginate(projects), with: entity, user: current_user
+ entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails
+ present_projects ProjectsFinder.new.execute(current_user), with: entity
end
desc 'Get a projects list for authenticated user' do
success Entities::BasicProjectDetails
end
params do
- optional :simple, type: Boolean, default: false,
- desc: 'Return only the ID, URL, name, and path of each project'
- use :filter_params
- use :pagination
+ use :collection_params
end
get do
authenticate!
- projects = current_user.authorized_projects
- projects = filter_projects(projects)
- entity = params[:simple] ? Entities::BasicProjectDetails : Entities::ProjectWithAccess
-
- present paginate(projects), with: entity, user: current_user
+ present_projects current_user.authorized_projects,
+ with: Entities::ProjectWithAccess
end
desc 'Get an owned projects list for authenticated user' do
success Entities::BasicProjectDetails
end
params do
- use :filter_params
- use :pagination
+ use :collection_params
+ use :statistics_params
end
get '/owned' do
authenticate!
- projects = current_user.owned_projects
- projects = filter_projects(projects)
-
- present paginate(projects), with: Entities::ProjectWithAccess, user: current_user
+ present_projects current_user.owned_projects,
+ with: Entities::ProjectWithAccess,
+ statistics: params[:statistics]
end
desc 'Gets starred project for the authenticated user' do
success Entities::BasicProjectDetails
end
params do
- use :filter_params
- use :pagination
+ use :collection_params
end
get '/starred' do
authenticate!
- projects = current_user.viewable_starred_projects
- projects = filter_projects(projects)
-
- present paginate(projects), with: Entities::Project, user: current_user
+ present_projects current_user.viewable_starred_projects
end
desc 'Get all projects for admin user' do
success Entities::BasicProjectDetails
end
params do
- use :filter_params
- use :pagination
+ use :collection_params
+ use :statistics_params
end
get '/all' do
authenticated_as_admin!
- projects = Project.all
- projects = filter_projects(projects)
-
- present paginate(projects), with: Entities::ProjectWithAccess, user: current_user
+ present_projects Project.all, with: Entities::ProjectWithAccess, statistics: params[:statistics]
end
desc 'Search for projects the current user has access to' do
@@ -221,7 +227,7 @@ module API
end
get ":id" do
entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails
- present user_project, with: entity, user: current_user,
+ present user_project, with: entity, current_user: current_user,
user_can_admin_project: can?(current_user, :admin_project, user_project)
end
diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake
index dbdd4e977e8becc230551f32abbda26a691ab8b6..a2eca74a3c833bddde99d6b9a2e704eb65759182 100644
--- a/lib/tasks/gitlab/import.rake
+++ b/lib/tasks/gitlab/import.rake
@@ -63,8 +63,7 @@ namespace :gitlab do
if project.persisted?
puts " * Created #{project.name} (#{repo_path})".color(:green)
- project.update_repository_size
- project.update_commit_count
+ ProjectCacheWorker.perform(project.id)
else
puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red)
puts " Errors: #{project.errors.messages}".color(:red)
diff --git a/lib/tasks/gitlab/update_commit_count.rake b/lib/tasks/gitlab/update_commit_count.rake
deleted file mode 100644
index 3bd10b0208bf7fa78cc40c784d871715c8767047..0000000000000000000000000000000000000000
--- a/lib/tasks/gitlab/update_commit_count.rake
+++ /dev/null
@@ -1,20 +0,0 @@
-namespace :gitlab do
- desc "GitLab | Update commit count for projects"
- task update_commit_count: :environment do
- projects = Project.where(commit_count: 0)
- puts "#{projects.size} projects need to be updated. This might take a while."
- ask_to_continue unless ENV['force'] == 'yes'
-
- projects.find_each(batch_size: 100) do |project|
- print "#{project.name_with_namespace.color(:yellow)} ... "
-
- unless project.repo_exists?
- puts "skipping, because the repo is empty".color(:magenta)
- next
- end
-
- project.update_commit_count
- puts project.commit_count.to_s.color(:green)
- end
- end
-end
diff --git a/spec/factories/lfs_objects.rb b/spec/factories/lfs_objects.rb
index a81645acd2b86814489de798a3c2d9fb8534a52a..477fab9e964d8c145e48e1032ac74541dbf64dd6 100644
--- a/spec/factories/lfs_objects.rb
+++ b/spec/factories/lfs_objects.rb
@@ -2,7 +2,7 @@ include ActionDispatch::TestProcess
FactoryGirl.define do
factory :lfs_object do
- oid "b68143e6463773b1b6c6fd009a76c32aeec041faff32ba2ed42fd7f708a17f80"
+ sequence(:oid) { |n| "b68143e6463773b1b6c6fd009a76c32aeec041faff32ba2ed42fd7f708a%05x" % n }
size 499013
end
diff --git a/spec/factories/project_statistics.rb b/spec/factories/project_statistics.rb
new file mode 100644
index 0000000000000000000000000000000000000000..72d4309621629f6b9748afab3ab0341b2ab2ca0a
--- /dev/null
+++ b/spec/factories/project_statistics.rb
@@ -0,0 +1,6 @@
+FactoryGirl.define do
+ factory :project_statistics do
+ project { create :project }
+ namespace { project.namespace }
+ end
+end
diff --git a/spec/helpers/storage_helper_spec.rb b/spec/helpers/storage_helper_spec.rb
new file mode 100644
index 0000000000000000000000000000000000000000..4627a1e1872668693adb35d2a96cc4c9a40a90e1
--- /dev/null
+++ b/spec/helpers/storage_helper_spec.rb
@@ -0,0 +1,21 @@
+require 'spec_helper'
+
+describe StorageHelper do
+ describe '#storage_counter' do
+ it 'formats bytes to one decimal place' do
+ expect(helper.storage_counter(1.23.megabytes)).to eq '1.2 MB'
+ end
+
+ it 'does not add decimals for sizes < 1 MB' do
+ expect(helper.storage_counter(23.5.kilobytes)).to eq '24 KB'
+ end
+
+ it 'does not add decimals for zeroes' do
+ expect(helper.storage_counter(2.megabytes)).to eq '2 MB'
+ end
+
+ it 'uses commas as thousands separator' do
+ expect(helper.storage_counter(100_000_000_000_000_000)).to eq '90,949.5 TB'
+ end
+ end
+end
diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml
index f420d71dee2dcd3972cd92f7b5e50ab7104008fa..ceed9c942c11bddbedc517e0ea0bd86f512b7184 100644
--- a/spec/lib/gitlab/import_export/all_models.yml
+++ b/spec/lib/gitlab/import_export/all_models.yml
@@ -192,6 +192,7 @@ project:
- authorized_users
- project_authorizations
- route
+- statistics
award_emoji:
- awardable
- user
diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb
index a7e90c8a3819d3fc1a35b1a1609095c856530384..7e1d1126b972eafcf1cf589c733c7cb8921d5568 100644
--- a/spec/models/ci/build_spec.rb
+++ b/spec/models/ci/build_spec.rb
@@ -85,4 +85,30 @@ describe Ci::Build, models: true do
it { expect(build.trace_file_path).to eq(build.old_path_to_trace) }
end
end
+
+ describe '#update_project_statistics' do
+ let!(:build) { create(:ci_build, artifacts_size: 23) }
+
+ it 'updates project statistics when the artifact size changes' do
+ expect(ProjectCacheWorker).to receive(:perform_async)
+ .with(build.project_id, [], [:build_artifacts_size])
+
+ build.artifacts_size = 42
+ build.save!
+ end
+
+ it 'does not update project statistics when the artifact size stays the same' do
+ expect(ProjectCacheWorker).not_to receive(:perform_async)
+
+ build.name = 'changed'
+ build.save!
+ end
+
+ it 'updates project statistics when the build is destroyed' do
+ expect(ProjectCacheWorker).to receive(:perform_async)
+ .with(build.project_id, [], [:build_artifacts_size])
+
+ build.destroy
+ end
+ end
end
diff --git a/spec/models/lfs_objects_project_spec.rb b/spec/models/lfs_objects_project_spec.rb
new file mode 100644
index 0000000000000000000000000000000000000000..7bc278e350f6abe9953d16dbb4d8d1adf032d4dc
--- /dev/null
+++ b/spec/models/lfs_objects_project_spec.rb
@@ -0,0 +1,36 @@
+require 'spec_helper'
+
+describe LfsObjectsProject, models: true do
+ subject { create(:lfs_objects_project, project: project) }
+ let(:project) { create(:empty_project) }
+
+ describe 'associations' do
+ it { is_expected.to belong_to(:project) }
+ it { is_expected.to belong_to(:lfs_object) }
+ end
+
+ describe 'validation' do
+ it { is_expected.to validate_presence_of(:lfs_object_id) }
+ it { is_expected.to validate_uniqueness_of(:lfs_object_id).scoped_to(:project_id).with_message("already exists in project") }
+
+ it { is_expected.to validate_presence_of(:project_id) }
+ end
+
+ describe '#update_project_statistics' do
+ it 'updates project statistics when the object is added' do
+ expect(ProjectCacheWorker).to receive(:perform_async)
+ .with(project.id, [], [:lfs_objects_size])
+
+ subject.save!
+ end
+
+ it 'updates project statistics when the object is removed' do
+ subject.save!
+
+ expect(ProjectCacheWorker).to receive(:perform_async)
+ .with(project.id, [], [:lfs_objects_size])
+
+ subject.destroy
+ end
+ end
+end
diff --git a/spec/models/namespace_spec.rb b/spec/models/namespace_spec.rb
index 9fd06bb6b23030fa73b269dd516e5db0291e8f86..600538ff5f435f01e6b2d9b6bbe858efeaa200eb 100644
--- a/spec/models/namespace_spec.rb
+++ b/spec/models/namespace_spec.rb
@@ -4,6 +4,7 @@ describe Namespace, models: true do
let!(:namespace) { create(:namespace) }
it { is_expected.to have_many :projects }
+ it { is_expected.to have_many :project_statistics }
it { is_expected.to validate_presence_of(:name) }
it { is_expected.to validate_uniqueness_of(:name).scoped_to(:parent_id) }
@@ -57,6 +58,50 @@ describe Namespace, models: true do
end
end
+ describe '.with_statistics' do
+ let(:namespace) { create :namespace }
+
+ let(:project1) do
+ create(:empty_project,
+ namespace: namespace,
+ statistics: build(:project_statistics,
+ storage_size: 606,
+ repository_size: 101,
+ lfs_objects_size: 202,
+ build_artifacts_size: 303))
+ end
+
+ let(:project2) do
+ create(:empty_project,
+ namespace: namespace,
+ statistics: build(:project_statistics,
+ storage_size: 60,
+ repository_size: 10,
+ lfs_objects_size: 20,
+ build_artifacts_size: 30))
+ end
+
+ it "sums all project storage counters in the namespace" do
+ project1
+ project2
+ statistics = Namespace.with_statistics.find(namespace.id)
+
+ expect(statistics.storage_size).to eq 666
+ expect(statistics.repository_size).to eq 111
+ expect(statistics.lfs_objects_size).to eq 222
+ expect(statistics.build_artifacts_size).to eq 333
+ end
+
+ it "correctly handles namespaces without projects" do
+ statistics = Namespace.with_statistics.find(namespace.id)
+
+ expect(statistics.storage_size).to eq 0
+ expect(statistics.repository_size).to eq 0
+ expect(statistics.lfs_objects_size).to eq 0
+ expect(statistics.build_artifacts_size).to eq 0
+ end
+ end
+
describe '#move_dir' do
before do
@namespace = create :namespace
diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb
index 88d5d14f855ea5e4ba9202a726d93cd196ad37fb..fb225eb76259b64b5b63412ff2ab6f04e49f2e80 100644
--- a/spec/models/project_spec.rb
+++ b/spec/models/project_spec.rb
@@ -49,6 +49,7 @@ describe Project, models: true do
it { is_expected.to have_one(:gitlab_issue_tracker_service).dependent(:destroy) }
it { is_expected.to have_one(:external_wiki_service).dependent(:destroy) }
it { is_expected.to have_one(:project_feature).dependent(:destroy) }
+ it { is_expected.to have_one(:statistics).class_name('ProjectStatistics').dependent(:delete) }
it { is_expected.to have_one(:import_data).class_name('ProjectImportData').dependent(:destroy) }
it { is_expected.to have_one(:last_event).class_name('Event') }
it { is_expected.to have_one(:forked_from_project).through(:forked_project_link) }
@@ -1729,6 +1730,26 @@ describe Project, models: true do
end
end
+ describe '#update_project_statistics' do
+ let(:project) { create(:empty_project) }
+
+ it "is called after creation" do
+ expect(project.statistics).to be_a ProjectStatistics
+ expect(project.statistics).to be_persisted
+ end
+
+ it "copies the namespace_id" do
+ expect(project.statistics.namespace_id).to eq project.namespace_id
+ end
+
+ it "updates the namespace_id when changed" do
+ namespace = create(:namespace)
+ project.update(namespace: namespace)
+
+ expect(project.statistics.namespace_id).to eq namespace.id
+ end
+ end
+
def enable_lfs
allow(Gitlab.config.lfs).to receive(:enabled).and_return(true)
end
diff --git a/spec/models/project_statistics_spec.rb b/spec/models/project_statistics_spec.rb
new file mode 100644
index 0000000000000000000000000000000000000000..77403cc9eb0ef6b692c424096310a6b85f3027da
--- /dev/null
+++ b/spec/models/project_statistics_spec.rb
@@ -0,0 +1,160 @@
+require 'rails_helper'
+
+describe ProjectStatistics, models: true do
+ let(:project) { create :empty_project }
+ let(:statistics) { project.statistics }
+
+ describe 'constants' do
+ describe 'STORAGE_COLUMNS' do
+ it 'is an array of symbols' do
+ expect(described_class::STORAGE_COLUMNS).to be_kind_of Array
+ expect(described_class::STORAGE_COLUMNS.map(&:class).uniq).to eq [Symbol]
+ end
+ end
+
+ describe 'STATISTICS_COLUMNS' do
+ it 'is an array of symbols' do
+ expect(described_class::STATISTICS_COLUMNS).to be_kind_of Array
+ expect(described_class::STATISTICS_COLUMNS.map(&:class).uniq).to eq [Symbol]
+ end
+
+ it 'includes all storage columns' do
+ expect(described_class::STATISTICS_COLUMNS & described_class::STORAGE_COLUMNS).to eq described_class::STORAGE_COLUMNS
+ end
+ end
+ end
+
+ describe 'associations' do
+ it { is_expected.to belong_to(:project) }
+ it { is_expected.to belong_to(:namespace) }
+ end
+
+ describe 'statistics columns' do
+ it "support values up to 8 exabytes" do
+ statistics.update!(
+ commit_count: 8.exabytes - 1,
+ repository_size: 2.exabytes,
+ lfs_objects_size: 2.exabytes,
+ build_artifacts_size: 4.exabytes - 1,
+ )
+
+ statistics.reload
+
+ expect(statistics.commit_count).to eq(8.exabytes - 1)
+ expect(statistics.repository_size).to eq(2.exabytes)
+ expect(statistics.lfs_objects_size).to eq(2.exabytes)
+ expect(statistics.build_artifacts_size).to eq(4.exabytes - 1)
+ expect(statistics.storage_size).to eq(8.exabytes - 1)
+ end
+ end
+
+ describe '#total_repository_size' do
+ it "sums repository and LFS object size" do
+ statistics.repository_size = 2
+ statistics.lfs_objects_size = 3
+ statistics.build_artifacts_size = 4
+
+ expect(statistics.total_repository_size).to eq 5
+ end
+ end
+
+ describe '#refresh!' do
+ before do
+ allow(statistics).to receive(:update_commit_count)
+ allow(statistics).to receive(:update_repository_size)
+ allow(statistics).to receive(:update_lfs_objects_size)
+ allow(statistics).to receive(:update_build_artifacts_size)
+ allow(statistics).to receive(:update_storage_size)
+ end
+
+ context "without arguments" do
+ before do
+ statistics.refresh!
+ end
+
+ it "sums all counters" do
+ expect(statistics).to have_received(:update_commit_count)
+ expect(statistics).to have_received(:update_repository_size)
+ expect(statistics).to have_received(:update_lfs_objects_size)
+ expect(statistics).to have_received(:update_build_artifacts_size)
+ end
+ end
+
+ context "when passing an only: argument" do
+ before do
+ statistics.refresh! only: [:lfs_objects_size]
+ end
+
+ it "only updates the given columns" do
+ expect(statistics).to have_received(:update_lfs_objects_size)
+ expect(statistics).not_to have_received(:update_commit_count)
+ expect(statistics).not_to have_received(:update_repository_size)
+ expect(statistics).not_to have_received(:update_build_artifacts_size)
+ end
+ end
+ end
+
+ describe '#update_commit_count' do
+ before do
+ allow(project.repository).to receive(:commit_count).and_return(23)
+ statistics.update_commit_count
+ end
+
+ it "stores the number of commits in the repository" do
+ expect(statistics.commit_count).to eq 23
+ end
+ end
+
+ describe '#update_repository_size' do
+ before do
+ allow(project.repository).to receive(:size).and_return(12.megabytes)
+ statistics.update_repository_size
+ end
+
+ it "stores the size of the repository" do
+ expect(statistics.repository_size).to eq 12.megabytes
+ end
+ end
+
+ describe '#update_lfs_objects_size' do
+ let!(:lfs_object1) { create(:lfs_object, size: 23.megabytes) }
+ let!(:lfs_object2) { create(:lfs_object, size: 34.megabytes) }
+ let!(:lfs_objects_project1) { create(:lfs_objects_project, project: project, lfs_object: lfs_object1) }
+ let!(:lfs_objects_project2) { create(:lfs_objects_project, project: project, lfs_object: lfs_object2) }
+
+ before do
+ statistics.update_lfs_objects_size
+ end
+
+ it "stores the size of related LFS objects" do
+ expect(statistics.lfs_objects_size).to eq 57.megabytes
+ end
+ end
+
+ describe '#update_build_artifacts_size' do
+ let!(:pipeline) { create(:ci_pipeline, project: project) }
+ let!(:build1) { create(:ci_build, pipeline: pipeline, artifacts_size: 45.megabytes) }
+ let!(:build2) { create(:ci_build, pipeline: pipeline, artifacts_size: 56.megabytes) }
+
+ before do
+ statistics.update_build_artifacts_size
+ end
+
+ it "stores the size of related build artifacts" do
+ expect(statistics.build_artifacts_size).to eq 101.megabytes
+ end
+ end
+
+ describe '#update_storage_size' do
+ it "sums all storage counters" do
+ statistics.update!(
+ repository_size: 2,
+ lfs_objects_size: 3,
+ )
+
+ statistics.reload
+
+ expect(statistics.storage_size).to eq 5
+ end
+ end
+end
diff --git a/spec/requests/api/groups_spec.rb b/spec/requests/api/groups_spec.rb
index cdeb965b413140d059f4ad3126df97f826dc8d60..0e8d6faea2731eabcecb541c7292e04bdfc13f1d 100644
--- a/spec/requests/api/groups_spec.rb
+++ b/spec/requests/api/groups_spec.rb
@@ -35,6 +35,14 @@ describe API::Groups, api: true do
expect(json_response.length).to eq(1)
expect(json_response.first['name']).to eq(group1.name)
end
+
+ it "does not include statistics" do
+ get api("/groups", user1), statistics: true
+
+ expect(response).to have_http_status(200)
+ expect(json_response).to be_an Array
+ expect(json_response.first).not_to include 'statistics'
+ end
end
context "when authenticated as admin" do
@@ -44,6 +52,31 @@ describe API::Groups, api: true do
expect(json_response).to be_an Array
expect(json_response.length).to eq(2)
end
+
+ it "does not include statistics by default" do
+ get api("/groups", admin)
+
+ expect(response).to have_http_status(200)
+ expect(json_response).to be_an Array
+ expect(json_response.first).not_to include('statistics')
+ end
+
+ it "includes statistics if requested" do
+ attributes = {
+ storage_size: 702,
+ repository_size: 123,
+ lfs_objects_size: 234,
+ build_artifacts_size: 345,
+ }
+
+ project1.statistics.update!(attributes)
+
+ get api("/groups", admin), statistics: true
+
+ expect(response).to have_http_status(200)
+ expect(json_response).to be_an Array
+ expect(json_response.first['statistics']).to eq attributes.stringify_keys
+ end
end
context "when using skip_groups in request" do
diff --git a/spec/requests/api/projects_spec.rb b/spec/requests/api/projects_spec.rb
index 8304c4080646389b89031030777884b9d2a46824..f5788d15f9378d023f925e9ab86b955f0d37f1f4 100644
--- a/spec/requests/api/projects_spec.rb
+++ b/spec/requests/api/projects_spec.rb
@@ -49,7 +49,7 @@ describe API::Projects, api: true do
end
end
- context 'when authenticated' do
+ context 'when authenticated as regular user' do
it 'returns an array of projects' do
get api('/projects', user)
expect(response).to have_http_status(200)
@@ -172,6 +172,22 @@ describe API::Projects, api: true do
end
end
end
+
+ it "does not include statistics by default" do
+ get api('/projects/all', admin)
+
+ expect(response).to have_http_status(200)
+ expect(json_response).to be_an Array
+ expect(json_response.first).not_to include('statistics')
+ end
+
+ it "includes statistics if requested" do
+ get api('/projects/all', admin), statistics: true
+
+ expect(response).to have_http_status(200)
+ expect(json_response).to be_an Array
+ expect(json_response.first).to include 'statistics'
+ end
end
end
@@ -196,6 +212,32 @@ describe API::Projects, api: true do
expect(json_response.first['name']).to eq(project4.name)
expect(json_response.first['owner']['username']).to eq(user4.username)
end
+
+ it "does not include statistics by default" do
+ get api('/projects/owned', user4)
+
+ expect(response).to have_http_status(200)
+ expect(json_response).to be_an Array
+ expect(json_response.first).not_to include('statistics')
+ end
+
+ it "includes statistics if requested" do
+ attributes = {
+ commit_count: 23,
+ storage_size: 702,
+ repository_size: 123,
+ lfs_objects_size: 234,
+ build_artifacts_size: 345,
+ }
+
+ project4.statistics.update!(attributes)
+
+ get api('/projects/owned', user4), statistics: true
+
+ expect(response).to have_http_status(200)
+ expect(json_response).to be_an Array
+ expect(json_response.first['statistics']).to eq attributes.stringify_keys
+ end
end
end
@@ -630,6 +672,18 @@ describe API::Projects, api: true do
expect(json_response['name']).to eq(project.name)
end
+ it 'exposes namespace fields' do
+ get api("/projects/#{project.id}", user)
+
+ expect(response).to have_http_status(200)
+ expect(json_response['namespace']).to eq({
+ 'id' => user.namespace.id,
+ 'name' => user.namespace.name,
+ 'path' => user.namespace.path,
+ 'kind' => user.namespace.kind,
+ })
+ end
+
describe 'permissions' do
context 'all projects' do
before { project.team << [user, :master] }
diff --git a/spec/services/git_push_service_spec.rb b/spec/services/git_push_service_spec.rb
index 3303e808a9caefe15680a8799bb405f7d844352b..2a0f00ce93721f49e203d9a5890e7f3e81929071 100644
--- a/spec/services/git_push_service_spec.rb
+++ b/spec/services/git_push_service_spec.rb
@@ -583,7 +583,7 @@ describe GitPushService, services: true do
service.push_commits = [commit]
expect(ProjectCacheWorker).to receive(:perform_async).
- with(project.id, %i(readme))
+ with(project.id, %i(readme), %i(commit_count repository_size))
service.update_caches
end
@@ -596,7 +596,7 @@ describe GitPushService, services: true do
it 'does not flush any conditional caches' do
expect(ProjectCacheWorker).to receive(:perform_async).
- with(project.id, []).
+ with(project.id, [], %i(commit_count repository_size)).
and_call_original
service.update_caches
diff --git a/spec/workers/project_cache_worker_spec.rb b/spec/workers/project_cache_worker_spec.rb
index 855c28b584e0ee3a2297b608757c3598b9a45b9a..f4f63b57a5fb26afc31714395391e308e439f7b7 100644
--- a/spec/workers/project_cache_worker_spec.rb
+++ b/spec/workers/project_cache_worker_spec.rb
@@ -1,8 +1,9 @@
require 'spec_helper'
describe ProjectCacheWorker do
- let(:project) { create(:project) }
let(:worker) { described_class.new }
+ let(:project) { create(:project) }
+ let(:statistics) { project.statistics }
describe '#perform' do
before do
@@ -12,7 +13,7 @@ describe ProjectCacheWorker do
context 'with a non-existing project' do
it 'does nothing' do
- expect(worker).not_to receive(:update_repository_size)
+ expect(worker).not_to receive(:update_statistics)
worker.perform(-1)
end
@@ -22,24 +23,19 @@ describe ProjectCacheWorker do
it 'does nothing' do
allow_any_instance_of(Repository).to receive(:exists?).and_return(false)
- expect(worker).not_to receive(:update_repository_size)
+ expect(worker).not_to receive(:update_statistics)
worker.perform(project.id)
end
end
context 'with an existing project' do
- it 'updates the repository size' do
- expect(worker).to receive(:update_repository_size).and_call_original
-
- worker.perform(project.id)
- end
-
- it 'updates the commit count' do
- expect_any_instance_of(Project).to receive(:update_commit_count).
- and_call_original
+ it 'updates the project statistics' do
+ expect(worker).to receive(:update_statistics)
+ .with(kind_of(Project), %i(repository_size))
+ .and_call_original
- worker.perform(project.id)
+ worker.perform(project.id, [], %w(repository_size))
end
it 'refreshes the method caches' do
@@ -47,33 +43,35 @@ describe ProjectCacheWorker do
with(%i(readme)).
and_call_original
- worker.perform(project.id, %i(readme))
+ worker.perform(project.id, %w(readme))
end
end
end
- describe '#update_repository_size' do
+ describe '#update_statistics' do
context 'when a lease could not be obtained' do
it 'does not update the repository size' do
allow(worker).to receive(:try_obtain_lease_for).
- with(project.id, :update_repository_size).
+ with(project.id, :update_statistics).
and_return(false)
- expect(project).not_to receive(:update_repository_size)
+ expect(statistics).not_to receive(:refresh!)
- worker.update_repository_size(project)
+ worker.update_statistics(project)
end
end
context 'when a lease could be obtained' do
- it 'updates the repository size' do
+ it 'updates the project statistics' do
allow(worker).to receive(:try_obtain_lease_for).
- with(project.id, :update_repository_size).
+ with(project.id, :update_statistics).
and_return(true)
- expect(project).to receive(:update_repository_size).and_call_original
+ expect(statistics).to receive(:refresh!)
+ .with(only: %i(repository_size))
+ .and_call_original
- worker.update_repository_size(project)
+ worker.update_statistics(project, %i(repository_size))
end
end
end