Skip to content
Snippets Groups Projects
Commit 045d07ba authored by Kamil Trzcińśki's avatar Kamil Trzcińśki
Browse files

Add Container Registry API

This includes a set of APIs to manipulate container registry.
This includes also an ability to delete tags based on requested
criteria, like keep-last-n, matching-name, older-than.
parent 267ce96e
No related branches found
No related tags found
No related merge requests found
Showing
with 600 additions and 22 deletions
# frozen_string_literal: true
 
class ContainerRepository < ActiveRecord::Base
include Gitlab::Utils::StrongMemoize
belongs_to :project
 
validates :name, length: { minimum: 0, allow_nil: false }
Loading
Loading
@@ -8,6 +10,8 @@ class ContainerRepository < ActiveRecord::Base
 
delegate :client, to: :registry
 
scope :ordered, -> { order(:name) }
# rubocop: disable CodeReuse/ServiceClass
def registry
@registry ||= begin
Loading
Loading
@@ -39,11 +43,12 @@ class ContainerRepository < ActiveRecord::Base
end
 
def tags
return @tags if defined?(@tags)
return [] unless manifest && manifest['tags']
 
@tags = manifest['tags'].map do |tag|
ContainerRegistry::Tag.new(self, tag)
strong_memoize(:tags) do
manifest['tags'].sort.map do |tag|
ContainerRegistry::Tag.new(self, tag)
end
end
end
 
Loading
Loading
# frozen_string_literal: true
class ContainerRepositoryPolicy < BasePolicy
delegate { @subject.project }
end
Loading
Loading
@@ -3,7 +3,7 @@
class ContainerRepositoryEntity < Grape::Entity
include RequestAwareEntity
 
expose :id, :path, :location
expose :id, :name, :path, :location, :created_at
 
expose :tags_path do |repository|
project_registry_repository_tags_path(project, repository, format: :json)
Loading
Loading
Loading
Loading
@@ -3,7 +3,7 @@
class ContainerTagEntity < Grape::Entity
include RequestAwareEntity
 
expose :name, :location, :revision, :short_revision, :total_size, :created_at
expose :name, :path, :location, :digest, :revision, :short_revision, :total_size, :created_at
 
expose :destroy_path, if: -> (*) { can_destroy? } do |tag|
project_registry_repository_tag_path(project, tag.repository, tag.name)
Loading
Loading
Loading
Loading
@@ -6,9 +6,14 @@
#
# `#try_obtain_lease` takes a block which will be run if it was able to
# obtain the lease. Implement `#lease_timeout` to configure the timeout
# for the exclusive lease. Optionally override `#lease_key` to set the
# for the exclusive lease.
#
# Optionally override `#lease_key` to set the
# lease key, it defaults to the class name with underscores.
#
# Optionally override `#lease_release?` to prevent the job to
# be re-executed more often than LEASE_TIMEOUT.
#
module ExclusiveLeaseGuard
extend ActiveSupport::Concern
 
Loading
Loading
@@ -23,7 +28,7 @@ module ExclusiveLeaseGuard
begin
yield lease
ensure
release_lease(lease)
release_lease(lease) if lease_release?
end
end
 
Loading
Loading
@@ -40,6 +45,10 @@ module ExclusiveLeaseGuard
"#{self.class.name} does not implement #{__method__}"
end
 
def lease_release?
true
end
def release_lease(uuid)
Gitlab::ExclusiveLease.cancel(lease_key, uuid)
end
Loading
Loading
# frozen_string_literal: true
module Projects
module ContainerRepository
class CleanupTagsService < BaseService
def execute(container_repository)
return error('feature disabled') unless can_use?
return error('access denied') unless can_admin?
tags = container_repository.tags
tags_by_digest = group_by_digest(tags)
tags = without_latest(tags)
tags = filter_by_name(tags)
tags = with_manifest(tags)
tags = order_by_date(tags)
tags = filter_keep_n(tags)
tags = filter_by_older_than(tags)
deleted_tags = delete_tags(tags, tags_by_digest)
success(deleted: deleted_tags.map(&:name))
end
private
def delete_tags(tags_to_delete, tags_by_digest)
deleted_digests = group_by_digest(tags_to_delete).select do |digest, tags|
delete_tag_digest(digest, tags, tags_by_digest[digest])
end
deleted_digests.values.flatten
end
def delete_tag_digest(digest, tags, other_tags)
# Issue: https://gitlab.com/gitlab-org/gitlab-ce/issues/21405
# we have to remove all tags due
# to Docker Distribution bug unable
# to delete single tag
return unless tags.count == other_tags.count
# delete all tags
tags.map(&:delete)
end
def group_by_digest(tags)
tags.group_by(&:digest)
end
def without_latest(tags)
tags.reject(&:latest?)
end
def with_manifest(tags)
tags.select(&:valid?)
end
def order_by_date(tags)
now = DateTime.now
tags.sort_by { |tag| tag.created_at || now }.reverse
end
def filter_by_name(tags)
regex = Gitlab::UntrustedRegexp.new("\\A#{params['name_regex']}\\z")
tags.select do |tag|
regex.scan(tag.name).any?
end
end
def filter_keep_n(tags)
tags.drop(params['keep_n'].to_i)
end
def filter_by_older_than(tags)
return tags unless params['older_than']
older_than = ChronicDuration.parse(params['older_than']).seconds.ago
tags.select do |tag|
tag.created_at && tag.created_at < older_than
end
end
def can_admin?
can?(current_user, :admin_container_image, project)
end
def can_use?
Feature.enabled?(:container_registry_cleanup, project, default_enabled: true)
end
end
end
end
Loading
Loading
@@ -90,13 +90,15 @@
- object_pool:object_pool_join
- object_pool:object_pool_destroy
 
- container_repository:delete_container_repository
- container_repository:cleanup_container_repository
- default
- mailers # ActionMailer::DeliveryJob.queue_name
 
- authorized_projects
- background_migration
- create_gpg_signature
- delete_container_repository
- delete_merged_branches
- delete_user
- email_receiver
Loading
Loading
# frozen_string_literal: true
class CleanupContainerRepositoryWorker
include ApplicationWorker
include ExclusiveLeaseGuard
queue_namespace :container_repository
LEASE_TIMEOUT = 1.hour
attr_reader :container_repository, :current_user
def perform(current_user_id, container_repository_id, params)
@current_user = User.find_by_id(current_user_id)
@container_repository = ContainerRepository.find_by_id(container_repository_id)
return unless valid?
try_obtain_lease do
Projects::ContainerRepository::CleanupTagsService
.new(project, current_user, params)
.execute(container_repository)
end
end
private
def valid?
current_user && container_repository && project
end
def project
container_repository&.project
end
# For ExclusiveLeaseGuard concern
def lease_key
@lease_key ||= "container_repository:cleanup_tags:#{container_repository.id}"
end
# For ExclusiveLeaseGuard concern
def lease_timeout
LEASE_TIMEOUT
end
# For ExclusiveLeaseGuard concern
def lease_release?
# we don't allow to execute this worker
# more often than LEASE_TIMEOUT
# for given container repository
false
end
end
Loading
Loading
@@ -4,6 +4,8 @@ class DeleteContainerRepositoryWorker
include ApplicationWorker
include ExclusiveLeaseGuard
 
queue_namespace :container_repository
LEASE_TIMEOUT = 1.hour
 
attr_reader :container_repository
Loading
Loading
---
title: Add Container Registry API with cleanup function
merge_request: 24303
author:
type: added
Loading
Loading
@@ -47,7 +47,6 @@
- [project_service, 1]
- [delete_user, 1]
- [todos_destroyer, 1]
- [delete_container_repository, 1]
- [delete_merged_branches, 1]
- [authorized_projects, 1]
- [expire_build_instance_artifacts, 1]
Loading
Loading
@@ -81,6 +80,7 @@
- [delete_diff_files, 1]
- [detect_repository_languages, 1]
- [auto_devops, 2]
- [container_repository, 1]
- [object_pool, 1]
- [repository_cleanup, 1]
- [delete_stored_files, 1]
Loading
Loading
# frozen_string_literal: true
class MigrateDeleteContainerRepositoryWorker < ActiveRecord::Migration[5.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def up
sidekiq_queue_migrate('delete_container_repository', to: 'container_repository:delete_container_repository')
end
def down
sidekiq_queue_migrate('container_repository:delete_container_repository', to: 'delete_container_repository')
end
end
Loading
Loading
@@ -16,6 +16,7 @@ The following API resources are available:
- [Broadcast messages](broadcast_messages.md)
- [Code snippets](snippets.md)
- [Commits](commits.md)
- [Container Registry](container_registry.md)
- [Custom attributes](custom_attributes.md)
- [Deploy keys](deploy_keys.md), and [deploy keys for multiple projects](deploy_key_multiple_projects.md)
- [Deployments](deployments.md)
Loading
Loading
# Container Registry API
## List registry repositories
Get a list of registry repositories in a project.
```
GET /projects/:id/registry/repositories
```
| Attribute | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user
```bash
curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories"
```
Example response:
```json
[
{
"id": 1,
"name": "",
"path": "group/project",
"location": "gitlab.example.com:5000/group/project",
"created_at": "2019-01-10T13:38:57.391Z"
},
{
"id": 2,
"name": "releases",
"path": "group/project/releases",
"location": "gitlab.example.com:5000/group/project/releases",
"created_at": "2019-01-10T13:39:08.229Z"
}
]
```
## Delete registry repository
Get a list of repository commits in a project.
This operation is executed asynchronously and it might take
time to get executed.
```
DELETE /projects/:id/registry/repositories/:repository_id
```
| Attribute | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user
| `repository_id` | integer | yes | The ID of registry repository
```bash
curl -X DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2"
```
## List repository tags
Get a list of tags for given registry repository.
```
GET /projects/:id/registry/repositories/:repository_id/tags
```
| Attribute | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user
| `repository_id` | integer | yes | The ID of registry repository
```bash
curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags"
```
Example response:
```json
[
{
"name": "A",
"path": "group/project:A",
"location": "gitlab.example.com:5000/group/project:A"
},
{
"name": "latest",
"path": "group/project:latest",
"location": "gitlab.example.com:5000/group/project:latest"
}
]
```
## Delete repository tags (in bulk)
Delete repository tags in bulk based on given criteria.
This API performs a following set of the operations:
1. It schedules asynchronous job executed in background,
1. It never removes tag named `latest`,
1. It removes the tags matching given `name_regex` only,
1. It orders all tags by creation date. The creation date is time of the manifest creation. It is not a time of tag push,
1. It keeps N latest matching tags (if specified),
1. It only removes tags that are older than (if specified).
These operations are executed asynchronously and it might
take time to get executed. This API can be run at most
once an hour for given container repository.
Due to [Docker Distribution deficiency](ce-21405) it does
not remove tags whose manifest is shared by multiple tags
```
DELETE /projects/:id/registry/repositories/:repository_id/tags
```
| Attribute | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user
| `repository_id` | integer | yes | The ID of registry repository
| `name_regex` | string | yes | The regex of the name to delete. To delete all tags specify `.*`
| `keep_n` | integer | no | The amount of latest tags of given name to keep
| `older_than` | string | no | Tags to delete that are older than given timespec, written in human readable form `1h`, `1d`, `1month` |
Examples:
1. Remove tag names that are matching GIT SHA, keep always at least 5, and remove ones that are older than 2 days:
```bash
curl -X DELETE -F 'name_regex=[0-9a-z]{40}' -F 'keep_n=5' -F 'older_than=2d' --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags"
```
2. Remove all tags, but keep always latest 5:
```bash
curl -X DELETE -F 'name_regex=.*' -F 'keep_n=5' --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags"
```
3. Remove all tags that are older than 1 month:
```bash
curl -X DELETE -F 'name_regex=.*' -F 'older_than=1month' --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags"
```
## Get a details repository tag
Get a details of registry repository tag
```
GET /projects/:id/registry/repositories/:repository_id/tags/:tag_name
```
| Attribute | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user
| `repository_id` | integer | yes | The ID of registry repository
| `tag_name` | string | yes | The name of tag
```bash
curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags/v10.0.0"
```
Example response:
```json
{
"name": "v10.0.0",
"path": "group/project:latest",
"location": "gitlab.example.com:5000/group/project:latest",
"revision": "e9ed9d87c881d8c2fd3a31b41904d01ba0b836e7fd15240d774d811a1c248181",
"short_revision": "e9ed9d87c",
"digest": "sha256:c3490dcf10ffb6530c1303522a1405dfaf7daecd8f38d3e6a1ba19ea1f8a1751",
"created_at": "2019-01-06T16:49:51.272+00:00",
"total_size": 350224384
}
```
## Delete a repository tag
Delete a registry repository tag
```
DELETE /projects/:id/registry/repositories/:repository_id/tags/:tag_name
```
| Attribute | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user
| `repository_id` | integer | yes | The ID of registry repository
| `tag_name` | string | yes | The name of tag
```bash
curl -X DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags/v10.0.0"
```
[ce-21405]: https://gitlab.com/gitlab-org/gitlab-ce/issues/21405
Loading
Loading
@@ -100,6 +100,7 @@ module API
mount ::API::CircuitBreakers
mount ::API::Commits
mount ::API::CommitStatuses
mount ::API::ContainerRegistry
mount ::API::DeployKeys
mount ::API::Deployments
mount ::API::Environments
Loading
Loading
# frozen_string_literal: true
module API
class ContainerRegistry < Grape::API
include PaginationParams
REGISTRY_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(
tag_name: API::NO_SLASH_URL_PART_REGEX)
before { error!('404 Not Found', 404) unless Feature.enabled?(:container_registry_api, user_project, default_enabled: true) }
before { authorize_read_container_images! }
params do
requires :id, type: String, desc: 'The ID of a project'
end
resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
desc 'Get a project container repositories' do
detail 'This feature was introduced in GitLab 11.8.'
success Entities::ContainerRegistry::Repository
end
params do
use :pagination
end
get ':id/registry/repositories' do
repositories = user_project.container_repositories.ordered
present paginate(repositories), with: Entities::ContainerRegistry::Repository
end
desc 'Delete repository' do
detail 'This feature was introduced in GitLab 11.8.'
end
params do
requires :repository_id, type: Integer, desc: 'The ID of the repository'
end
delete ':id/registry/repositories/:repository_id', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do
authorize_admin_container_image!
DeleteContainerRepositoryWorker.perform_async(current_user.id, repository.id)
status :accepted
end
desc 'Get a list of repositories tags' do
detail 'This feature was introduced in GitLab 11.8.'
success Entities::ContainerRegistry::Tag
end
params do
requires :repository_id, type: Integer, desc: 'The ID of the repository'
use :pagination
end
get ':id/registry/repositories/:repository_id/tags', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do
authorize_read_container_image!
tags = Kaminari.paginate_array(repository.tags)
present paginate(tags), with: Entities::ContainerRegistry::Tag
end
desc 'Delete repository tags (in bulk)' do
detail 'This feature was introduced in GitLab 11.8.'
end
params do
requires :repository_id, type: Integer, desc: 'The ID of the repository'
requires :name_regex, type: String, desc: 'The tag name regexp to delete, specify .* to delete all'
optional :keep_n, type: Integer, desc: 'Keep n of latest tags with matching name'
optional :older_than, type: String, desc: 'Delete older than: 1h, 1d, 1month'
end
delete ':id/registry/repositories/:repository_id/tags', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do
authorize_admin_container_image!
CleanupContainerRepositoryWorker.perform_async(current_user.id, repository.id,
declared_params.except(:repository_id)) # rubocop: disable CodeReuse/ActiveRecord
status :accepted
end
desc 'Get a details about repository tag' do
detail 'This feature was introduced in GitLab 11.8.'
success Entities::ContainerRegistry::TagDetails
end
params do
requires :repository_id, type: Integer, desc: 'The ID of the repository'
requires :tag_name, type: String, desc: 'The name of the tag'
end
get ':id/registry/repositories/:repository_id/tags/:tag_name', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do
authorize_read_container_image!
validate_tag!
present tag, with: Entities::ContainerRegistry::TagDetails
end
desc 'Delete repository tag' do
detail 'This feature was introduced in GitLab 11.8.'
end
params do
requires :repository_id, type: Integer, desc: 'The ID of the repository'
requires :tag_name, type: String, desc: 'The name of the tag'
end
delete ':id/registry/repositories/:repository_id/tags/:tag_name', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do
authorize_destroy_container_image!
validate_tag!
tag.delete
status :ok
end
end
helpers do
def authorize_read_container_images!
authorize! :read_container_image, user_project
end
def authorize_read_container_image!
authorize! :read_container_image, repository
end
def authorize_update_container_image!
authorize! :update_container_image, repository
end
def authorize_destroy_container_image!
authorize! :admin_container_image, repository
end
def authorize_admin_container_image!
authorize! :admin_container_image, repository
end
def repository
@repository ||= user_project.container_repositories.find(params[:repository_id])
end
def tag
@tag ||= repository.tag(params[:tag_name])
end
def validate_tag!
not_found!('Tag') unless tag.valid?
end
end
end
end
# frozen_string_literal: true
module API
module Entities
module ContainerRegistry
class Repository < Grape::Entity
expose :id
expose :name
expose :path
expose :location
expose :created_at
end
class Tag < Grape::Entity
expose :name
expose :path
expose :location
end
class TagDetails < Tag
expose :revision
expose :short_revision
expose :digest
expose :created_at
expose :total_size
end
end
end
end
Loading
Loading
@@ -2,6 +2,8 @@
 
module ContainerRegistry
class Tag
include Gitlab::Utils::StrongMemoize
attr_reader :repository, :name
 
delegate :registry, :client, to: :repository
Loading
Loading
@@ -15,6 +17,10 @@ module ContainerRegistry
manifest.present?
end
 
def latest?
name == "latest"
end
def v1?
manifest && manifest['schemaVersion'] == 1
end
Loading
Loading
@@ -24,7 +30,9 @@ module ContainerRegistry
end
 
def manifest
@manifest ||= client.repository_manifest(repository.path, name)
strong_memoize(:manifest) do
client.repository_manifest(repository.path, name)
end
end
 
def path
Loading
Loading
@@ -42,36 +50,44 @@ module ContainerRegistry
end
 
def digest
@digest ||= client.repository_tag_digest(repository.path, name)
strong_memoize(:digest) do
client.repository_tag_digest(repository.path, name)
end
end
 
def config_blob
return @config_blob if defined?(@config_blob)
return unless manifest && manifest['config']
 
@config_blob = repository.blob(manifest['config'])
strong_memoize(:config_blob) do
repository.blob(manifest['config'])
end
end
 
def config
return unless config_blob
return unless config_blob&.data
 
@config ||= ContainerRegistry::Config.new(self, config_blob) if config_blob.data
strong_memoize(:config) do
ContainerRegistry::Config.new(self, config_blob)
end
end
 
def created_at
return unless config
 
@created_at ||= DateTime.rfc3339(config['created'])
strong_memoize(:created_at) do
DateTime.rfc3339(config['created'])
end
end
 
def layers
return @layers if defined?(@layers)
return unless manifest
 
layers = manifest['layers'] || manifest['fsLayers']
strong_memoize(:layers) do
layers = manifest['layers'] || manifest['fsLayers']
 
@layers = layers.map do |layer|
repository.blob(layer)
layers.map do |layer|
repository.blob(layer)
end
end
end
 
Loading
Loading
Loading
Loading
@@ -19,7 +19,7 @@ describe Projects::Registry::TagsController do
end
 
before do
stub_container_registry_tags(repository: /image/, tags: tags)
stub_container_registry_tags(repository: /image/, tags: tags, with_manifest: true)
end
 
context 'when user can control the registry' do
Loading
Loading
FactoryBot.define do
factory :container_repository do
name 'test_container_image'
name 'test_image'
project
 
transient do
Loading
Loading
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment