Merge branch 'mattkasa/terraform-module-packages' into 'master'

Add Terraform Module Registry [RUN ALL RSPEC] [RUN AS-IF-FOSS]

See merge request gitlab-org/gitlab!55018

app/controllers/admin/plan_limits_controller.rb

@@ -35,6 +35,7 @@ def plan_limits_params
       npm_max_file_size
       nuget_max_file_size
       pypi_max_file_size
+      terraform_module_max_file_size
       generic_packages_max_file_size
     ])
   end

app/controllers/terraform/services_controller.rb

+# frozen_string_literal: true
+
+class Terraform::ServicesController < ApplicationController
+  skip_before_action :authenticate_user!
+
+  feature_category :infrastructure_as_code
+
+  def index
+    render json: { 'modules.v1' => "/api/#{::API::API.version}/packages/terraform/modules/v1/" }
+  end
+end

app/finders/concerns/packages/finder_helper.rb

@@ -42,7 +42,7 @@ def package_type
     end
     def filter_by_package_type(packages)
-      return packages unless package_type
+      return packages.without_package_type(:terraform_module) unless package_type
       raise InvalidPackageTypeError unless ::Packages::Package.package_types.key?(package_type)
       packages.with_package_type(package_type)
@@ -54,6 +54,12 @@ def filter_by_package_name(packages)
       packages.search_by_name(params[:package_name])
     end
+    def filter_by_package_version(packages)
+      return packages unless params[:package_version].present?
+
+      packages.with_version(params[:package_version])
+    end
+
     def filter_with_version(packages)
       return packages if params[:include_versionless].present?

app/finders/packages/group_packages_finder.rb

@@ -32,6 +32,7 @@ def packages_for_group_projects(installable_only: false)
       packages = filter_with_version(packages)
       packages = filter_by_package_type(packages)
       packages = filter_by_package_name(packages)
+      packages = filter_by_package_version(packages)
       installable_only ? packages.installable : filter_by_status(packages)
     end

app/graphql/types/packages/package_type_enum.rb

@@ -5,7 +5,8 @@ module Packages
     class PackageTypeEnum < BaseEnum
       PACKAGE_TYPE_NAMES = {
         pypi: 'PyPI',
-        npm: 'npm'
+        npm: 'npm',
+        terraform_module: 'Terraform Module'
       }.freeze
       ::Packages::Package.package_types.keys.each do |package_type|

app/models/packages/package.rb

@@ -51,6 +51,7 @@ class Packages::Package < ApplicationRecord
   validates :name, format: { with: Gitlab::Regex.helm_package_regex }, if: :helm?
   validates :name, format: { with: Gitlab::Regex.npm_package_name_regex }, if: :npm?
   validates :name, format: { with: Gitlab::Regex.nuget_package_name_regex }, if: :nuget?
+  validates :name, format: { with: Gitlab::Regex.terraform_module_package_name_regex }, if: :terraform_module?
   validates :name, format: { with: Gitlab::Regex.debian_package_name_regex }, if: :debian_package?
   validates :name, inclusion: { in: %w[incoming] }, if: :debian_incoming?
   validates :version, format: { with: Gitlab::Regex.nuget_version_regex }, if: :nuget?
@@ -59,7 +60,7 @@ class Packages::Package < ApplicationRecord
   validates :version, format: { with: Gitlab::Regex.pypi_version_regex }, if: :pypi?
   validates :version, format: { with: Gitlab::Regex.prefixed_semver_regex }, if: :golang?
   validates :version, format: { with: Gitlab::Regex.prefixed_semver_regex }, if: :helm?
-  validates :version, format: { with: Gitlab::Regex.semver_regex }, if: -> { composer_tag_version? || npm? }
+  validates :version, format: { with: Gitlab::Regex.semver_regex }, if: -> { composer_tag_version? || npm? || terraform_module? }
   validates :version,
     presence: true,
@@ -73,7 +74,7 @@ class Packages::Package < ApplicationRecord
   enum package_type: { maven: 1, npm: 2, conan: 3, nuget: 4, pypi: 5,
                        composer: 6, generic: 7, golang: 8, debian: 9,
-                       rubygems: 10, helm: 11 }
+                       rubygems: 10, helm: 11, terraform_module: 12 }
   enum status: { default: 0, hidden: 1, processing: 2, error: 3 }
@@ -85,6 +86,7 @@ class Packages::Package < ApplicationRecord
   scope :with_version, ->(version) { where(version: version) }
   scope :without_version_like, -> (version) { where.not(arel_table[:version].matches(version)) }
   scope :with_package_type, ->(package_type) { where(package_type: package_type) }
+  scope :without_package_type, ->(package_type) { where.not(package_type: package_type) }
   scope :with_status, ->(status) { where(status: status) }
   scope :displayable, -> { with_status(DISPLAYABLE_STATUSES) }
   scope :installable, -> { with_status(INSTALLABLE_STATUSES) }

app/presenters/terraform/modules_presenter.rb

+# frozen_string_literal: true
+
+module Terraform
+  class ModulesPresenter < Gitlab::View::Presenter::Simple
+    attr_accessor :packages, :system
+
+    presents :modules
+
+    def initialize(packages, system)
+      @packages = packages
+      @system = system
+    end
+
+    def modules
+      project_url = @packages.first&.project&.web_url
+      versions = @packages.map do |package|
+        {
+          'version' => package.version,
+          'submodules' => [],
+          'root' => {
+            'dependencies' => [],
+            'providers' => [
+              {
+                'name' => @system,
+                'version' => ''
+              }
+            ]
+          }
+        }
+      end
+
+      [
+        {
+          'versions' => versions,
+          'source' => project_url
+        }.compact
+      ]
+    end
+  end
+end

app/services/packages/terraform_module/create_package_service.rb

+# frozen_string_literal: true
+
+module Packages
+  module TerraformModule
+    class CreatePackageService < ::Packages::CreatePackageService
+      include Gitlab::Utils::StrongMemoize
+
+      def execute
+        return error('Version is empty.', 400) if params[:module_version].blank?
+        return error('Package already exists.', 403) if current_package_exists_elsewhere?
+        return error('Package version already exists.', 403) if current_package_version_exists?
+        return error('File is too large.', 400) if file_size_exceeded?
+
+        ActiveRecord::Base.transaction { create_terraform_module_package! }
+      end
+
+      private
+
+      def create_terraform_module_package!
+        package = create_package!(:terraform_module, name: name, version: params[:module_version])
+
+        ::Packages::CreatePackageFileService.new(package, file_params).execute
+
+        package
+      end
+
+      def current_package_exists_elsewhere?
+        ::Packages::Package
+          .for_projects(project.root_namespace.all_projects.id_not_in(project.id))
+          .with_package_type(:terraform_module)
+          .with_name(name)
+          .exists?
+      end
+
+      def current_package_version_exists?
+        project.packages
+          .with_package_type(:terraform_module)
+          .with_name(name)
+          .with_version(params[:module_version])
+          .exists?
+      end
+
+      def name
+        strong_memoize(:name) do
+          "#{params[:module_name]}/#{params[:module_system]}"
+        end
+      end
+
+      def file_name
+        strong_memoize(:file_name) do
+          "#{params[:module_name]}-#{params[:module_system]}-#{params[:module_version]}.tgz"
+        end
+      end
+
+      def file_params
+        {
+          file: params[:file],
+          size: params[:file].size,
+          file_sha256: params[:file].sha256,
+          file_name: file_name,
+          build: params[:build]
+        }
+      end
+
+      def file_size_exceeded?
+        project.actual_limits.exceeded?(:generic_packages_max_file_size, params[:file].size)
+      end
+    end
+  end
+end

app/views/admin/application_settings/_package_registry.html.haml

@@ -44,6 +44,9 @@
                   .form-group
                     = f.label :pypi_max_file_size, _('Maximum PyPI package file size in bytes'), class: 'label-bold'
                     = f.number_field :pypi_max_file_size, class: 'form-control gl-form-input'
+                  .form-group
+                    = f.label :terraform_module_max_file_size, _('Maximum Terraform Module package file size in bytes'), class: 'label-bold'
+                    = f.number_field :terraform_module_max_file_size, class: 'form-control gl-form-input'
                   .form-group
                     = f.label :generic_packages_max_file_size, _('Generic package file size in bytes'), class: 'label-bold'
                     = f.number_field :generic_packages_max_file_size, class: 'form-control gl-form-input'

changelogs/unreleased/mattkasa-terraform-module-packages.yml

+---
+title: Add Terraform Module Registry
+merge_request: 55018
+author:
+type: added

config/metrics/counts_28d/20210410012206_i_package_terraform_module_deploy_token_monthly.yml

+---
+key_path: redis_hll_counters.deploy_token_packages.i_package_terraform_module_deploy_token_monthly
+description: Number of distinct users authorized via deploy token creating Terraform Module packages in recent 28 days
+product_section: ops
+product_stage: configure
+product_group: group::configure
+product_category: infrastructure_as_code
+value_type: number
+status: implemented
+milestone: '13.11'
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55018
+time_frame: 28d
+data_source: redis_hll
+distribution:
+- ce
+- ee
+tier:
+- free
+- premium
+- ultimate

config/metrics/counts_28d/20210410012208_i_package_terraform_module_user_monthly.yml

+---
+key_path: redis_hll_counters.user_packages.i_package_terraform_module_user_monthly
+description: Number of distinct users creating Terraform Module packages in recent 28 days
+product_section: ops
+product_stage: configure
+product_group: group::configure
+product_category: infrastructure_as_code
+value_type: number
+status: implemented
+milestone: '13.11'
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55018
+time_frame: 28d
+data_source: redis_hll
+distribution:
+- ce
+- ee
+tier:
+- free
+- premium
+- ultimate

config/metrics/counts_7d/20210410012207_i_package_terraform_module_deploy_token_weekly.yml

+---
+key_path: redis_hll_counters.deploy_token_packages.i_package_terraform_module_deploy_token_weekly
+description: Number of distinct users authorized via deploy token creating Terraform Module packages in recent 7 days
+product_section: ops
+product_stage: configure
+product_group: group::configure
+product_category: infrastructure_as_code
+value_type: number
+status: implemented
+milestone: '13.11'
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55018
+time_frame: 7d
+data_source: redis_hll
+distribution:
+- ce
+- ee
+tier:
+- free
+- premium
+- ultimate

config/metrics/counts_7d/20210410012209_i_package_terraform_module_user_weekly.yml

+---
+key_path: redis_hll_counters.user_packages.i_package_terraform_module_user_weekly
+description: Number of distinct users creating Terraform Module packages in recent 7 days
+product_section: ops
+product_stage: configure
+product_group: group::configure
+product_category: infrastructure_as_code
+value_type: number
+status: implemented
+milestone: '13.11'
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55018
+time_frame: 7d
+data_source: redis_hll
+distribution:
+- ce
+- ee
+tier:
+- free
+- premium
+- ultimate

config/metrics/counts_all/20210410012200_package_events_i_package_terraform_module_delete_package.yml

+---
+key_path: counts.package_events_i_package_terraform_module_delete_package
+description: Total count of Terraform Module packages delete events
+product_section: ops
+product_stage: configure
+product_group: group::configure
+product_category: infrastructure_as_code
+value_type: number
+status: implemented
+milestone: '13.11'
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55018
+time_frame: all
+data_source: redis
+distribution:
+- ce
+- ee
+tier:
+- free
+- premium
+- ultimate

config/metrics/counts_all/20210410012202_package_events_i_package_terraform_module_pull_package.yml

+---
+key_path: counts.package_events_i_package_terraform_module_pull_package
+description: Total count of pull Terraform Module packages events
+product_section: ops
+product_stage: configure
+product_group: group::configure
+product_category: infrastructure_as_code
+value_type: number
+status: implemented
+milestone: '13.11'
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55018
+time_frame: all
+data_source: redis
+distribution:
+- ce
+- ee
+tier:
+- free
+- premium
+- ultimate

config/metrics/counts_all/20210410012204_package_events_i_package_terraform_module_push_package.yml

+---
+key_path: counts.package_events_i_package_terraform_module_push_package
+description: Total count of push Terraform Module packages events
+product_section: ops
+product_stage: configure
+product_group: group::configure
+product_category: infrastructure_as_code
+value_type: number
+status: implemented
+milestone: '13.11'
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55018
+time_frame: all
+data_source: redis
+distribution:
+- ce
+- ee
+tier:
+- free
+- premium
+- ultimate

config/routes.rb

@@ -77,6 +77,9 @@
   # Health check
   get 'health_check(/:checks)' => 'health_check#index', as: :health_check
+  # Terraform service discovery
+  get '.well-known/terraform.json' => 'terraform/services#index', as: :terraform_services
+
   # Begin of the /-/ scope.
   # Use this scope for all new global routes.
   scope path: '-' do

doc/api/graphql/reference/index.md

@@ -14232,6 +14232,7 @@ Values for sorting package.
 | <a id="packagetypeenumnuget"></a>`NUGET` | Packages from the Nuget package manager. |
 | <a id="packagetypeenumpypi"></a>`PYPI` | Packages from the PyPI package manager. |
 | <a id="packagetypeenumrubygems"></a>`RUBYGEMS` | Packages from the Rubygems package manager. |
+| <a id="packagetypeenumterraform_module"></a>`TERRAFORM_MODULE` | Packages from the Terraform Module package manager. |
 ### `PipelineConfigSourceEnum`

doc/development/usage_ping/dictionary.md

@@ -3754,6 +3754,42 @@ Status: `data_available`
 Tiers: `free`
+### `counts.package_events_i_package_terraform_module_delete_package`
+
+Total count of Terraform Module packages delete events
+
+[YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/counts_all/20210410012200_package_events_i_package_terraform_module_delete_package.yml)
+
+Group: `group::configure`
+
+Status: `implemented`
+
+Tiers: `free`, `premium`, `ultimate`
+
+### `counts.package_events_i_package_terraform_module_pull_package`
+
+Total count of pull Terraform Module packages events
+
+[YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/counts_all/20210410012202_package_events_i_package_terraform_module_pull_package.yml)
+
+Group: `group::configure`
+
+Status: `implemented`
+
+Tiers: `free`, `premium`, `ultimate`
+
+### `counts.package_events_i_package_terraform_module_push_package`
+
+Total count of push Terraform Module packages events
+
+[YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/counts_all/20210410012204_package_events_i_package_terraform_module_push_package.yml)
+
+Group: `group::configure`
+
+Status: `implemented`
+
+Tiers: `free`, `premium`, `ultimate`
+
 ### `counts.packages`
 Number of packages
@@ -9898,6 +9934,30 @@ Status: `data_available`
 Tiers:
+### `redis_hll_counters.deploy_token_packages.i_package_terraform_module_deploy_token_monthly`
+
+Number of distinct users authorized via deploy token creating Terraform Module packages in recent 28 days
+
+[YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/counts_28d/20210410012206_i_package_terraform_module_deploy_token_monthly.yml)
+
+Group: `group::configure`
+
+Status: `implemented`
+
+Tiers: `free`, `premium`, `ultimate`
+
+### `redis_hll_counters.deploy_token_packages.i_package_terraform_module_deploy_token_weekly`
+
+Number of distinct users authorized via deploy token creating Terraform Module packages in recent 7 days
+
+[YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/counts_7d/20210410012207_i_package_terraform_module_deploy_token_weekly.yml)
+
+Group: `group::configure`
+
+Status: `implemented`
+
+Tiers: `free`, `premium`, `ultimate`
+
 ### `redis_hll_counters.ecosystem.ecosystem_total_unique_counts_monthly`
 Missing description
@@ -14866,6 +14926,30 @@ Status: `data_available`
 Tiers:
+### `redis_hll_counters.user_packages.i_package_terraform_module_user_monthly`
+
+Number of distinct users creating Terraform Module packages in recent 28 days
+
+[YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/counts_28d/20210410012208_i_package_terraform_module_user_monthly.yml)
+
+Group: `group::configure`
+
+Status: `implemented`
+
+Tiers: `free`, `premium`, `ultimate`
+
+### `redis_hll_counters.user_packages.i_package_terraform_module_user_weekly`
+
+Number of distinct users creating Terraform Module packages in recent 7 days
+
+[YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/counts_7d/20210410012209_i_package_terraform_module_user_weekly.yml)
+
+Group: `group::configure`
+
+Status: `implemented`
+
+Tiers: `free`, `premium`, `ultimate`
+
 ### `redis_hll_counters.user_packages.user_packages_total_unique_counts_monthly`
 Missing description