Merge branch 'internal-release-support' into 'master'

Support internal releases

See merge request https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/8045



Merged-by: Andrew Patterson <apatterson@gitlab.com>
Approved-by: Andrew Patterson <apatterson@gitlab.com>
Reviewed-by: Balasankar 'Balu' C <balasankar@gitlab.com>
Reviewed-by: Clemens Beck <cbeck@gitlab.com>
Co-authored-by: Balasankar 'Balu' C <balasankar@gitlab.com>

gitlab-ci-config/check-packages.yml

@@ -30,6 +30,10 @@ before_script:
     - bash scripts/ci/deploy_package.sh
   tags: !reference [.distribution-amd64-tags]
   retry: 2
+  rules:
+    - if: '$PIPELINE_TYPE == "INTERNAL_RELEASE_CHECK_PACKAGE_PIPELINE"'
+      when: never
+    - when: always
 .apt-install:
   extends: .job_base
@@ -65,18 +69,24 @@ before_script:
 Ubuntu-20.04-check:
   image: "${BUILDER_IMAGE_REGISTRY}/ubuntu_20.04:latest"
   extends: .apt-install
+  rules:
+    - when: always
 Ubuntu-20.04-arm64-check:
   image: "${BUILDER_IMAGE_REGISTRY}/ubuntu_20.04_arm64:latest"
   extends: .apt-arm-install
 Ubuntu-22.04-check:
   image: "${BUILDER_IMAGE_REGISTRY}/ubuntu_22.04:latest"
   extends: .apt-install
+  rules:
+    - when: always
 Ubuntu-22.04-arm64-check:
   image: "${BUILDER_IMAGE_REGISTRY}/ubuntu_22.04_arm64:latest"
   extends: .apt-arm-install
 Ubuntu-24.04-check:
   image: "${BUILDER_IMAGE_REGISTRY}/ubuntu_24.04:latest"
   extends: .apt-install
+  rules:
+    - when: always
 Ubuntu-24.04-arm64-check:
   image: "${BUILDER_IMAGE_REGISTRY}/ubuntu_24.04_arm64:latest"
   extends: .apt-arm-install
@@ -121,6 +131,7 @@ Ubuntu-20.04-fips-check:
   extends: .apt-install
   rules:
     - if: '$PIPELINE_TYPE == "EE_CHECK_PACKAGE_AVAILABILITY_PIPELINE"'
+    - if: '$PIPELINE_TYPE == "INTERNAL_RELEASE_CHECK_PACKAGE_PIPELINE"'
   variables:
     USE_SYSTEM_SSL: "true"
     USE_SYSTEM_LIBGCRYPT: "true"
@@ -216,3 +227,7 @@ Docker:
     - bash scripts/ci/deploy_docker.sh
   tags: !reference [.distribution-amd64-tags]
   retry: 2
+  rules:
+    - if: '$PIPELINE_TYPE == "INTERNAL_RELEASE_CHECK_PACKAGE_PIPELINE"'
+      when: never
+    - when: always

gitlab-ci-config/dev-gitlab-org.yml

@@ -286,19 +286,27 @@
 Ubuntu-20.04-branch:
   image: "${BUILDER_IMAGE_REGISTRY}/ubuntu_20.04:${BUILDER_IMAGE_REVISION}"
   extends: .branch_template
+  rules:
+    - !reference [.default_rules, rules]
+    - if: '$PIPELINE_TYPE =~ /(NIGHTLY|BRANCH|INTERNAL_RELEASE)_BUILD_PIPELINE$/'
+    - if: '$PIPELINE_TYPE == "TRIGGER_CACHE_UPDATE_PIPELINE"'
 Ubuntu-22.04-branch:
   image: "${BUILDER_IMAGE_REGISTRY}/ubuntu_22.04:${BUILDER_IMAGE_REVISION}"
   extends: .branch_template
   rules:
     - !reference [.default_rules, rules]
-    - if: '$PIPELINE_TYPE =~ /_(NIGHTLY|BRANCH)_BUILD_PIPELINE$/'
+    - if: '$PIPELINE_TYPE =~ /(NIGHTLY|BRANCH|INTERNAL_RELEASE)_BUILD_PIPELINE$/'
     - if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/'
     - if: '$PIPELINE_TYPE == "TRIGGER_CACHE_UPDATE_PIPELINE"'
 Ubuntu-24.04-branch:
   image: "${BUILDER_IMAGE_REGISTRY}/ubuntu_24.04:${BUILDER_IMAGE_REVISION}"
   extends: .branch_template
+  rules:
+    - !reference [.default_rules, rules]
+    - if: '$PIPELINE_TYPE =~ /(NIGHTLY|BRANCH|INTERNAL_RELEASE)_BUILD_PIPELINE$/'
+    - if: '$PIPELINE_TYPE == "TRIGGER_CACHE_UPDATE_PIPELINE"'
 Debian-11-branch:
   image: "${BUILDER_IMAGE_REGISTRY}/debian_11:${BUILDER_IMAGE_REVISION}"
@@ -401,6 +409,7 @@ Ubuntu-20.04-fips-branch:
   rules:
     - !reference [.default_rules, rules]
     - if: '$PIPELINE_TYPE == "EE_NIGHTLY_BUILD_PIPELINE"'
+    - if: '$PIPELINE_TYPE == "INTERNAL_RELEASE_BUILD_PIPELINE"'
     - if: '$PIPELINE_TYPE == "EE_BRANCH_BUILD_PIPELINE"'
       when: manual
       allow_failure: true
@@ -540,7 +549,7 @@ Ubuntu-20.04-staging:
       optional: true
   rules:
     - !reference [.default_rules, rules]
-    - if: '$PIPELINE_TYPE =~ /(_RC|_TAG|AUTO_DEPLOY|NIGHTLY)_BUILD_PIPELINE$/'
+    - if: '$PIPELINE_TYPE =~ /(_RC|_TAG|AUTO_DEPLOY|NIGHTLY|INTERNAL_RELEASE)_BUILD_PIPELINE$/'
 Ubuntu-22.04-staging:
   extends: .staging_upload_template
@@ -551,7 +560,7 @@ Ubuntu-22.04-staging:
       optional: true
   rules:
     - !reference [.default_rules, rules]
-    - if: '$PIPELINE_TYPE =~ /(_RC|_TAG|AUTO_DEPLOY|NIGHTLY)_BUILD_PIPELINE$/'
+    - if: '$PIPELINE_TYPE =~ /(_RC|_TAG|AUTO_DEPLOY|NIGHTLY|INTERNAL_RELEASE)_BUILD_PIPELINE$/'
 Ubuntu-22.04-arm64-staging:
   extends: .staging_upload_template
@@ -710,6 +719,9 @@ Ubuntu-24.04-staging:
       optional: true
     - job: Ubuntu-24.04-branch
       optional: true
+  rules:
+    - !reference [.default_rules, rules]
+    - if: '$PIPELINE_TYPE =~ /(_RC|_TAG|AUTO_DEPLOY|NIGHTLY|INTERNAL_RELEASE)_BUILD_PIPELINE$/'
 Ubuntu-20.04-arm64-staging:
   extends: .staging_upload_template
   needs:
@@ -865,6 +877,11 @@ Ubuntu-20.04-fips-staging:
       optional: true
     - job: Ubuntu-20.04-fips-branch
       optional: true
+  rules:
+    - !reference [.default_rules, rules]
+    - if: '$PIPELINE_TYPE =~ /^EE_(RC|TAG)_BUILD_PIPELINE$/'
+    - if: '$PIPELINE_TYPE == "INTERNAL_RELEASE_BUILD_PIPELINE"'
+    - if: '$PIPELINE_TYPE == "EE_NIGHTLY_BUILD_PIPELINE"'
 ### Package releases
 Ubuntu-20.04-release:
@@ -1180,6 +1197,12 @@ check-packages-functionality:
     - if: '$PIPELINE_TYPE =~ /EE_(RC|TAG)_BUILD_PIPELINE$/'
       variables:
         PIPELINE_TYPE: 'EE_CHECK_PACKAGE_FUNCTIONALITY_PIPELINE'
+    - if: '$PIPELINE_TYPE == "INTERNAL_RELEASE_BUILD_PIPELINE"'
+      variables:
+        ee: 'true'
+        PIPELINE_TYPE: 'INTERNAL_RELEASE_CHECK_PACKAGE_PIPELINE'
+        INTERNAL_RELEASE: 'true'
+        INTERNAL_RELEASE_ITERATION: "${INTERNAL_RELEASE_ITERATION}"
 # Check if the packages are available from public repositories
 check-packages-availability:

gitlab-ci-config/workflow-rules.yml

@@ -41,6 +41,10 @@ workflow:
     - if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == $CANONICAL_PROJECT_PATH && $CI_PIPELINE_SOURCE == "merge_request_event"'
       variables:
         PIPELINE_TYPE: GITLAB_MR_PIPELINE
+    - if: '$PIPELINE_TYPE == null && $INTERNAL_RELEASE == "true" && $INTERNAL_RELEASE_ITERATION && $CI_PROJECT_PATH == $DEV_PROJECT_PATH && $CI_COMMIT_BRANCH =~ /^.*-stable$/'
+      variables:
+        PIPELINE_TYPE: INTERNAL_RELEASE_BUILD_PIPELINE
+        CACHE_EDITION: "STABLE"
     - if: '$PIPELINE_TYPE == null && $NIGHTLY && $CI_COMMIT_BRANCH && ($ee == "true" || $GITLAB_VERSION =~ /-ee$/)'
       variables:
         PIPELINE_TYPE: EE_NIGHTLY_BUILD_PIPELINE

lib/gitlab/build/check.rb

@@ -85,6 +85,10 @@ module Build
         Gitlab::Util.get_env('NIGHTLY') == 'true'
       end
+      def is_internal_release?
+        Gitlab::Util.get_env('INTERNAL_RELEASE') == 'true'
+      end
+
       def no_changes?
         system(*%w[git diff --quiet])
       end

lib/gitlab/build/info/package.rb

@@ -38,12 +38,27 @@ module Build
             Omnibus.load_configuration('omnibus.rb')
             Omnibus::BuildVersion.semver
           else
+            # Non-tag builds have their versions relative to the latest git tag
+            # in the repo
             latest_git_tag = Info::Git.latest_tag.strip
             latest_version = latest_git_tag && !latest_git_tag.empty? ? latest_git_tag[0, latest_git_tag.match("[+]").begin(0)] : '0.0.1'
-            commit_sha = Build::Info::Git.commit_sha
-            ver_tag = "#{latest_version}+" + (Build::Check.is_nightly? ? "rnightly" : "rfbranch")
-            ver_tag += ".fips" if fips
-            [ver_tag, Gitlab::Util.get_env('CI_PIPELINE_ID'), commit_sha].compact.join('.')
+
+            # For internal release builds, we append `internal` suffix.
+            if Build::Check.is_internal_release?
+              "#{latest_version}+internal#{internal_release_iteration}"
+            else
+              # For nightly builds, we append `rnightly`
+              # For other regular feature branch builds, we append `rfbranch`
+              ver_tag = "#{latest_version}+" + (Build::Check.is_nightly? ? "rnightly" : "rfbranch")
+
+              # Differentiate between FIPS and regular builds
+              ver_tag += ".fips" if fips
+
+              # `CI_PIPELINE_ID` and commit SHA are appended to differentiate
+              # between two pipelines against same branch
+              commit_sha = Build::Info::Git.commit_sha
+              [ver_tag, Gitlab::Util.get_env('CI_PIPELINE_ID'), commit_sha].compact.join('.')
+            end
           end
         end
@@ -52,6 +67,10 @@ module Build
           "#{semver}-#{Gitlab::BuildIteration.new.build_iteration}"
         end
+        def internal_release_iteration
+          Gitlab::Util.get_env('INTERNAL_RELEASE_ITERATION')
+        end
+
         def file_list
           Dir.glob(PACKAGE_GLOB)
         end

scripts/ci/deploy_package.sh

@@ -2,7 +2,7 @@
 set -eo pipefail
-if [ "${deploy_instance}" != "true" ]; then
+if [ "${deploy_instance}" != "true" ] || [ "${USE_SYSTEM_SSL}" == "true" ]; then
   exit 0;
 fi

spec/lib/gitlab/build/info/package_spec.rb

@@ -49,6 +49,17 @@ RSpec.describe Build::Info::Package do
         stub_env_var('CI_PIPELINE_ID', '12345')
       end
+      context 'for internal releases' do
+        before do
+          allow(Build::Info::Package).to receive(:internal_release_iteration).and_return(1)
+          allow(Build::Check).to receive(:is_internal_release?).and_return(true)
+        end
+
+        it 'returns computed version as expected' do
+          expect(described_class.semver_version).to eq('16.2.0+internal1')
+        end
+      end
+
       context 'on nightlies' do
         before do
           allow(Build::Check).to receive(:is_nightly?).and_return(true)