Merge branch 'rearchitect-ci-config' into 'master'

Rearchitect CI config with rules

See merge request gitlab-org/omnibus-gitlab!4188

.gitlab-ci.yml

@@ -23,12 +23,20 @@ stages:
 workflow:
   rules:
-    # For branches, create a pipeline.
-    # FIXME: We should change the following to `- if: 'CI_MERGE_REQUEST_IID'`
+    # Do not create a pipeline on branch push to QA mirror
+    - if: '$CI_PROJECT_NAME == "omnibus-gitlab-mirror" && $CI_PIPELINE_SOURCE == "push"'
+      when: never
+    # No pipeline on auto-deploy branches as a tag will definitely follow
+    - if: '$CI_COMMIT_BRANCH =~ /^[0-9]+-[0-9]+-auto-deploy-[0-9]+$/'
+      when: never
+    # For all other branches, create a pipeline. We are explicitly specifying
+    # this so that this rule gets matched earlier before MR pipelines gets
+    # triggered, thus causing two pipelines for a branch push - a regular one
+    # and a detached one. If we ever decide not to run pipelines on branch
+    # pushes that doesn't cause an MR, we can change the following to
+    # $CI_MERGE_REQUEST_IID
     - if: '$CI_COMMIT_BRANCH'
-    # For `master` branch, create a pipeline (this includes on schedules, pushes, merges, etc.).
-    - if: '$CI_COMMIT_BRANCH == "master"'
-    # For tags, create a pipeline.
+    # For tags, always create a pipeline.
     - if: '$CI_COMMIT_TAG'
 default:
@@ -136,21 +144,17 @@ fetch-assets:
   artifacts:
     paths:
       - ${ASSET_PATH}
-  only:
-    refs:
-      - schedules@gitlab-org/omnibus-gitlab
-      - branches@gitlab/omnibus-gitlab
-      - tags@gitlab/omnibus-gitlab
-      - triggers
-      - pipelines
-  except:
-    variables:
-      - $COMPILE_ASSETS == "true"
-      - $DEPS_PIPELINE
-      # Format of the auto-deploy tag
-      - $CI_COMMIT_TAG =~ /^\d+\.\d+\.\d+\+[^ ]{7,}\.[^ ]{7,}$/
-      - $CI_COMMIT_REF_NAME =~ /^[0-9]+-[0-9]+-auto-deploy-[0-9]+$/
-
+  rules:
+    - if: '$COMPILE_ASSETS == "true"'
+      when: never
+    - if: '$DEPS_PIPELINE'
+      when: never
+    - if: '$CI_COMMIT_TAG =~ /^\d+\.\d+\.\d+\+[^ ]{7,}\.[^ ]{7,}$/'
+      when: never
+    # Run on dev.gitlab.org (except auto-deploy tag covered above) and
+    # multi-project pipelines on omnibus-gitlab-mirror
+    - if: '$CI_PROJECT_PATH == "gitlab/omnibus-gitlab"'
+    - if: '$CI_PIPELINE_SOURCE == "pipeline" && $CI_PROJECT_PATH == "gitlab-org/build/omnibus-gitlab-mirror"'
 include:
   - '/gitlab-ci-config/gitlab-com.yml'
   - '/gitlab-ci-config/dev-gitlab-org.yml'

gitlab-ci-config/dev-gitlab-org.yml

@@ -5,39 +5,195 @@
 # Templates #
 #############
-.regular-dev-branches-only: &regular-dev-branches-only
-  only:
-    - branches@gitlab/omnibus-gitlab
-  except:
-    variables:
-      - $CI_COMMIT_REF_NAME =~ /^[0-9]+-[0-9]+-auto-deploy-[0-9]+$/
-
-.except-auto-deploy-tag: &except-auto-deploy-tag
-  except:
-    variables:
-      # Format of the auto-deploy tag for auto-deploy builds.
-      # https://gitlab.com/gitlab-org/release/docs/blob/master/general/deploy/auto-deploy.md#auto-deploy-tagging
-      - $CI_COMMIT_TAG =~ /^\d+\.\d+\.\d+\+[^ ]{7,}\.[^ ]{7,}$/
+.unless-dev: &unless-dev
+  if: '$CI_PROJECT_PATH != "gitlab/omnibus-gitlab"'
+
+.if-branch: &if-branch
+  if: '$CI_COMMIT_BRANCH'
+
+.if-tag: &if-tag
+  if: '$CI_COMMIT_TAG'
+
+.if-auto-deploy-tag: &if-auto-deploy-tag
+  if: '$CI_COMMIT_TAG =~ /^\d+\.\d+\.\d+\+[^ ]{7,}\.[^ ]{7,}$/'
+
+.if-rc-tag: &if-rc-tag
+  if: '$CI_COMMIT_TAG =~ /^.*\+rc.*/'
+
+.if-ee-tag: &if-ee-tag
+  if: '$CI_COMMIT_TAG =~ /^.*ee\.\d.*$/'
+
+.if-ee-branch: &if-ee-branch
+  if: '$CI_COMMIT_BRANCH =~ /^.*-ee$/'
+
+.if-ce-tag: &if-ce-tag
+  if: '$CI_COMMIT_TAG =~ /^.*ce\.\d.*$/'
+
+.if-ee-var: &if-ee-var
+  if: '$ee == "true"'
+
+.if-nightly: &if-nightly
+  if: '$CI_PIPELINE_SOURCE == "schedule" && $NIGHTLY == "true"'
+
+.unless-nightly: &unless-nightly
+  if: '$CI_PIPELINE_SOURCE != "schedule" || $NIGHTLY != "true"'
+
+.dev-branches-only: &dev-branches-only
+  rules:
+    - <<: *unless-dev
+      when: never
+    - <<: *if-branch
 .dev-tag-only: &dev-tag-only
-  only:
-    - tags@gitlab/omnibus-gitlab
+  rules:
+    - <<: *unless-dev
+      when: never
+    - <<: *if-tag
+
+.dev-tag-only-manual: &dev-tag-only-manual
+  rules:
+    - <<: *unless-dev
+      when: never
+    - <<: *if-tag
+      when: manual
 .dev-tag-only-except-auto-deploy: &dev-tag-only-except-auto-deploy
-  <<: *dev-tag-only
-  <<: *except-auto-deploy-tag
+  rules:
+    - <<: *unless-dev
+      when: never
+    - <<: *if-auto-deploy-tag
+      when: never
+    - <<: *if-tag
+
+.dev-tag-only-except-auto-deploy-manual: &dev-tag-only-except-auto-deploy-manual
+  rules:
+    - <<: *unless-dev
+      when: never
+    - <<: *if-auto-deploy-tag
+      when: never
+    - <<: *if-tag
+      when: manual
+
+.dev-tag-only-except-auto-deploy-or-rc: &dev-tag-only-except-auto-deploy-or-rc
+  rules:
+    - <<: *unless-dev
+      when: never
+    - <<: *if-rc-tag
+      when: never
+    - <<: *if-auto-deploy-tag
+      when: never
+    - <<: *if-tag
+
+.dev-tag-only-except-auto-deploy-or-rc-manual: &dev-tag-only-except-auto-deploy-or-rc-manual
+  rules:
+    - <<: *unless-dev
+      when: never
+    - <<: *if-rc-tag
+      when: never
+    - <<: *if-auto-deploy-tag
+      when: never
+    - <<: *if-tag
+      when: manual
+
+.dev-ce-tag-only-except-rc-manual: &dev-ce-tag-only-except-rc-manual
+  rules:
+    - <<: *unless-dev
+      when: never
+    - <<: *if-rc-tag
+      when: never
+    - <<: *if-ce-tag
+      when: manual
+
+.dev-ee-tag-only-except-auto-deploy-or-rc-manual: &dev-ee-tag-only-except-auto-deploy-or-rc-manual
+  rules:
+    - <<: *unless-dev
+      when: never
+    - <<: *if-rc-tag
+      when: never
+    - <<: *if-auto-deploy-tag
+      when: never
+    - <<: *if-ee-tag
+      when: manual
+
+.dev-ce-branch-only-except-nightly: &dev-ce-branch-only-except-nightly
+  rules:
+    - <<: *unless-dev
+      when: never
+    - <<: *if-ee-var
+      when: never
+    - <<: *if-ee-branch
+      when: never
+    # https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/3199
+    # removed RPi jobs from nightly.
+    - <<: *if-nightly
+      when: never
+    - <<: *if-branch
+
+.dev-ee-branch-only: &dev-ee-branch-only
+  rules:
+    - <<: *unless-dev
+      when: never
+    - if: '$CI_COMMIT_BRANCH == ""'
+      when: never
+    - <<: *if-ee-var
+    - <<: *if-ee-branch
+
+.dev-ce-tag-only: &dev-ce-tag-only
+  rules:
+    - <<: *unless-dev
+      when: never
+    - <<: *if-ce-tag
+
+.dev-ce-tag-only-manual: &dev-ce-tag-only-manual
+  rules:
+    - <<: *unless-dev
+      when: never
+    - <<: *if-ce-tag
+      when: manual
 .dev-ee-tag-only: &dev-ee-tag-only
-  only:
-    refs:
-      - /.*ee\.\d/@gitlab/omnibus-gitlab
-  <<: *except-auto-deploy-tag
-
-.dev-except-branches: &dev-except-branches
-   only:
-    - tags@gitlab/omnibus-gitlab
-    - schedules@gitlab/omnibus-gitlab
-    - triggers@gitlab/omnibus-gitlab
+  rules:
+    - <<: *unless-dev
+      when: never
+    - <<: *if-ee-tag
+
+.dev-ee-tag-only-manual: &dev-ee-tag-only-manual
+  rules:
+    - <<: *unless-dev
+      when: never
+    - <<: *if-ee-tag
+      when: manual
+
+.dev-tag-and-nightly: &dev-tag-and-nightly
+  rules:
+    - <<: *unless-dev
+      when: never
+    - <<: *if-tag
+    - <<: *if-nightly
+
+.dev-nightly-and-tag-except-auto-deploy: &dev-nightly-and-tag-except-auto-deploy
+  rules:
+    - <<: *unless-dev
+      when: never
+    - <<: *if-auto-deploy-tag
+      when: never
+    - <<: *if-tag
+    - <<: *if-nightly
+
+.dev-nightly-only-manual: &dev-nightly-only-manual
+  rules:
+    - <<: *unless-dev
+      when: never
+    - <<: *if-nightly
+      when: manual
+
+.dev-ee-nightly-only: &dev-ee-nightly-only
+  rules:
+    - <<: *unless-dev
+      when: never
+    - <<: *unless-nightly
+      when: never
+    - <<: *if-ee-var
 .package-artifacts: &package-artifacts
   when: always
@@ -76,7 +232,7 @@
     - bundle exec rspec --color --format RspecJunitFormatter --out junit_ci_rspec.xml --format documentation ci_build_specs
   tags:
   - docker-builder
-  <<: *regular-dev-branches-only
+  <<: *dev-branches-only
   artifacts:
     <<: *package-artifacts
     reports:
@@ -93,13 +249,7 @@
     NO_SOURCEMAPS: "true"
   tags:
   - docker-rpi-builder
-  except:
-    refs:
-      - /.*-stable-ee/
-    variables:
-      - $ee == "true"
-      - $NIGHTLY == "true"
-      - $CI_COMMIT_REF_NAME =~ /^[0-9]+-[0-9]+-auto-deploy-[0-9]+$/
+  <<: *dev-ce-branch-only-except-nightly
   artifacts: *package-artifacts
   retry: 1
@@ -140,14 +290,7 @@
     NO_SOURCEMAPS: "true"
   tags:
   - docker-rpi-builder
-  except:
-    refs:
-      - /.*ee\.\d/
-    variables:
-      - $ee == "true"
-      # Format of the auto-deploy tag for auto-deploy builds.
-      # https://gitlab.com/gitlab-org/release/docs/blob/master/general/deploy/auto-deploy.md#auto-deploy-tagging
-      - $CI_COMMIT_TAG =~ /^\d+\.\d+\.\d+\+[^ ]{7,}\.[^ ]{7,}$/
+  <<: *dev-ce-tag-only
 ### This stage is used for uploading packages
 ### 1. To a staging package repository
@@ -174,8 +317,7 @@
   script:
     - bundle exec rake repository:upload:production
   retry: 2
-  <<: *dev-tag-only-except-auto-deploy
-  when: manual
+  <<: *dev-tag-only-except-auto-deploy-manual
   cache: *tag-cache
   tags:
   - promotion
@@ -186,15 +328,7 @@
     RASPBERRY_REPO: "raspberry-pi2"
   stage: raspbian-release
   retry: 2
-  <<: *dev-tag-only-except-auto-deploy
-  except:
-    refs:
-      - /.*ee\.\d/
-    variables:
-      - $ee == "true"
-      # Format of the auto-deploy tag for auto-deploy builds.
-      # https://gitlab.com/gitlab-org/release/docs/blob/master/general/deploy/auto-deploy.md#auto-deploy-tagging
-      - $CI_COMMIT_TAG =~ /^\d+\.\d+\.\d+\+[^ ]{7,}\.[^ ]{7,}$/
+  <<: *dev-ce-tag-only-manual
 .aws_ami_build: &aws_ami_build
   stage: package-and-image-release
@@ -207,8 +341,7 @@
   allow_failure: true
   tags:
   - docker-builder
-  when: manual
-  <<: *dev-tag-only-except-auto-deploy
+  <<: *dev-tag-only-except-auto-deploy-or-rc-manual
   cache: *tag-cache
   dependencies: []
   retry: 1
@@ -217,7 +350,6 @@
   image: "${BUILDER_IMAGE_REGISTRY}/ruby_docker:${BUILDER_IMAGE_REVISION}"
   stage: aws-marketplace-release
   allow_failure: true
-  when: manual
   script:
     - curl --request POST
       --form "token=$AWS_RELEASE_TRIGGER_TOKEN"
@@ -225,15 +357,7 @@
       --form "variables[GITLAB_VERSION]=$CI_COMMIT_TAG"
       --form "variables[AWS_LISTING_NAME]=$AWS_LISTING_NAME"
       https://dev.gitlab.org/api/v4/projects/1035/trigger/pipeline
-  only:
-    refs:
-      - /.*ee\.\d/@gitlab/omnibus-gitlab
-  except:
-    variables:
-      - $CI_COMMIT_TAG =~ /^.*\+rc.*/
-      # Format of the auto-deploy tag for auto-deploy builds.
-      # https://gitlab.com/gitlab-org/release/docs/blob/master/general/deploy/auto-deploy.md#auto-deploy-tagging
-      - $CI_COMMIT_TAG =~ /^\d+\.\d+\.\d+\+[^ ]{7,}\.[^ ]{7,}$/
+  <<: *dev-ee-tag-only-except-auto-deploy-or-rc-manual
   dependencies:
     - AWS
@@ -278,9 +402,11 @@ OpenSUSE-15.1-branch:
 SLES-12.2-branch:
   image: "${BUILDER_IMAGE_REGISTRY}/sles_12sp2:${BUILDER_IMAGE_REVISION}"
   <<: *branch_build
+  <<: *dev-ee-branch-only
 SLES-12.5-branch:
   image: "${BUILDER_IMAGE_REGISTRY}/sles_12sp5:${BUILDER_IMAGE_REVISION}"
   <<: *branch_build
+  <<: *dev-ee-branch-only
 Docker-branch:
   extends: .docker_job
@@ -289,7 +415,7 @@ Docker-branch:
   - bundle exec rake docker:build:image
   - bundle exec rake docker:push:staging
   - bundle exec rake docker:push:nightly
-  <<: *regular-dev-branches-only
+  <<: *dev-branches-only
   tags:
   - docker-builder
   cache: *branch-cache
@@ -301,9 +427,7 @@ Docker-branch-measure-memory:
   stage: metrics
   # To be made back to automatic once
   # https://gitlab.com/gitlab-org/gitlab/-/issues/215000 is done.
-  when: manual
-  only:
-    - schedules@gitlab/omnibus-gitlab
+  <<: *dev-nightly-only-manual
   tags:
   - docker-builder
   cache: *branch-cache
@@ -324,7 +448,7 @@ QA-branch:
     paths:
     - screenshots/
   allow_failure: true
-  <<: *regular-dev-branches-only
+  <<: *dev-branches-only
   tags:
   - docker-builder
   cache: *branch-cache
@@ -343,11 +467,7 @@ HA-Validate-Nightly:
   script:
     - '[[ -n "${NIGHTLY}" ]] || exit 0'
     - bundle exec rake qa:ha:nightly
-  only:
-    refs:
-      - schedules@gitlab/omnibus-gitlab
-    variables:
-      - $ee == "true"
+  <<: *dev-ee-nightly-only
   cache: *branch-cache
   dependencies: []
@@ -363,8 +483,7 @@ Ubuntu-16.04:
   stage: gitlab_com:package
   extends: .tag_template
   # Also build for auto-deploy tags
-  except:
-    variables: []
+  <<: *dev-tag-only
 Ubuntu-18.04:
   image: "${BUILDER_IMAGE_REGISTRY}/ubuntu_18.04:${BUILDER_IMAGE_REVISION}"
@@ -410,9 +529,7 @@ Ubuntu-16.04-upload-deploy:
   script:
     - bundle exec rake repository:upload:staging[$STAGING_REPO]
     - if [ -n "$DEPLOYER_TRIGGER_TOKEN" ]; then bundle exec rake gitlab_com:deployer; fi
-  <<: *dev-except-branches
-  except:
-    variables: []
+  <<: *dev-tag-and-nightly
 Ubuntu-18.04-staging:
   stage: gitlab_com:upload_deploy
@@ -420,7 +537,7 @@ Ubuntu-18.04-staging:
   dependencies:
     - Ubuntu-18.04
     - Ubuntu-18.04-branch
-  <<: *dev-except-branches
+  <<: *dev-nightly-and-tag-except-auto-deploy
 ### Package builds
 Debian-9:
@@ -447,15 +564,11 @@ OpenSUSE-15.1:
 SLES-12.2:
   image: "${BUILDER_IMAGE_REGISTRY}/sles_12sp2:${BUILDER_IMAGE_REVISION}"
   <<: *tag_build
-  only:
-    refs:
-      - /.*ee\.\d/@gitlab/omnibus-gitlab
+  <<: *dev-ee-tag-only
 SLES-12.5:
-  image: "${BUILDER_IMAGE_REGISTRY}/sles_12sp5:${BUILDER_IMAGE_REVISION}"
+  image: "${BUILDER_IMAGE_REGISTRY}/sles_12sp2:${BUILDER_IMAGE_REVISION}"
   <<: *tag_build
-  only:
-    refs:
-      - /.*ee\.\d/@gitlab/omnibus-gitlab
+  <<: *dev-ee-tag-only
 create_omnibus_manifest:
   image: "${BUILDER_IMAGE_REGISTRY}/debian_10:${BUILDER_IMAGE_REVISION}"
@@ -464,8 +577,7 @@ create_omnibus_manifest:
     TERM: xterm-256color
   script:
     - bundle exec omnibus manifest gitlab -l nothing 2> /dev/null > version-manifest.json
-  <<: *dev-except-branches
-  <<: *except-auto-deploy-tag
+  <<: *dev-nightly-and-tag-except-auto-deploy
   cache: *tag-cache
   artifacts:
     expire_in: 7 days
@@ -494,8 +606,7 @@ dependency_scanning:
     # Temporarily hardcoding a working SHA. Check https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5530
     # for details
     - curl -sSL https://gitlab.com/gitlab-org/security-products/gitlab-depscan/raw/cae7e018a8c18582502f60c9d24e66b3c79b45ed/gitlab-depscan.sh|bash -s version-manifest.json
-  <<: *dev-except-branches
-  <<: *except-auto-deploy-tag
+  <<: *dev-nightly-and-tag-except-auto-deploy
   needs:
     - create_omnibus_manifest
   artifacts:
@@ -522,8 +633,7 @@ Docker:
 Docker-measure-memory:
   extends: .measure_memory_script_artifacts
   stage: metrics
-  <<: *dev-tag-only-except-auto-deploy
-  when: manual
+  <<: *dev-tag-only-except-auto-deploy-manual
   tags:
   - docker-builder
   cache: *tag-cache
@@ -570,11 +680,13 @@ SLES-12.2-staging:
     - SLES-12.2
     - SLES-12.2-branch
   <<: *staging_upload
+  <<: *dev-ee-tag-only-manual
 SLES-12.5-staging:
   dependencies:
     - SLES-12.5
     - SLES-12.5-branch
   <<: *staging_upload
+  <<: *dev-ee-tag-only-manual
 ### Package releases
 Ubuntu-16.04-release:
@@ -614,43 +726,39 @@ OpenSUSE-15.1-release:
     - OpenSUSE-15.1
   <<: *production_release
 SLES-12.2-release:
+  <<: *production_release
   dependencies:
     - SLES-12.2
-  only:
-    refs:
-      - /.*ee\.\d/@gitlab/omnibus-gitlab
-  <<: *production_release
+  <<: *dev-ee-tag-only-except-auto-deploy-or-rc-manual
 SLES-12.5-release:
+  <<: *production_release
   dependencies:
     - SLES-12.5
-  only:
-    refs:
-      - /.*ee\.\d/@gitlab/omnibus-gitlab
-  <<: *production_release
+  <<: *dev-ee-tag-only-except-auto-deploy-or-rc-manual
 Docker-Release:
   extends: .docker_job
   stage: package-and-image-release
   script:
     - bundle exec rake release:docker
-  <<: *dev-tag-only-except-auto-deploy
+  <<: *dev-tag-only-except-auto-deploy-manual
   cache: *tag-cache
   dependencies: []
-  when: manual
-AWS: *aws_ami_build
+AWS:
+  <<: *aws_ami_build
 AWS-Ultimate:
   variables:
     AWS_RELEASE_TYPE: "ultimate"
   <<: *aws_ami_build
-  <<: *dev-ee-tag-only
+  <<: *dev-ee-tag-only-except-auto-deploy-or-rc-manual
 AWS-Premium:
   variables:
     AWS_RELEASE_TYPE: "premium"
   <<: *aws_ami_build
-  <<: *dev-ee-tag-only
+  <<: *dev-ee-tag-only-except-auto-deploy-or-rc-manual
 QA-Tag:
   extends: .docker_job
@@ -665,14 +773,13 @@ QA-Tag:
     paths:
     - screenshots/
   allow_failure: true
-  <<: *dev-tag-only-except-auto-deploy
+  <<: *dev-tag-only-except-auto-deploy-manual
   tags:
   - docker-builder
   cache: *tag-cache
   dependencies:
     - Ubuntu-16.04
   retry: 1
-  when: manual
 upgrade:
   stage: metrics
@@ -681,8 +788,7 @@ upgrade:
     - echo $GCP_SERVICE_ACCOUNT > service_account.json
     - bundle exec rake metrics:upgrade_package
   allow_failure: true
-  when: manual
-  <<: *dev-tag-only-except-auto-deploy
+  <<: *dev-tag-only-except-auto-deploy-manual
   tags:
   - docker-builder
   cache: *tag-cache
@@ -694,8 +800,7 @@ license-upload:
   script:
     - bundle exec rake license:upload
   allow_failure: true
-  when: manual
-  <<: *dev-tag-only-except-auto-deploy
+  <<: *dev-tag-only-except-auto-deploy-manual
   tags:
   - docker-builder
   cache: *tag-cache
@@ -707,9 +812,7 @@ HA-Validate-Tagged:
   image: "dev.gitlab.org:5005/cookbooks/gitlab-omnibus-builder/ubuntu_16.04:${BUILDER_IMAGE_REVISION}"
   script:
     - bundle exec rake qa:ha:tag
-  only:
-    refs:
-      - /.*ee\.\d/@gitlab/omnibus-gitlab
+  <<: *dev-ee-tag-only
   cache: *tag-cache
   dependencies: []
   allow_failure: true
@@ -727,9 +830,7 @@ AWS-CE-Release:
   variables:
     AWS_LISTING_NAME: "GitLab Community Edition"
   extends: .aws_marketplace_release
-  only:
-    refs:
-      - /.*ce\.\d/@gitlab/omnibus-gitlab
+  <<: *dev-ce-tag-only-except-rc-manual
 AWS-EE-Ultimate-Release:
   variables: