Merge branch 'master' into 11-10-stable

lib/gitlab/ci/config/entry/service.rb

@@ -10,16 +10,18 @@ module Gitlab
         class Service < Image
           include ::Gitlab::Config::Entry::Validatable
-          ALLOWED_KEYS = %i[name entrypoint command alias].freeze
+          ALLOWED_KEYS = %i[name entrypoint command alias ports].freeze
           validations do
             validates :config, hash_or_string: true
             validates :config, allowed_keys: ALLOWED_KEYS
+            validates :config, disallowed_keys: %i[ports], unless: :with_image_ports?
             validates :name, type: String, presence: true
             validates :entrypoint, array_of_strings: true, allow_nil: true
             validates :command, array_of_strings: true, allow_nil: true
             validates :alias, type: String, allow_nil: true
+            validates :alias, type: String, presence: true, unless: ->(record) { record.ports.blank? }
           end
           def alias

lib/gitlab/ci/config/entry/services.rb

@@ -12,6 +12,7 @@ module Gitlab
           validations do
             validates :config, type: Array
+            validates :config, services_with_ports_alias_unique: true, if: ->(record) { record.opt(:with_image_ports) }
           end
           def compose!(deps = nil)
@@ -20,6 +21,7 @@ module Gitlab
               @config.each do |config|
                 @entries << ::Gitlab::Config::Entry::Factory.new(Entry::Service)
                   .value(config || {})
+                  .with(key: "service", parent: self, description: "service definition.") # rubocop:disable CodeReuse/ActiveRecord
                   .create!
               end

lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml

+performance:
+  stage: performance
+  image: docker:stable
+  allow_failure: true
+  services:
+    - docker:stable-dind
+  script:
+    - |
+      if ! docker info &>/dev/null; then
+        if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then
+          export DOCKER_HOST='tcp://localhost:2375'
+        fi
+      fi
+    - export CI_ENVIRONMENT_URL=$(cat environment_url.txt)
+    - mkdir gitlab-exporter
+    - wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/10-5/index.js
+    - mkdir sitespeed-results
+    - |
+      if [ -f .gitlab-urls.txt ]
+      then
+        sed -i -e 's@^@'"$CI_ENVIRONMENT_URL"'@' .gitlab-urls.txt
+        docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results .gitlab-urls.txt
+      else
+        docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL"
+      fi
+    - mv sitespeed-results/data/performance.json performance.json
+  artifacts:
+    paths:
+    - performance.json
+    - sitespeed-results/
+  only:
+    refs:
+      - branches
+      - tags
+    kubernetes: active
+  except:
+    variables:
+      - $PERFORMANCE_DISABLED

lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml

+build:
+  stage: build
+  image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image/master:stable"
+  services:
+    - docker:stable-dind
+  script:
+    - |
+      if [[ -z "$CI_COMMIT_TAG" ]]; then
+        export CI_APPLICATION_REPOSITORY=${CI_APPLICATION_REPOSITORY:-$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG}
+        export CI_APPLICATION_TAG=${CI_APPLICATION_TAG:-$CI_COMMIT_SHA}
+      else
+        export CI_APPLICATION_REPOSITORY=${CI_APPLICATION_REPOSITORY:-$CI_REGISTRY_IMAGE}
+        export CI_APPLICATION_TAG=${CI_APPLICATION_TAG:-$CI_COMMIT_TAG}
+      fi
+    - /build/build.sh
+  only:
+    - branches
+    - tags

lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml

+code_quality:
+  stage: test
+  image: docker:stable
+  allow_failure: true
+  services:
+    - docker:stable-dind
+  script:
+    - export CQ_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')
+    - |
+      if ! docker info &>/dev/null; then
+        if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then
+          export DOCKER_HOST='tcp://localhost:2375'
+        fi
+      fi
+    - |
+      docker run --env SOURCE_CODE="$PWD" \
+            --volume "$PWD":/code \
+            --volume /var/run/docker.sock:/var/run/docker.sock \
+            "registry.gitlab.com/gitlab-org/security-products/codequality:$CQ_VERSION" /code
+  artifacts:
+    paths: [gl-code-quality-report.json]
+  only:
+    - branches
+    - tags
+  except:
+    variables:
+      - $CODE_QUALITY_DISABLED

lib/gitlab/ci/templates/Jobs/DAST.gitlab-ci.yml

+dast:
+  stage: dast
+  image: docker:stable
+  variables:
+    DOCKER_DRIVER: overlay2
+  allow_failure: true
+  services:
+    - docker:stable-dind
+  script:
+    - export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)}
+    - export DAST_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')}
+    - |
+      if ! docker info &>/dev/null; then
+        if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then
+          export DOCKER_HOST='tcp://localhost:2375'
+        fi
+      fi
+    - |
+      function dast_run() {
+        docker run \
+        --env DAST_TARGET_AVAILABILITY_TIMEOUT \
+        --volume "$PWD:/output" \
+        --volume /var/run/docker.sock:/var/run/docker.sock \
+        -w /output \
+        "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION" \
+        /analyze -t $DAST_WEBSITE \
+        "$@"
+      }
+    - |
+      if [ -n "$DAST_AUTH_URL" ]
+      then
+        dast_run \
+          --auth-url $DAST_AUTH_URL \
+          --auth-username $DAST_USERNAME \
+          --auth-password $DAST_PASSWORD \
+          --auth-username-field $DAST_USERNAME_FIELD \
+          --auth-password-field $DAST_PASSWORD_FIELD
+      else
+        dast_run
+      fi
+  artifacts:
+    reports:
+      dast: gl-dast-report.json
+  only:
+    refs:
+      - branches
+      - tags
+    variables:
+      - $GITLAB_FEATURES =~ /\bdast\b/
+  except:
+    refs:
+      - master
+    variables:
+      - $DAST_DISABLED

lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml

+review:
+  stage: review
+  script:
+    - check_kube_domain
+    - install_dependencies
+    - download_chart
+    - ensure_namespace
+    - initialize_tiller
+    - create_secret
+    - deploy
+    - persist_environment_url
+  environment:
+    name: review/$CI_COMMIT_REF_NAME
+    url: http://$CI_PROJECT_ID-$CI_ENVIRONMENT_SLUG.$KUBE_INGRESS_BASE_DOMAIN
+    on_stop: stop_review
+  artifacts:
+    paths: [environment_url.txt]
+  only:
+    refs:
+      - branches
+      - tags
+    kubernetes: active
+  except:
+    refs:
+      - master
+    variables:
+      - $REVIEW_DISABLED
+
+stop_review:
+  stage: cleanup
+  variables:
+    GIT_STRATEGY: none
+  script:
+    - install_dependencies
+    - initialize_tiller
+    - delete
+  environment:
+    name: review/$CI_COMMIT_REF_NAME
+    action: stop
+  when: manual
+  allow_failure: true
+  only:
+    refs:
+      - branches
+      - tags
+    kubernetes: active
+  except:
+    refs:
+      - master
+    variables:
+      - $REVIEW_DISABLED
+
+# Staging deploys are disabled by default since
+# continuous deployment to production is enabled by default
+# If you prefer to automatically deploy to staging and
+# only manually promote to production, enable this job by setting
+# STAGING_ENABLED.
+
+staging:
+  stage: staging
+  script:
+    - check_kube_domain
+    - install_dependencies
+    - download_chart
+    - ensure_namespace
+    - initialize_tiller
+    - create_secret
+    - deploy
+  environment:
+    name: staging
+    url: http://$CI_PROJECT_PATH_SLUG-staging.$KUBE_INGRESS_BASE_DOMAIN
+  only:
+    refs:
+      - master
+    kubernetes: active
+    variables:
+      - $STAGING_ENABLED
+
+# Canaries are disabled by default, but if you want them,
+# and know what the downsides are, you can enable this by setting
+# CANARY_ENABLED.
+
+canary:
+  stage: canary
+  script:
+    - check_kube_domain
+    - install_dependencies
+    - download_chart
+    - ensure_namespace
+    - initialize_tiller
+    - create_secret
+    - deploy canary
+  environment:
+    name: production
+    url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN
+  when: manual
+  only:
+    refs:
+      - master
+    kubernetes: active
+    variables:
+      - $CANARY_ENABLED
+
+.production: &production_template
+  stage: production
+  script:
+    - check_kube_domain
+    - install_dependencies
+    - download_chart
+    - ensure_namespace
+    - initialize_tiller
+    - create_secret
+    - deploy
+    - delete canary
+    - delete rollout
+    - persist_environment_url
+  environment:
+    name: production
+    url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN
+  artifacts:
+    paths: [environment_url.txt]
+
+production:
+  <<: *production_template
+  only:
+    refs:
+      - master
+    kubernetes: active
+  except:
+    variables:
+      - $STAGING_ENABLED
+      - $CANARY_ENABLED
+      - $INCREMENTAL_ROLLOUT_ENABLED
+      - $INCREMENTAL_ROLLOUT_MODE
+
+production_manual:
+  <<: *production_template
+  when: manual
+  allow_failure: false
+  only:
+    refs:
+      - master
+    kubernetes: active
+    variables:
+      - $STAGING_ENABLED
+      - $CANARY_ENABLED
+  except:
+    variables:
+      - $INCREMENTAL_ROLLOUT_ENABLED
+      - $INCREMENTAL_ROLLOUT_MODE
+
+# This job implements incremental rollout on for every push to `master`.
+
+.rollout: &rollout_template
+  script:
+    - check_kube_domain
+    - install_dependencies
+    - download_chart
+    - ensure_namespace
+    - initialize_tiller
+    - create_secret
+    - deploy rollout $ROLLOUT_PERCENTAGE
+    - scale stable $((100-ROLLOUT_PERCENTAGE))
+    - delete canary
+    - persist_environment_url
+  environment:
+    name: production
+    url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN
+  artifacts:
+    paths: [environment_url.txt]
+
+.manual_rollout_template: &manual_rollout_template
+  <<: *rollout_template
+  stage: production
+  when: manual
+  # This selectors are backward compatible mode with $INCREMENTAL_ROLLOUT_ENABLED (before 11.4)
+  only:
+    refs:
+      - master
+    kubernetes: active
+    variables:
+      - $INCREMENTAL_ROLLOUT_MODE == "manual"
+      - $INCREMENTAL_ROLLOUT_ENABLED
+  except:
+    variables:
+      - $INCREMENTAL_ROLLOUT_MODE == "timed"
+
+.timed_rollout_template: &timed_rollout_template
+  <<: *rollout_template
+  when: delayed
+  start_in: 5 minutes
+  only:
+    refs:
+      - master
+    kubernetes: active
+    variables:
+      - $INCREMENTAL_ROLLOUT_MODE == "timed"
+
+timed rollout 10%:
+  <<: *timed_rollout_template
+  stage: incremental rollout 10%
+  variables:
+    ROLLOUT_PERCENTAGE: 10
+
+timed rollout 25%:
+  <<: *timed_rollout_template
+  stage: incremental rollout 25%
+  variables:
+    ROLLOUT_PERCENTAGE: 25
+
+timed rollout 50%:
+  <<: *timed_rollout_template
+  stage: incremental rollout 50%
+  variables:
+    ROLLOUT_PERCENTAGE: 50
+
+timed rollout 100%:
+  <<: *timed_rollout_template
+  <<: *production_template
+  stage: incremental rollout 100%
+  variables:
+    ROLLOUT_PERCENTAGE: 100
+
+rollout 10%:
+  <<: *manual_rollout_template
+  variables:
+    ROLLOUT_PERCENTAGE: 10
+
+rollout 25%:
+  <<: *manual_rollout_template
+  variables:
+    ROLLOUT_PERCENTAGE: 25
+
+rollout 50%:
+  <<: *manual_rollout_template
+  variables:
+    ROLLOUT_PERCENTAGE: 50
+
+rollout 100%:
+  <<: *manual_rollout_template
+  <<: *production_template
+  allow_failure: false
+
+.deploy_helpers: &deploy_helpers |
+  [[ "$TRACE" ]] && set -x
+  auto_database_url=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${CI_ENVIRONMENT_SLUG}-postgres:5432/${POSTGRES_DB}
+  export DATABASE_URL=${DATABASE_URL-$auto_database_url}
+  export TILLER_NAMESPACE=$KUBE_NAMESPACE
+  # Extract "MAJOR.MINOR" from CI_SERVER_VERSION and generate "MAJOR-MINOR-stable" for Security Products
+
+  function get_replicas() {
+    track="${1:-stable}"
+    percentage="${2:-100}"
+
+    env_track=$( echo $track | tr -s  '[:lower:]'  '[:upper:]' )
+    env_slug=$( echo ${CI_ENVIRONMENT_SLUG//-/_} | tr -s  '[:lower:]'  '[:upper:]' )
+
+    if [[ "$track" == "stable" ]] || [[ "$track" == "rollout" ]]; then
+      # for stable track get number of replicas from `PRODUCTION_REPLICAS`
+      eval new_replicas=\$${env_slug}_REPLICAS
+      if [[ -z "$new_replicas" ]]; then
+        new_replicas=$REPLICAS
+      fi
+    else
+      # for all tracks get number of replicas from `CANARY_PRODUCTION_REPLICAS`
+      eval new_replicas=\$${env_track}_${env_slug}_REPLICAS
+      if [[ -z "$new_replicas" ]]; then
+        eval new_replicas=\${env_track}_REPLICAS
+      fi
+    fi
+
+    replicas="${new_replicas:-1}"
+    replicas="$(($replicas * $percentage / 100))"
+
+    # always return at least one replicas
+    if [[ $replicas -gt 0 ]]; then
+      echo "$replicas"
+    else
+      echo 1
+    fi
+  }
+
+  # Extracts variables prefixed with K8S_SECRET_
+  # and creates a Kubernetes secret.
+  #
+  # e.g. If we have the following environment variables:
+  #   K8S_SECRET_A=value1
+  #   K8S_SECRET_B=multi\ word\ value
+  #
+  # Then we will create a secret with the following key-value pairs:
+  #   data:
+  #     A: dmFsdWUxCg==
+  #     B: bXVsdGkgd29yZCB2YWx1ZQo=
+  function create_application_secret() {
+    track="${1-stable}"
+    export APPLICATION_SECRET_NAME=$(application_secret_name "$track")
+
+    env | sed -n "s/^K8S_SECRET_\(.*\)$/\1/p" > k8s_prefixed_variables
+
+    kubectl create secret \
+      -n "$KUBE_NAMESPACE" generic "$APPLICATION_SECRET_NAME" \
+      --from-env-file k8s_prefixed_variables -o yaml --dry-run |
+      kubectl replace -n "$KUBE_NAMESPACE" --force -f -
+
+    export APPLICATION_SECRET_CHECKSUM=$(cat k8s_prefixed_variables | sha256sum | cut -d ' ' -f 1)
+
+    rm k8s_prefixed_variables
+  }
+
+  function deploy_name() {
+    name="$CI_ENVIRONMENT_SLUG"
+    track="${1-stable}"
+
+    if [[ "$track" != "stable" ]]; then
+      name="$name-$track"
+    fi
+
+    echo $name
+  }
+
+  function application_secret_name() {
+    track="${1-stable}"
+    name=$(deploy_name "$track")
+
+    echo "${name}-secret"
+  }
+
+  function deploy() {
+    track="${1-stable}"
+    percentage="${2:-100}"
+    name=$(deploy_name "$track")
+
+    if [[ -z "$CI_COMMIT_TAG" ]]; then
+      image_repository=${CI_APPLICATION_REPOSITORY:-$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG}
+      image_tag=${CI_APPLICATION_TAG:-$CI_COMMIT_SHA}
+    else
+      image_repository=${CI_APPLICATION_REPOSITORY:-$CI_REGISTRY_IMAGE}
+      image_tag=${CI_APPLICATION_TAG:-$CI_COMMIT_TAG}
+    fi
+
+    replicas="1"
+    service_enabled="true"
+    postgres_enabled="$POSTGRES_ENABLED"
+
+    # if track is different than stable,
+    # re-use all attached resources
+    if [[ "$track" != "stable" ]]; then
+      service_enabled="false"
+      postgres_enabled="false"
+    fi
+
+    replicas=$(get_replicas "$track" "$percentage")
+
+    if [[ "$CI_PROJECT_VISIBILITY" != "public" ]]; then
+      secret_name='gitlab-registry'
+    else
+      secret_name=''
+    fi
+
+    create_application_secret "$track"
+
+    env_slug=$(echo ${CI_ENVIRONMENT_SLUG//-/_} | tr -s '[:lower:]' '[:upper:]')
+    eval env_ADDITIONAL_HOSTS=\$${env_slug}_ADDITIONAL_HOSTS
+    if [ -n "$env_ADDITIONAL_HOSTS" ]; then
+      additional_hosts="{$env_ADDITIONAL_HOSTS}"
+    elif [ -n "$ADDITIONAL_HOSTS" ]; then
+      additional_hosts="{$ADDITIONAL_HOSTS}"
+    fi
+
+    if [[ -n "$DB_INITIALIZE" && -z "$(helm ls -q "^$name$")" ]]; then
+      echo "Deploying first release with database initialization..."
+      helm upgrade --install \
+        --wait \
+        --set service.enabled="$service_enabled" \
+        --set gitlab.app="$CI_PROJECT_PATH_SLUG" \
+        --set gitlab.env="$CI_ENVIRONMENT_SLUG" \
+        --set releaseOverride="$CI_ENVIRONMENT_SLUG" \
+        --set image.repository="$image_repository" \
+        --set image.tag="$image_tag" \
+        --set image.pullPolicy=IfNotPresent \
+        --set image.secrets[0].name="$secret_name" \
+        --set application.track="$track" \
+        --set application.database_url="$DATABASE_URL" \
+        --set application.secretName="$APPLICATION_SECRET_NAME" \
+        --set application.secretChecksum="$APPLICATION_SECRET_CHECKSUM" \
+        --set service.commonName="le.$KUBE_INGRESS_BASE_DOMAIN" \
+        --set service.url="$CI_ENVIRONMENT_URL" \
+        --set service.additionalHosts="$additional_hosts" \
+        --set replicaCount="$replicas" \
+        --set postgresql.enabled="$postgres_enabled" \
+        --set postgresql.nameOverride="postgres" \
+        --set postgresql.postgresUser="$POSTGRES_USER" \
+        --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \
+        --set postgresql.postgresDatabase="$POSTGRES_DB" \
+        --set postgresql.imageTag="$POSTGRES_VERSION" \
+        --set application.initializeCommand="$DB_INITIALIZE" \
+        --namespace="$KUBE_NAMESPACE" \
+        "$name" \
+        chart/
+
+      echo "Deploying second release..."
+      helm upgrade --reuse-values \
+        --wait \
+        --set application.initializeCommand="" \
+        --set application.migrateCommand="$DB_MIGRATE" \
+        --namespace="$KUBE_NAMESPACE" \
+        "$name" \
+        chart/
+    else
+      echo "Deploying new release..."
+      helm upgrade --install \
+        --wait \
+        --set service.enabled="$service_enabled" \
+        --set gitlab.app="$CI_PROJECT_PATH_SLUG" \
+        --set gitlab.env="$CI_ENVIRONMENT_SLUG" \
+        --set releaseOverride="$CI_ENVIRONMENT_SLUG" \
+        --set image.repository="$image_repository" \
+        --set image.tag="$image_tag" \
+        --set image.pullPolicy=IfNotPresent \
+        --set image.secrets[0].name="$secret_name" \
+        --set application.track="$track" \
+        --set application.database_url="$DATABASE_URL" \
+        --set application.secretName="$APPLICATION_SECRET_NAME" \
+        --set application.secretChecksum="$APPLICATION_SECRET_CHECKSUM" \
+        --set service.commonName="le.$KUBE_INGRESS_BASE_DOMAIN" \
+        --set service.url="$CI_ENVIRONMENT_URL" \
+        --set service.additionalHosts="$additional_hosts" \
+        --set replicaCount="$replicas" \
+        --set postgresql.enabled="$postgres_enabled" \
+        --set postgresql.nameOverride="postgres" \
+        --set postgresql.postgresUser="$POSTGRES_USER" \
+        --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \
+        --set postgresql.postgresDatabase="$POSTGRES_DB" \
+        --set application.migrateCommand="$DB_MIGRATE" \
+        --namespace="$KUBE_NAMESPACE" \
+        "$name" \
+        chart/
+    fi
+
+    kubectl rollout status -n "$KUBE_NAMESPACE" -w "$ROLLOUT_RESOURCE_TYPE/$name"
+  }
+
+  function scale() {
+    track="${1-stable}"
+    percentage="${2-100}"
+    name=$(deploy_name "$track")
+
+    replicas=$(get_replicas "$track" "$percentage")
+
+    if [[ -n "$(helm ls -q "^$name$")" ]]; then
+      helm upgrade --reuse-values \
+        --wait \
+        --set replicaCount="$replicas" \
+        --namespace="$KUBE_NAMESPACE" \
+        "$name" \
+        chart/
+    fi
+  }
+
+  function install_dependencies() {
+    apk add -U openssl curl tar gzip bash ca-certificates git
+    curl -sSL -o /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub
+    curl -sSL -O https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.28-r0/glibc-2.28-r0.apk
+    apk add glibc-2.28-r0.apk
+    rm glibc-2.28-r0.apk
+
+    curl -sS "https://kubernetes-helm.storage.googleapis.com/helm-v${HELM_VERSION}-linux-amd64.tar.gz" | tar zx
+    mv linux-amd64/helm /usr/bin/
+    mv linux-amd64/tiller /usr/bin/
+    helm version --client
+    tiller -version
+
+    curl -sSL -o /usr/bin/kubectl "https://storage.googleapis.com/kubernetes-release/release/v${KUBERNETES_VERSION}/bin/linux/amd64/kubectl"
+    chmod +x /usr/bin/kubectl
+    kubectl version --client
+  }
+
+  function download_chart() {
+    if [[ ! -d chart ]]; then
+      auto_chart=${AUTO_DEVOPS_CHART:-gitlab/auto-deploy-app}
+      auto_chart_name=$(basename $auto_chart)
+      auto_chart_name=${auto_chart_name%.tgz}
+      auto_chart_name=${auto_chart_name%.tar.gz}
+    else
+      auto_chart="chart"
+      auto_chart_name="chart"
+    fi
+
+    helm init --client-only
+    helm repo add gitlab ${AUTO_DEVOPS_CHART_REPOSITORY:-https://charts.gitlab.io}
+    if [[ ! -d "$auto_chart" ]]; then
+      helm fetch ${auto_chart} --untar
+    fi
+    if [ "$auto_chart_name" != "chart" ]; then
+      mv ${auto_chart_name} chart
+    fi
+
+    helm dependency update chart/
+    helm dependency build chart/
+  }
+
+  function ensure_namespace() {
+    kubectl describe namespace "$KUBE_NAMESPACE" || kubectl create namespace "$KUBE_NAMESPACE"
+  }
+
+  # Function to ensure backwards compatibility with AUTO_DEVOPS_DOMAIN
+  function ensure_kube_ingress_base_domain() {
+    if [ -z ${KUBE_INGRESS_BASE_DOMAIN+x} ] && [ -n "$AUTO_DEVOPS_DOMAIN" ] ; then
+      export KUBE_INGRESS_BASE_DOMAIN=$AUTO_DEVOPS_DOMAIN
+    fi
+  }
+
+  function check_kube_domain() {
+    ensure_kube_ingress_base_domain
+
+    if [[ -z "$KUBE_INGRESS_BASE_DOMAIN" ]]; then
+      echo "In order to deploy or use Review Apps,"
+      echo "AUTO_DEVOPS_DOMAIN or KUBE_INGRESS_BASE_DOMAIN variables must be set"
+      echo "From 11.8, you can set KUBE_INGRESS_BASE_DOMAIN in cluster settings"
+      echo "or by defining a variable at group or project level."
+      echo "You can also manually add it in .gitlab-ci.yml"
+      echo "AUTO_DEVOPS_DOMAIN support will be dropped on 12.0"
+      false
+    else
+      true
+    fi
+  }
+
+  function initialize_tiller() {
+    echo "Checking Tiller..."
+
+    export HELM_HOST="localhost:44134"
+    tiller -listen ${HELM_HOST} -alsologtostderr > /dev/null 2>&1 &
+    echo "Tiller is listening on ${HELM_HOST}"
+
+    if ! helm version --debug; then
+      echo "Failed to init Tiller."
+      return 1
+    fi
+    echo ""
+  }
+
+  function create_secret() {
+    echo "Create secret..."
+    if [[ "$CI_PROJECT_VISIBILITY" == "public" ]]; then
+      return
+    fi
+
+    kubectl create secret -n "$KUBE_NAMESPACE" \
+      docker-registry gitlab-registry \
+      --docker-server="$CI_REGISTRY" \
+      --docker-username="${CI_DEPLOY_USER:-$CI_REGISTRY_USER}" \
+      --docker-password="${CI_DEPLOY_PASSWORD:-$CI_REGISTRY_PASSWORD}" \
+      --docker-email="$GITLAB_USER_EMAIL" \
+      -o yaml --dry-run | kubectl replace -n "$KUBE_NAMESPACE" --force -f -
+  }
+
+  function persist_environment_url() {
+      echo $CI_ENVIRONMENT_URL > environment_url.txt
+  }
+
+  function delete() {
+    track="${1-stable}"
+    name=$(deploy_name "$track")
+
+    if [[ -n "$(helm ls -q "^$name$")" ]]; then
+      helm delete --purge "$name"
+    fi
+
+    secret_name=$(application_secret_name "$track")
+    kubectl delete secret --ignore-not-found -n "$KUBE_NAMESPACE" "$secret_name"
+  }
+
+before_script:
+  - *deploy_helpers

lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml

+test:
+  services:
+    - postgres:latest
+  variables:
+    POSTGRES_DB: test
+  stage: test
+  image: gliderlabs/herokuish:latest
+  script:
+    - |
+      if [ -z ${KUBERNETES_PORT+x} ]; then
+        DB_HOST=postgres
+      else
+        DB_HOST=localhost
+      fi
+    - export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${DB_HOST}:5432/${POSTGRES_DB}"
+    - cp -R . /tmp/app
+    - /bin/herokuish buildpack test
+  only:
+    - branches
+    - tags
+  except:
+    variables:
+      - $TEST_DISABLED

lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml

@@ -28,6 +28,12 @@ container_scanning:
     - docker:stable-dind
   script:
     - if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then { export DOCKER_SERVICE="localhost" ; export DOCKER_HOST="tcp://${DOCKER_SERVICE}:2375" ; } fi
+    - |
+      if [[ -n "$CI_REGISTRY_USER" ]]; then
+        echo "Logging to GitLab Container Registry with CI credentials..."
+        docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY"
+        echo ""
+      fi
     - docker run -d --name db arminc/clair-db:latest
     - docker run -p 6060:6060 --link db:postgres -d --name clair --restart on-failure arminc/clair-local-scan:${CLAIR_LOCAL_SCAN_VERSION}
     - apk add -U wget ca-certificates
@@ -36,7 +42,6 @@ container_scanning:
     - mv clair-scanner_linux_amd64 clair-scanner
     - chmod +x clair-scanner
     - touch clair-whitelist.yml
-    - while( ! wget -q -O /dev/null http://${DOCKER_SERVICE}:6060/v1/namespaces ) ; do sleep 1 ; done
     - retries=0
     - echo "Waiting for clair daemon to start"
     - while( ! wget -T 10 -q -O /dev/null http://${DOCKER_SERVICE}:6060/v1/namespaces ) ; do sleep 1 ; echo -n "." ; if [ $retries -eq 10 ] ; then echo " Timeout, aborting." ; exit 1 ; fi ; retries=$(($retries+1)) ; done

lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml

@@ -4,6 +4,9 @@
 # List of the variables: https://gitlab.com/gitlab-org/security-products/dast#settings
 # How to set: https://docs.gitlab.com/ee/ci/yaml/#variables
+include:
+  - template: Jobs/DAST.gitlab-ci.yml
+
 variables:
   DAST_WEBSITE: http://example.com # Please edit to be your website to scan for vulnerabilities
@@ -14,46 +17,10 @@ stages:
   - dast
 dast:
-  stage: dast
-  image: docker:stable
-  variables:
-    DOCKER_DRIVER: overlay2
-  allow_failure: true
-  services:
-    - docker:stable-dind
-  script:
-    - export DAST_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')}
-    - |
-      function dast_run() {
-        docker run \
-        --env DAST_TARGET_AVAILABILITY_TIMEOUT \
-        --volume "$PWD:/output" \
-        --volume /var/run/docker.sock:/var/run/docker.sock \
-        -w /output \
-        "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION" \
-        /analyze -t $DAST_WEBSITE \
-        "$@"
-      }
-    - |
-      if [ -n "$DAST_AUTH_URL" ]
-      then
-        dast_run \
-          --auth-url $DAST_AUTH_URL \
-          --auth-username $DAST_USERNAME \
-          --auth-password $DAST_PASSWORD \
-          --auth-username-field $DAST_USERNAME_FIELD \
-          --auth-password-field $DAST_PASSWORD_FIELD
-      else
-        dast_run
-      fi
-  artifacts:
-    reports:
-      dast: gl-dast-report.json
   only:
     refs:
       - branches
-    variables:
-      - $GITLAB_FEATURES =~ /\bdast\b/
   except:
+    refs: [] # Override default from template
     variables:
       - $DAST_DISABLED

lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml

@@ -14,6 +14,12 @@ dependency_scanning:
     - docker:stable-dind
   script:
     - export DS_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')}
+    - |
+      if ! docker info &>/dev/null; then
+        if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then
+          export DOCKER_HOST='tcp://localhost:2375'
+        fi
+      fi
     - |
       docker run \
         --env DS_ANALYZER_IMAGES \

lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml

@@ -14,6 +14,12 @@ sast:
     - docker:stable-dind
   script:
     - export SAST_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')}
+    - |
+      if ! docker info &>/dev/null; then
+        if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then
+          export DOCKER_HOST='tcp://localhost:2375'
+        fi
+      fi
     - |
       docker run \
         --env SAST_ANALYZER_IMAGES \

lib/gitlab/config/entry/configurable.rb

@@ -21,7 +21,7 @@ module Gitlab
           include Validatable
           validations do
-            validates :config, type: Hash
+            validates :config, type: Hash, unless: :skip_config_hash_validation?
           end
         end
@@ -30,6 +30,10 @@ module Gitlab
           return unless valid?
           self.class.nodes.each do |key, factory|
+            # If we override the config type validation
+            # we can end with different config types like String
+            next unless config.is_a?(Hash)
+
             factory
               .value(config[key])
               .with(key: key, parent: self)
@@ -45,6 +49,10 @@ module Gitlab
         end
         # rubocop: enable CodeReuse/ActiveRecord
+        def skip_config_hash_validation?
+          false
+        end
+
         class_methods do
           def nodes
             Hash[(@nodes || {}).map { |key, factory| [key, factory.dup] }]

lib/gitlab/config/entry/factory.rb

@@ -61,7 +61,7 @@ module Gitlab
         end
         def fabricate(entry, value = nil)
-          entry.new(value, @metadata).tap do |node|
+          entry.new(value, @metadata) do |node|
             node.key = @attributes[:key]
             node.parent = @attributes[:parent]
             node.default = @attributes[:default]

lib/gitlab/config/entry/node.rb

@@ -17,6 +17,8 @@ module Gitlab
           @metadata = metadata
           @entries = {}
+          yield(self) if block_given?
+
           self.class.aspects.to_a.each do |aspect|
             instance_exec(&aspect)
           end
@@ -44,6 +46,12 @@ module Gitlab
           @parent ? @parent.ancestors + [@parent] : []
         end
+        def opt(key)
+          opt = metadata[key]
+          opt = @parent.opt(key) if opt.nil? && @parent
+          opt
+        end
+
         def valid?
           errors.none?
         end
@@ -85,6 +93,18 @@ module Gitlab
           "#<#{self.class.name} #{unspecified}{#{key}: #{val.inspect}}>"
         end
+        def hash?
+          @config.is_a?(Hash)
+        end
+
+        def string?
+          @config.is_a?(String)
+        end
+
+        def integer?
+          @config.is_a?(Integer)
+        end
+
         def self.default(**)
         end

lib/gitlab/config/entry/simplifiable.rb

@@ -19,7 +19,10 @@ module Gitlab
           entry = self.class.entry_class(strategy)
-          super(@subject = entry.new(config, metadata))
+          @subject = entry.new(config, metadata)
+
+          yield(@subject) if block_given?
+          super(@subject)
         end
         def self.strategy(name, **opts)

lib/gitlab/config/entry/validators.rb

@@ -15,6 +15,17 @@ module Gitlab
           end
         end
+        class DisallowedKeysValidator < ActiveModel::EachValidator
+          def validate_each(record, attribute, value)
+            present_keys = value.try(:keys).to_a & options[:in]
+
+            if present_keys.any?
+              record.errors.add(attribute, "contains disallowed keys: " +
+                present_keys.join(', '))
+            end
+          end
+        end
+
         class AllowedValuesValidator < ActiveModel::EachValidator
           def validate_each(record, attribute, value)
             unless options[:in].include?(value.to_s)
@@ -186,6 +197,97 @@ module Gitlab
             end
           end
         end
+
+        class PortNamePresentAndUniqueValidator < ActiveModel::EachValidator
+          def validate_each(record, attribute, value)
+            return unless value.is_a?(Array)
+
+            ports_size = value.count
+            return if ports_size <= 1
+
+            named_ports = value.select { |e| e.is_a?(Hash) }.map { |e| e[:name] }.compact.map(&:downcase)
+
+            if ports_size != named_ports.size
+              record.errors.add(attribute, 'when there is more than one port, a unique name should be added')
+            end
+
+            if ports_size != named_ports.uniq.size
+              record.errors.add(attribute, 'each port name must be different')
+            end
+          end
+        end
+
+        class PortUniqueValidator < ActiveModel::EachValidator
+          def validate_each(record, attribute, value)
+            value = ports(value)
+            return unless value.is_a?(Array)
+
+            ports_size = value.count
+            return if ports_size <= 1
+
+            if transform_ports(value).size != ports_size
+              record.errors.add(attribute, 'each port number can only be referenced once')
+            end
+          end
+
+          private
+
+          def ports(current_data)
+            current_data
+          end
+
+          def transform_ports(raw_ports)
+            raw_ports.map do |port|
+              case port
+              when Integer
+                port
+              when Hash
+                port[:number]
+              end
+            end.uniq
+          end
+        end
+
+        class JobPortUniqueValidator < PortUniqueValidator
+          private
+
+          def ports(current_data)
+            return unless current_data.is_a?(Hash)
+
+            (image_ports(current_data) + services_ports(current_data)).compact
+          end
+
+          def image_ports(current_data)
+            return [] unless current_data[:image].is_a?(Hash)
+
+            current_data.dig(:image, :ports).to_a
+          end
+
+          def services_ports(current_data)
+            current_data.dig(:services).to_a.flat_map { |service| service.is_a?(Hash) ? service[:ports] : nil }
+          end
+        end
+
+        class ServicesWithPortsAliasUniqueValidator < ActiveModel::EachValidator
+          def validate_each(record, attribute, value)
+            current_aliases = aliases(value)
+            return if current_aliases.empty?
+
+            unless aliases_unique?(current_aliases)
+              record.errors.add(:config, 'alias must be unique in services with ports')
+            end
+          end
+
+          private
+
+          def aliases(value)
+            value.select { |s| s.is_a?(Hash) && s[:ports] }.pluck(:alias) # rubocop:disable CodeReuse/ActiveRecord
+          end
+
+          def aliases_unique?(aliases)
+            aliases.size == aliases.uniq.size
+          end
+        end
       end
     end
   end

lib/gitlab/graphql/tracing.rb

-# frozen_string_literal: true
-
-module Gitlab
-  module Graphql
-    class Tracing < GraphQL::Tracing::PlatformTracing
-      self.platform_keys = {
-        'lex' => 'graphql.lex',
-        'parse' => 'graphql.parse',
-        'validate' => 'graphql.validate',
-        'analyze_query' => 'graphql.analyze',
-        'analyze_multiplex' => 'graphql.analyze',
-        'execute_multiplex' => 'graphql.execute',
-        'execute_query' => 'graphql.execute',
-        'execute_query_lazy' => 'graphql.execute',
-        'execute_field' => 'graphql.execute',
-        'execute_field_lazy' => 'graphql.execute'
-      }
-
-      def platform_field_key(type, field)
-        "#{type.name}.#{field.name}"
-      end
-
-      def platform_trace(platform_key, key, data, &block)
-        start = Gitlab::Metrics::System.monotonic_time
-
-        yield
-      ensure
-        duration = Gitlab::Metrics::System.monotonic_time - start
-
-        graphql_duration_seconds.observe({ platform_key: platform_key, key: key }, duration)
-      end
-
-      private
-
-      def graphql_duration_seconds
-        @graphql_duration_seconds ||= Gitlab::Metrics.histogram(
-          :graphql_duration_seconds,
-          'GraphQL execution time'
-        )
-      end
-    end
-  end
-end

lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb

@@ -7,12 +7,16 @@ module Gitlab
         include QueryAdditionalMetrics
         # rubocop: disable CodeReuse/ActiveRecord
-        def query(environment_id)
+        def query(environment_id, timeframe_start = 8.hours.ago, timeframe_end = Time.now)
           ::Environment.find_by(id: environment_id).try do |environment|
             query_metrics(
               environment.project,
               environment,
-              common_query_context(environment, timeframe_start: 8.hours.ago.to_f, timeframe_end: Time.now.to_f)
+              common_query_context(
+                environment,
+                timeframe_start: timeframe_start.to_f,
+                timeframe_end: timeframe_end.to_f
+              )
             )
           end
         end