From 139e57a4769057c45cf686e074d3b315de0cfeb7 Mon Sep 17 00:00:00 2001
From: Mark Pundsack <mpundsack@gitlab.com>
Date: Mon, 13 Mar 2017 15:09:21 -0500
Subject: [PATCH 1/5] Create auto-deploy-kubernetes
---
Auto-deploy-kubernetes.yml | 250 +++++++++++++++++++++++++++++++++++++
1 file changed, 250 insertions(+)
create mode 100644 Auto-deploy-kubernetes.yml
diff --git a/Auto-deploy-kubernetes.yml b/Auto-deploy-kubernetes.yml
new file mode 100644
index 0000000..077f2c9
--- /dev/null
+++ b/Auto-deploy-kubernetes.yml
@@ -0,0 +1,250 @@
+# Explanation on the scripts:
+# https://gitlab.com/gitlab-examples/kubernetes-deploy/blob/master/README.md
+image: registry.gitlab.com/gitlab-examples/kubernetes-deploy
+
+variables:
+ # Application deployment domain
+ KUBE_DOMAIN: domain.example.com
+
+stages:
+ - build
+ - test
+ - review
+ - staging
+ - production
+ - cleanup
+
+# Define some helper functions
+.common: &common
+ |
+ set -eo pipefail
+
+ [[ "$TRACE" ]] && set -x
+
+ export CI_CONTAINER_NAME="ci_job_build_$CI_BUILD_ID"
+ export CI_REGISTRY_TAG="$CI_BUILD_REF_NAME"
+
+ create_kubeconfig() {
+ echo "Generating kubeconfig..."
+ export KUBECONFIG="$(pwd)/kubeconfig"
+ export KUBE_CLUSTER_OPTIONS=
+ if [[ -n "$KUBE_CA_PEM" ]]; then
+ echo "Using KUBE_CA_PEM..."
+ echo "$KUBE_CA_PEM" > "$(pwd)/kube.ca.pem"
+ export KUBE_CLUSTER_OPTIONS=--certificate-authority="$(pwd)/kube.ca.pem"
+ fi
+ kubectl config set-cluster gitlab-deploy --server="$KUBE_URL" $KUBE_CLUSTER_OPTIONS
+ kubectl config set-credentials gitlab-deploy --token="$KUBE_TOKEN" $KUBE_CLUSTER_OPTIONS
+ kubectl config set-context gitlab-deploy --cluster=gitlab-deploy --user=gitlab-deploy --namespace="$KUBE_NAMESPACE"
+ kubectl config use-context gitlab-deploy
+ echo ""
+ }
+
+ ensure_deploy_variables() {
+ if [[ -z "$KUBE_URL" ]]; then
+ echo "Missing KUBE_URL."
+ exit 1
+ fi
+
+ if [[ -z "$KUBE_TOKEN" ]]; then
+ echo "Missing KUBE_TOKEN."
+ exit 1
+ fi
+
+ if [[ -z "$KUBE_NAMESPACE" ]]; then
+ echo "Missing KUBE_NAMESPACE."
+ exit 1
+ fi
+
+ if [[ -z "$CI_ENVIRONMENT_SLUG" ]]; then
+ echo "Missing CI_ENVIRONMENT_SLUG."
+ exit 1
+ fi
+
+ if [[ -z "$CI_ENVIRONMENT_URL" ]]; then
+ echo "Missing CI_ENVIRONMENT_URL."
+ exit 1
+ fi
+ }
+
+ ping_kube() {
+ if kubectl version > /dev/null; then
+ echo "Kubernetes is online!"
+ echo ""
+ else
+ echo "Cannot connect to Kubernetes."
+ return 1
+ fi
+ }
+
+.auto_build: &auto_build
+ |
+ echo "Checking docker engine..."
+ if ! docker info &>/dev/null; then echo "Missing docker engine to build images."; echo "Running docker:dind locally with graph driver pointing to '/cache/docker'"
+ if ! grep -q overlay /proc/filesystems; then
+ echo "Missing overlay filesystem. Are you running recent enough kernel?"
+ exit 1
+ fi
+ if [[ ! -d /cache ]]; then
+ mkdir -p /cache
+ mount -t tmpfs tmpfs /cache
+ fi
+ dockerd --host=unix:///var/run/docker.sock --storage-driver=overlay --graph=/cache/docker & &>/docker.log
+ trap 'kill %%' EXIT
+
+ echo "Waiting for docker..."
+ for i in $(seq 1 60); do
+ if docker info &> /dev/null; then
+ break
+ fi
+ sleep 1s
+ done
+
+ if [[ "$i" == 60 ]]; then
+ echo "Failed to start docker:dind..."
+ cat /docker.log
+ exit 1
+ fi
+ echo ""
+ fi
+
+ docker rm -f "$CI_CONTAINER_NAME" &>/dev/null || true
+
+ echo "Building application..."
+
+ if [[ -f Dockerfile ]]; then
+ echo "Building Dockerfile-based application..."
+ # Build Dockerfile
+ docker build -t "$CI_REGISTRY_IMAGE:$CI_REGISTRY_TAG" .
+ else
+ # Build heroku-based application
+ echo "Building Heroku-based application using gliderlabs/herokuish docker image..."
+ docker run -i --name="$CI_CONTAINER_NAME" -v "$(pwd):/tmp/app:ro" -v "/cache/herokuish:/tmp/cache" gliderlabs/herokuish /bin/herokuish buildpack build
+ docker commit "$CI_CONTAINER_NAME" "$CI_REGISTRY_IMAGE:$CI_REGISTRY_TAG"
+ docker rm "$CI_CONTAINER_NAME" >/dev/null
+ echo ""
+
+ # Create a start command, start `web`
+ echo "Configuring $CI_REGISTRY_IMAGE:$CI_REGISTRY_TAG docker image..."
+ docker create --expose 5000 --env PORT=5000 --name="$CI_CONTAINER_NAME" "$CI_REGISTRY_IMAGE:$CI_REGISTRY_TAG" /bin/herokuish procfile start web
+ docker commit "$CI_CONTAINER_NAME" "$CI_REGISTRY_IMAGE:$CI_REGISTRY_TAG"
+ docker rm "$CI_CONTAINER_NAME" >/dev/null
+ echo ""
+ fi
+
+ if [[ -n "$CI_BUILD_TOKEN" ]]; then
+ echo "Logging to GitLab Container Registry with CI credentials..."
+ docker login -u gitlab-ci-token -p "$CI_BUILD_TOKEN" "$CI_REGISTRY"
+ echo ""
+ fi
+
+ echo "Pushing to GitLab Container Registry..."
+ docker push "$CI_REGISTRY_IMAGE:$CI_REGISTRY_TAG"
+ echo ""
+
+.auto_deploy: &auto_deploy
+ |
+ CI_ENVIRONMENT_HOSTNAME="${CI_ENVIRONMENT_URL}"
+ CI_ENVIRONMENT_HOSTNAME="${CI_ENVIRONMENT_HOSTNAME/http:\/\//}"
+ CI_ENVIRONMENT_HOSTNAME="${CI_ENVIRONMENT_HOSTNAME/https:\/\//}"
+
+ echo ciEnvironmentSlug=$CI_ENVIRONMENT_SLUG,ciPipelineID=\\"$CI_PIPELINE_ID\\",ciBuildID=\\"$CI_BUILD_ID\\",ciEnvironmentHostname=$CI_ENVIRONMENT_HOSTNAME,ciRegistryImage=$CI_REGISTRY_IMAGE,ciRegistryTag=$CI_REGISTRY_TAG
+
+ helm status $CI_PROJECT_NAME-$CI_PROJECT_ID-$CI_ENVIRONMENT_SLUG
+ if [[ $? -ne 0 ]]; then
+ echo "Creating initial release"
+ helm install --wait --name $CI_PROJECT_NAME-$CI_PROJECT_ID-$CI_ENVIRONMENT_SLUG --namespace $KUBE_NAMESPACE --set ciEnvironmentSlug=$CI_ENVIRONMENT_SLUG,ciPipelineID=\\"$CI_PIPELINE_ID\\",ciBuildID=\\"$CI_BUILD_ID\\",ciEnvironmentHostname=$CI_ENVIRONMENT_HOSTNAME,ciRegistryImage=$CI_REGISTRY_IMAGE,ciRegistryTag=$CI_REGISTRY_TAG gitlab/auto-deploy-app
+ else
+ echo "Upgrading release"
+ helm upgrade --wait $CI_PROJECT_NAME-$CI_PROJECT_ID-$CI_ENVIRONMENT_SLUG --namespace $KUBE_NAMESPACE --set ciEnvironmentSlug=$CI_ENVIRONMENT_SLUG,ciPipelineID=\\"$CI_PIPELINE_ID\\",ciBuildID=\\"$CI_BUILD_ID\\",ciEnvironmentHostname=$CI_ENVIRONMENT_HOSTNAME,ciRegistryImage=$CI_REGISTRY_IMAGE,ciRegistryTag=$CI_REGISTRY_TAG gitlab/auto-deploy-app
+ fi
+
+ echo "Application is accessible at: ${CI_ENVIRONMENT_URL}"
+ echo ""
+
+.auto_destroy: &auto_destroy
+ |
+ create_kubeconfig
+
+ echo "Removing app..."
+ helm delete $CI_PROJECT_NAME-$CI_PROJECT_ID-$CI_ENVIRONMENT_SLUG
+
+before_script:
+ - *common
+
+build:
+ stage: build
+ script:
+ - *auto_build
+ only:
+ - branches
+
+production:
+ stage: production
+ variables:
+ CI_ENVIRONMENT_URL: http://$CI_PROJECT_NAME.$KUBE_DOMAIN
+ script:
+ - *auto_deploy
+ environment:
+ name: production
+ url: http://$CI_PROJECT_NAME.$KUBE_DOMAIN
+ when: manual
+ only:
+ - master
+
+staging:
+ stage: staging
+ variables:
+ CI_ENVIRONMENT_URL: http://$CI_PROJECT_NAME-staging.$KUBE_DOMAIN
+ script:
+ - *auto_deploy
+ environment:
+ name: staging
+ url: http://$CI_PROJECT_NAME-staging.$KUBE_DOMAIN
+ only:
+ - master
+
+review:
+ stage: review
+ variables:
+ CI_ENVIRONMENT_URL: http://$CI_PROJECT_NAME-$CI_ENVIRONMENT_SLUG.$KUBE_DOMAIN
+ before_script:
+ - *common
+ - apk update && apk add curl bash openssl sudo
+ - curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get | bash
+ - ensure_deploy_variables
+ - create_kubeconfig
+ - helm init --upgrade
+ - helm repo add gitlab https://charts.gitlab.io
+ script:
+ - *auto_deploy
+ environment:
+ name: review/$CI_BUILD_REF_SLUG
+ url: http://$CI_PROJECT_NAME-$CI_ENVIRONMENT_SLUG.$KUBE_DOMAIN
+ on_stop: stop_review
+ only:
+ - branches
+ except:
+ - master
+
+stop_review:
+ stage: cleanup
+ variables:
+ GIT_STRATEGY: none
+ before_script:
+ - *common
+ - apk update && apk add curl bash openssl sudo
+ - curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get | bash
+ - ensure_deploy_variables
+ - create_kubeconfig
+ - helm init --client-only
+ script:
+ - *auto_destroy
+ environment:
+ name: review/$CI_BUILD_REF_SLUG
+ action: stop
+ when: manual
+ only:
+ - branches
+ except:
+ - master
--
GitLab
From 928844faec0036e381af5f5346d99e89c559111c Mon Sep 17 00:00:00 2001
From: Mark Pundsack <mpundsack@gitlab.com>
Date: Mon, 13 Mar 2017 16:15:14 -0500
Subject: [PATCH 2/5] Refactor CI/CD configuration
---
Auto-deploy-kubernetes.yml | 112 ++++++++++++++++++++++++++++++-------
1 file changed, 91 insertions(+), 21 deletions(-)
diff --git a/Auto-deploy-kubernetes.yml b/Auto-deploy-kubernetes.yml
index 077f2c9..7819fd7 100644
--- a/Auto-deploy-kubernetes.yml
+++ b/Auto-deploy-kubernetes.yml
@@ -1,20 +1,10 @@
# Explanation on the scripts:
# https://gitlab.com/gitlab-examples/kubernetes-deploy/blob/master/README.md
-image: registry.gitlab.com/gitlab-examples/kubernetes-deploy
-
-variables:
- # Application deployment domain
- KUBE_DOMAIN: domain.example.com
-
-stages:
- - build
- - test
- - review
- - staging
- - production
- - cleanup
+##############################
# Define some helper functions
+##############################
+
.common: &common
|
set -eo pipefail
@@ -77,6 +67,10 @@ stages:
fi
}
+##############################
+# Define auto deploy functions
+##############################
+
.auto_build: &auto_build
|
echo "Checking docker engine..."
@@ -169,17 +163,28 @@ stages:
echo "Removing app..."
helm delete $CI_PROJECT_NAME-$CI_PROJECT_ID-$CI_ENVIRONMENT_SLUG
-before_script:
- - *common
-
-build:
+##############################
+# Define basic job definitions
+##############################
+
+.auto_deploy_kubernetes: &auto_deploy_kubernetes
+ image: registry.gitlab.com/gitlab-examples/kubernetes-deploy
+ stages:
+ - build
+ - test
+ - review
+ - staging
+ - production
+ - cleanup
+
+.build_branches: &build_branches
stage: build
script:
- *auto_build
only:
- branches
-production:
+.manual_deploy_to_production: &manual_deploy_to_production
stage: production
variables:
CI_ENVIRONMENT_URL: http://$CI_PROJECT_NAME.$KUBE_DOMAIN
@@ -192,7 +197,31 @@ production:
only:
- master
-staging:
+.auto_deploy_to_production: &manual_deploy_to_production
+ stage: production
+ variables:
+ CI_ENVIRONMENT_URL: http://$CI_PROJECT_NAME.$KUBE_DOMAIN
+ script:
+ - *auto_deploy
+ environment:
+ name: production
+ url: http://$CI_PROJECT_NAME.$KUBE_DOMAIN
+ only:
+ - master
+
+.auto_deploy_tags_to_production: &manual_deploy_to_production
+ stage: production
+ variables:
+ CI_ENVIRONMENT_URL: http://$CI_PROJECT_NAME.$KUBE_DOMAIN
+ script:
+ - *auto_deploy
+ environment:
+ name: production
+ url: http://$CI_PROJECT_NAME.$KUBE_DOMAIN
+ only:
+ - tags
+
+.auto_deploy_to_staging: &auto_deploy_to_staging
stage: staging
variables:
CI_ENVIRONMENT_URL: http://$CI_PROJECT_NAME-staging.$KUBE_DOMAIN
@@ -204,7 +233,7 @@ staging:
only:
- master
-review:
+.start_review: &start_review
stage: review
variables:
CI_ENVIRONMENT_URL: http://$CI_PROJECT_NAME-$CI_ENVIRONMENT_SLUG.$KUBE_DOMAIN
@@ -227,7 +256,7 @@ review:
except:
- master
-stop_review:
+.stop_review: &stop_review
stage: cleanup
variables:
GIT_STRATEGY: none
@@ -248,3 +277,44 @@ stop_review:
- branches
except:
- master
+
+##############################
+# Define overall pipeline
+##############################
+
+.auto_deploy_kubernetes: &auto_deploy_kubernetes
+ image: registry.gitlab.com/gitlab-examples/kubernetes-deploy
+
+ stages:
+ - build
+ - test
+ - review
+ - staging
+ - production
+ - cleanup
+
+ before_script:
+ - *common
+
+ build:
+ <<: *build_branches
+
+ production:
+ <<: *manual_deploy_to_production
+
+ staging:
+ <<: *auto_deploy_to_staging
+
+ review:
+ <<: *start_review
+
+ stop_review:
+ <<: *stop_review
+
+# -----------------------------------------
+
+<<: *auto_deploy_kubernetes
+
+variables:
+ # Application deployment domain
+ KUBE_DOMAIN: domain.example.com
--
GitLab
From cf41b446959df9274c6aff7baf9b44244b6f5ccb Mon Sep 17 00:00:00 2001
From: Mark Pundsack <mpundsack@gitlab.com>
Date: Mon, 13 Mar 2017 16:29:17 -0500
Subject: [PATCH 3/5] Refactor
---
Auto-deploy-kubernetes.yml | 17 ++++-------------
1 file changed, 4 insertions(+), 13 deletions(-)
diff --git a/Auto-deploy-kubernetes.yml b/Auto-deploy-kubernetes.yml
index 7819fd7..dd6ae00 100644
--- a/Auto-deploy-kubernetes.yml
+++ b/Auto-deploy-kubernetes.yml
@@ -167,15 +167,10 @@
# Define basic job definitions
##############################
-.auto_deploy_kubernetes: &auto_deploy_kubernetes
+.kubernetes-deploy: &kubernetes-deploy
image: registry.gitlab.com/gitlab-examples/kubernetes-deploy
- stages:
- - build
- - test
- - review
- - staging
- - production
- - cleanup
+ before_script:
+ - *common
.build_branches: &build_branches
stage: build
@@ -283,19 +278,15 @@
##############################
.auto_deploy_kubernetes: &auto_deploy_kubernetes
- image: registry.gitlab.com/gitlab-examples/kubernetes-deploy
+ <<: *kubernetes-deploy
stages:
- build
- - test
- review
- staging
- production
- cleanup
- before_script:
- - *common
-
build:
<<: *build_branches
--
GitLab
From c7d95f72c448af563503d59a0ae187e57e3d8419 Mon Sep 17 00:00:00 2001
From: Mark Pundsack <mpundsack@gitlab.com>
Date: Mon, 13 Mar 2017 16:49:26 -0500
Subject: [PATCH 4/5] Small fixes
---
Auto-deploy-kubernetes.yml | 24 ++++++++++++++++--------
1 file changed, 16 insertions(+), 8 deletions(-)
diff --git a/Auto-deploy-kubernetes.yml b/Auto-deploy-kubernetes.yml
index dd6ae00..0ee597f 100644
--- a/Auto-deploy-kubernetes.yml
+++ b/Auto-deploy-kubernetes.yml
@@ -1,6 +1,3 @@
-# Explanation on the scripts:
-# https://gitlab.com/gitlab-examples/kubernetes-deploy/blob/master/README.md
-
##############################
# Define some helper functions
##############################
@@ -167,11 +164,16 @@
# Define basic job definitions
##############################
-.kubernetes-deploy: &kubernetes-deploy
+.auto_deploy_image: &auto_deploy_image
image: registry.gitlab.com/gitlab-examples/kubernetes-deploy
before_script:
- *common
+# .openshift-deploy: &openshift-deploy
+# image: registry.gitlab.com/gitlab-examples/openshift-deploy
+# before_script:
+# - *common
+
.build_branches: &build_branches
stage: build
script:
@@ -277,8 +279,8 @@
# Define overall pipeline
##############################
-.auto_deploy_kubernetes: &auto_deploy_kubernetes
- <<: *kubernetes-deploy
+.auto_deploy: &auto_deploy
+ <<: *auto_deploy_image
stages:
- build
@@ -290,6 +292,10 @@
build:
<<: *build_branches
+ # Choose between:
+ # *manual_deploy_to_production
+ # *auto_deploy_to_production
+ # *auto_deploy_tags_to_production
production:
<<: *manual_deploy_to_production
@@ -302,9 +308,11 @@
stop_review:
<<: *stop_review
-# -----------------------------------------
+##############################
+# Project's `.gitlab-ci.yml`
+##############################
-<<: *auto_deploy_kubernetes
+<<: *auto_deploy
variables:
# Application deployment domain
--
GitLab
From 74c3397fa251fbeb750b9c6e6f130127544a8761 Mon Sep 17 00:00:00 2001
From: "Z.J. van de Weg" <git@zjvandeweg.nl>
Date: Tue, 14 Mar 2017 08:05:58 +0100
Subject: [PATCH 5/5] Improve fail message for test
---
verify_templates.rb | 13 ++++++++++---
1 file changed, 10 insertions(+), 3 deletions(-)
diff --git a/verify_templates.rb b/verify_templates.rb
index 63e6527..ba0f0df 100644
--- a/verify_templates.rb
+++ b/verify_templates.rb
@@ -6,7 +6,13 @@ uri = URI.parse 'https://gitlab.com/api/v3/ci/lint'
Dir.glob("#{File.dirname(__FILE__)}/**/*.yml").each do |file|
response = Net::HTTP.post_form(uri, content: File.read(file))
- file = file.match(/((\w|\+|#)+)\.gitlab-ci/)[1]
+ match = file.match(/(?<file>(\w|\+|#)+)\.gitlab-ci.yml/)
+ template_name = match ? match[:file] : nil
+
+ unless template_name
+ puts "Invalid file format for #{file}"
+ exit(1)
+ end
if JSON.parse(response.body)['status'] == 'valid'
puts "\e[32mvalid\e[0m: #{file}" # Color 'valid' green
@@ -17,5 +23,6 @@ Dir.glob("#{File.dirname(__FILE__)}/**/*.yml").each do |file|
end
# Given we test all the templates, the coverage is 100%, always. To showcase
-# how this is done, we print it here.
-puts "Coverage: 100%"
\ No newline at end of file
+# how this is done, we print it here. The regex to parse this would be:
+# /Coverage:\s(d+)%/
+puts "Coverage: 100%"
--
GitLab