feat: add templates with multiple jobs

1d0390d3 · Andrew Newdigate · d8fd2d17 · 1d0390d3 · 1d0390d3 · 1d0390d3
Unverified Commit 1d0390d3 authored 2 years ago by Andrew Newdigate
--- a/.gitlab-ci-asdf-versions.yml
+++ b/.gitlab-ci-asdf-versions.yml
@@ -2,4 +2,4 @@
 variables:
    GL_ASDF_GOLANG_VERSION: 1.17.6
    GL_ASDF_SHELLCHECK_VERSION: 0.7.0
-    GL_ASDF_SHFMT_VERSION: 3.0.2
+    GL_ASDF_SHFMT_VERSION: 3.4.3
--- a/.gitlab-ci-default-asdf-versions.yml
+++ b/.gitlab-ci-default-asdf-versions.yml
+# DO NOT MANUALLY EDIT; Run ./scripts/update-asdf-version-variables to update this
+variables:
+    GL_COMMON_CI_TASKS_DEFAULT_ASDF_GOLANG_VERSION: 1.17.6
+    GL_COMMON_CI_TASKS_DEFAULT_ASDF_SHELLCHECK_VERSION: 0.7.0
+    GL_COMMON_CI_TASKS_DEFAULT_ASDF_SHFMT_VERSION: 3.4.3
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -4,9 +4,4 @@ stages:
  
 include:
  - local: .gitlab-ci-asdf-versions.yml
-
-  # disabled as there are no golang files to run
-  # - local: golangci-lint.yml
-  - local: editorconfig-check.yml
-  - local: semantic-release.yml
-  - local: asdf-tool-versions.yml
+  - local: templates/standard.yml
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
+---
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.2.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-case-conflict
+      - id: check-symlinks
+      - id: check-json
+      - id: detect-private-key
+      - id: mixed-line-ending
+        args: ['--fix=lf']
+      - id: no-commit-to-branch
+        args: [--branch, main]
+      - id: pretty-format-json
+        args: [--autofix, --no-sort-keys]
+  - repo: local
+    hooks:
+    -   id: shellcheck
+        name: shellcheck
+        entry: shellcheck -P . -P ./test/ -P ./test/lib/ -P ./test/integration/ -x
+        language: system
+        files: '^(.*\.sh|bin/[^\.]*)$'
+  - repo: https://github.com/scop/pre-commit-shfmt
+    rev: v3.4.3-1 # Note: keep this in sync with .tool-versions
+    hooks:
+      - id: shfmt
+        args: [-w, -s, -i, "2"]
--- a/.releaserc.json
+++ b/.releaserc.json
 {
-  "branches": ["main"],
+  "branches": [
+    "main"
+  ],
  "plugins": [
    [
      "@semantic-release/commit-analyzer",

--- a/.tool-versions
+++ b/.tool-versions
@@ -4,4 +4,4 @@
  
 golang 1.17.6
 shellcheck 0.7.0
-shfmt 3.0.2
+shfmt 3.4.3
--- a/README.md
+++ b/README.md
@@ -6,8 +6,43 @@ This project includes reuseable GitLab-CI jobs for the GitLab Dedicated Project
  
 It is possible to use these templates without a `ref` tag when including them in your project, but this is not recommended as it may break your build process and make it non-repeatable. For example, if you revert a change, the templates will still reference the latest template which may no longer be compatible. For this reason, it's recommended you always use refs for your templates.
  
+## Templates
+
+It is preferable to use templates over individual CI jobs as it reduces clutter, increases standardization and improves consistency across projects. Additionally, new functionality can be added to projects with only an upgrade to the `ref`.
+
+### [`templates/standard.yml`](./templates/standard.yml)
+
+This template should be included in all Infrastructure projects. It includes standard checks, gitlab-scanners, validations and release processes common to all projects using this template library.
+
+Includes the following tasks:
+
+1. [`gitlab-scanners.yml`](#gitlab-scanners): adds various GitLab SAST, Dependency Scanner, Secret Detection, Licence Scanning, Container Scanning and IAC Scanner tools.
+1. [`editorconfig-check.yml`](#editorconfig-check): ensures that the repository adheres to the [EditorConfig](https://editorconfig.org/) rules for the project.
+1. [`semantic-release.yml`](#semantic-release): runs [`semantic-release`](https://semantic-release.gitbook.io/semantic-release/) on your repository.
+1. [`asdf-tool-versions.yml`](#asdf-tool-versions): ensures that `.tool-versions` file is synced with .gitlab-ci-asdf-versions.yml, that all the plugins are declared in `./scripts/install-asdf-plugins.sh` and that ASDF is generally working.
+1. [`shfmt.yml`](#shfmt): validates that shell-scripts use a canonical formatting.
+
+```yaml
+# Requires stages validate and release
+stages:
+  - validate
+  - release
+
+include:
+  # - local: .gitlab-ci-asdf-versions.yml
+  # This template should be included in all Infrastructure projects.
+  # It includes standard checks, gitlab-scanners, validations and release processes
+  # common to all projects using this template library.
+  # see https://gitlab.com/gitlab-com/gl-infra/common-ci-tasks/-/blob/main/README.md#templatesstandardyml
+  - project: 'gitlab-com/gl-infra/common-ci-tasks'
+    ref: v1.15.0 # renovate:managed
+    file: templates/standard.yml
+```
+
 ## CI Jobs
  
+Alternatively to using the templates above, you can include the following individual jobs.
+
 ### `gitlab-scanners`
  
 This adds various GitLab SAST, Dependency Scanner, Secret Detection, Licence Scanning, Container Scanning and IAC Scanner tools.
@@ -20,7 +55,7 @@ stages:
  
 include:
  - project: 'gitlab-com/gl-infra/common-ci-tasks'
-    ref: v1.12.0 # renovate:managed
+    ref: v1.15.0 # renovate:managed
    file: 'gitlab-scanners.yml'
 ```
  
@@ -34,15 +69,16 @@ stages:
  
 include:
  - project: 'gitlab-com/gl-infra/common-ci-tasks'
-    ref: v1.12.0 # renovate:managed
+    ref: v1.15.0 # renovate:managed
    file: 'golangci-lint.yml'
 ```
  
-### `editorconfig_check`
+### [`editorconfig-check`](./editorconfig-check.yml)
  
 This job ensures that the repository adheres to the [EditorConfig](https://editorconfig.org/) rules for the project.
  
 * Default Stage: `validate`
+* Expects config file: `.editorconfig`
  
 ```yaml
 stages:
@@ -50,11 +86,11 @@ stages:
  
 include:
  - project: 'gitlab-com/gl-infra/common-ci-tasks'
-    ref: v1.12.0 # renovate:managed
+    ref: v1.15.0 # renovate:managed
    file: 'editorconfig-check.yml'
 ```
  
-### `semantic_release`
+### [`semantic-release`](./semantic-release.yml)
  
 This job will run [`semantic-release`](https://semantic-release.gitbook.io/semantic-release/) on your repository. This allows the repository to be automatically tagged based on [conventional commit](https://www.conventionalcommits.org/en/v1.0.0/) messages.
  
@@ -91,7 +127,7 @@ stages:
  
 include:
  - project: 'gitlab-com/gl-infra/common-ci-tasks'
-    ref: v1.12.0 # renovate:managed
+    ref: v1.15.0 # renovate:managed
    file: 'semantic-release.yml'
 ```
  
@@ -173,7 +209,7 @@ variables:
 include:
  # build binary release artifacts with goreleaser
  - project: 'gitlab-com/gl-infra/common-ci-tasks'
-    ref: v1.12.0 # renovate:managed
+    ref: v1.15.0 # renovate:managed
    file: goreleaser.yml
 ```
  
@@ -202,7 +238,7 @@ include:
  # and that ASDF is generally working
  # see https://gitlab.com/gitlab-com/gl-infra/common-ci-tasks/-/blob/main/README.md#asdf-tool-versions
  - project: 'gitlab-com/gl-infra/common-ci-tasks'
-    ref: v1.12.0 # renovate:managed
+    ref: v1.15.0 # renovate:managed
    file: asdf-tool-versions.yml
 ```
  
@@ -236,7 +272,7 @@ include:
  
  # Runs tflint on all terraform module directories
  - project: 'gitlab-com/gl-infra/common-ci-tasks'
-    ref: v1.12.0 # renovate:managed
+    ref: v1.15.0 # renovate:managed
    file: tflint.yml
 ```
  
@@ -260,7 +296,7 @@ include:
  
  # Ensures that all terraform files are correctly formatted
  - project: 'gitlab-com/gl-infra/common-ci-tasks'
-    ref: v1.12.0 # renovate:managed
+    ref: v1.15.0 # renovate:managed
    file: terraform-format.yml
 ```
  
@@ -287,7 +323,7 @@ include:
  
  # Ensures that all terraform files are syntactically valid
  - project: 'gitlab-com/gl-infra/common-ci-tasks'
-    ref: v1.12.0 # renovate:managed
+    ref: v1.15.0 # renovate:managed
    file: terraform-validate.yml
 ```
  
@@ -322,7 +358,7 @@ include:
  # Upgrades dependencies on a schedule
  # see https://gitlab.com/gitlab-com/gl-infra/common-ci-tasks/-/blob/main/README.md#renovate-bot
  - project: 'gitlab-com/gl-infra/common-ci-tasks'
-    ref: v1.12.0 # renovate:managed
+    ref: v1.15.0 # renovate:managed
    file: renovate-bot.yml
 ```
  
@@ -347,7 +383,7 @@ include:
  # Runs Go unit tests
  # see https://gitlab.com/gitlab-com/gl-infra/common-ci-tasks/-/blob/main/README.md#go-unittests
  - project: 'gitlab-com/gl-infra/common-ci-tasks'
-    ref: v1.12.0 # renovate:managed
+    ref: v1.15.0 # renovate:managed
    file: go-unittests.yml
 ```
  
@@ -372,7 +408,7 @@ include:
  # Perform `go mod tidy` and ensure that go.mod and go.sum are tidy.
  # see https://gitlab.com/gitlab-com/gl-infra/common-ci-tasks/-/blob/main/README.md#go-mod-tidy
  - project: 'gitlab-com/gl-infra/common-ci-tasks'
-    ref: v1.12.0 # renovate:managed
+    ref: v1.15.0 # renovate:managed
    file: go-mod-tidy.yml
 ```
  

--- a/golangci-lint.yml
+++ b/golangci-lint.yml
@@ -28,4 +28,3 @@ golangci_lint:
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-
--- a/goreleaser.yml
+++ b/goreleaser.yml
@@ -9,8 +9,9 @@ goreleaser_validate:
    entrypoint: [""]
  script: goreleaser check
  rules:
-    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+      exists:
+      - .goreleaser.yml
  
 goreleaser:
  stage: release
@@ -26,6 +27,8 @@ goreleaser:
  rules:
    # Only run this release job for tags, not every commit
    - if: '$CI_COMMIT_TAG'
+      exists:
+      - .goreleaser.yml
  
  script: |
    docker run --rm --privileged \

--- a/renovate-bot.yml
+++ b/renovate-bot.yml
@@ -17,9 +17,10 @@ renovate_bot:
      --cache-dir "${CI_PROJECT_DIR}/renovate-cache"
      ${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAME}
  rules:
-    - if: '$RENOVATE_SCHEDULED'
    # Run after changes to the main branch in case dependencies have changed
-    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+    - if: '$RENOVATE_SCHEDULED || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+      exists:
+      - renovate.json
  cache:
    key: renovate-cache
    paths:

--- a/scripts/install-asdf-plugins.sh
+++ b/scripts/install-asdf-plugins.sh
@@ -24,7 +24,7 @@ install_plugin() {
    return 1
  }
  
-  asdf shell "${plugin}" "$(asdf current "${plugin}"|awk '{print $2}')"
+  asdf shell "${plugin}" "$(asdf current "${plugin}" | awk '{print $2}')"
 }
  
 check_global_golang_install() {
@@ -38,7 +38,7 @@ check_global_golang_install() {
 # Install golang first as some of the other plugins require it
 install_plugin golang
  
-if [[ -z "${CI:-}" ]]; then
+if [[ -z ${CI:-} ]]; then
  # The go-jsonnet plugin requires a global golang version to be configured
  # and will otherwise fail to install
  #

--- a/scripts/update-asdf-version-variables.sh
+++ b/scripts/update-asdf-version-variables.sh
@@ -28,4 +28,20 @@ generate() {
    '
 }
  
+generate_defaults() {
+  asdf_current |
+    grep "${ROOT_DIR}/" |
+    awk '
+    BEGIN {
+      print "# DO NOT MANUALLY EDIT; Run ./scripts/update-asdf-version-variables to update this";
+      print "variables:"
+    }
+    {
+      gsub("-", "_", $1);
+      print "    GL_COMMON_CI_TASKS_DEFAULT_ASDF_" toupper($1) "_VERSION: " $2
+    }
+    '
+}
+
 generate >"${ROOT_DIR}/.gitlab-ci-asdf-versions.yml"
+generate_defaults >"${ROOT_DIR}/.gitlab-ci-default-asdf-versions.yml"
--- a/semantic-release.yml
+++ b/semantic-release.yml
@@ -40,3 +40,5 @@ semantic_release:
      when: never
  
    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+      exists:
+        - .releaserc.json
--- a/shfmt.yml
+++ b/shfmt.yml
@@ -2,10 +2,10 @@ shfmt:
  stage: validate
  needs: []
  image:
-    name: golang:1.17 # Client project doesn't need Golang, so don't use ASDF vars here.
+    name: golang:${GL_COMMON_CI_TASKS_DEFAULT_ASDF_GOLANG_VERSION} # Client project doesn't need Golang, so use default ASDF vars here.
    entrypoint: [""]
  script:
-    - go install mvdan.cc/sh/v3/cmd/shfmt@v${GL_ASDF_SHFMT_VERSION}
+    - go install mvdan.cc/sh/v3/cmd/shfmt@v${GL_ASDF_SHFMT_VERSION:-$GL_COMMON_CI_TASKS_DEFAULT_ASDF_SHFMT_VERSION}
    # Find all *.sh files, and any executable script files
    - (
        find . -type f \( -perm -u=x -not -name "*.sh" \) -not -path "*/.*/*" -print0 | xargs -0 grep -rIzlE '^#![[:blank:]]*(/usr/bin/env sh|/usr/bin/env bash|/bin/sh|/bin/bash)';

--- a/templates/standard.yml
+++ b/templates/standard.yml
+include:
+  - local: .gitlab-ci-default-asdf-versions.yml
+  - local: gitlab-scanners.yml
+  - local: editorconfig-check.yml
+  - local: semantic-release.yml
+  - local: asdf-tool-versions.yml
+  - local: shfmt.yml