Merge branch 'common-excludes' into 'main'

feat: Add COMMON_TASK_VALIDATIONS_EXCLUDES_REGEXP option

See merge request gitlab-com/gl-infra/common-ci-tasks!29

README.md

@@ -217,6 +217,7 @@ Setup process:
 1. Ensure that the `GL_ASDF_TFLINT_VERSION` version for tflint is configured.
 1. Directories can be excluded from tflint using the `TFLINT_EXCLUDE_REGEX` variable. See the example below.
 1. The task will generate a [junit test output file](https://docs.gitlab.com/ee/ci/unit_test_reports.html) for any failed linter checks.
+1. Supports the `COMMON_TASK_VALIDATIONS_EXCLUDES_REGEXP` variable, for excluding paths from validation.
 ```yaml
 stages:
@@ -226,8 +227,8 @@ stages:
 variables:
   GL_ASDF_TFLINT_VERSION: ...
   GL_ASDF_TERRAFORM_VERSION: ...
-  # To exclude any directories, set this variable
-  TFLINT_EXCLUDE_REGEX: './exclude1|./exclude2'
+  # Exclude vendor and files directories from validation
+  COMMON_TASK_VALIDATIONS_EXCLUDES_REGEXP: '^\.(/vendor/|/files/)'
 include:
   # Not required, but recommended
@@ -268,6 +269,7 @@ include:
 Runs [`terraform validate`](https://www.terraform.io/cli/commands/validate) to ensure that all Terraform files are correctly formatted.
 1. Ensure that the `GL_ASDF_TERRAFORM_VERSION` version for terraform is configured.
+1. Supports the `COMMON_TASK_VALIDATIONS_EXCLUDES_REGEXP` variable, for excluding paths from validation.
 ```yaml
 stages:
@@ -276,6 +278,8 @@ stages:
 # Not needed if .gitlab-ci-asdf-versions.yml is included...
 variables:
   GL_ASDF_TERRAFORM_VERSION: ...
+  # Exclude vendor and files directories from validation
+  COMMON_TASK_VALIDATIONS_EXCLUDES_REGEXP: '^\.(/vendor/|/files/)'
 include:
   # Not required, but recommended
@@ -378,6 +382,7 @@ include:
 Validates that shell-scripts use a canonical formatting.
 1. Ensure that the `GL_ASDF_SHFMT_VERSION` version for Go is configured.
+1. Supports the `COMMON_TASK_VALIDATIONS_EXCLUDES_REGEXP` variable, for excluding paths from validation.
 ```yaml
 stages:
@@ -386,6 +391,8 @@ stages:
 # Not needed if .gitlab-ci-asdf-versions.yml is included...
 variables:
   GL_ASDF_SHFMT_VERSION: ...
+  # Exclude vendor and files directories from validation
+  COMMON_TASK_VALIDATIONS_EXCLUDES_REGEXP: '^\.(/vendor/|/files/)'
 include:
   # Not required, but recommended

shfmt.yml

@@ -10,7 +10,7 @@ shfmt:
     - (
         find . -type f \( -perm -u=x -not -name "*.sh" \) -not -path "*/.*/*" -print0 | xargs -0 grep -rIzlE '^#![[:blank:]]*(/usr/bin/env sh|/usr/bin/env bash|/bin/sh|/bin/bash)';
         find . -type f -name "*.sh" -not -path "*/.*/*"
-      ) | xargs shfmt -w -l -s  -i 2 -d
+      ) | grep -vE "${COMMON_TASK_VALIDATIONS_EXCLUDES_REGEXP:-__ignored__}" | xargs shfmt -w -l -s  -i 2 -d
   rules:
     - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

terraform-validate.yml

-tflint:
+terraform_validate:
   stage: validate
   needs: []
   image:
@@ -6,7 +6,7 @@ tflint:
     entrypoint: [""]
   script:
     # Loop through all the directories containing *.tf files and run terraform validate in them
-    - find . -type f -name '*.tf' -exec dirname {} \;|sort -u|grep -v "${TF_VALIDATE_EXCLUDE_REGEX:-__none__}" | while read -r dir; do
+    - find . -type f -name '*.tf' | grep -vE "${COMMON_TASK_VALIDATIONS_EXCLUDES_REGEXP:-__ignored__}" | sed -e 's#/[^/]*$##' | sort -u | grep -v "${TFLINT_EXCLUDE_REGEX:-__ignored__}" | while read -r dir; do
         echo "${dir} -------------------------------------------------------";
         cd "${CI_PROJECT_DIR}/${dir}" || exit 1;
         terraform init -backend=false -reconfigure;

tflint.yml

@@ -13,7 +13,7 @@ tflint:
     # Find all TF files and lint the directories of those files
     - tflint --init -c .tflint.hcl
     # Loop through all the directories containing *.tf files and run tflint in them
-    - find . -type f -name '*.tf' -exec dirname {} \;|sort -u|grep -v "${TFLINT_EXCLUDE_REGEX:-__none__}" | while read -r dir; do
+    - find . -type f -name '*.tf' | grep -vE "${COMMON_TASK_VALIDATIONS_EXCLUDES_REGEXP:-__ignored__}" | sed -e 's#/[^/]*$##' | sort -u | grep -v "${TFLINT_EXCLUDE_REGEX:-__ignored__}" | while read -r dir; do
         junit_file="$(echo "$dir"|sed -r 's/[^a-zA-Z0-9]+/-/g' | sed -r s/^-+\|-+$//g).xml";
         echo "${dir} -------------------------------------------------------";
         cd "${CI_PROJECT_DIR}/${dir}" || exit 1;