Merge branch '101-git-submodule-strategy' into 'master'

Add support for GIT_SUBMODULE_STRATEGY Closes #101 See merge request !443

Merge branch '101-git-submodule-strategy' into 'master'
c8a5244b · Tomasz Maczukin · 7f332f18 · 272d1f3f · c8a5244b · c8a5244b
Commit c8a5244b authored 8 years ago by Tomasz Maczukin
--- a/common/build.go
+++ b/common/build.go
@@ -22,6 +22,15 @@ const (
 	GitNone
 )
  
+type SubmoduleStrategy int
+
+const (
+	SubmoduleInvalid SubmoduleStrategy = iota
+	SubmoduleNone
+	SubmoduleNormal
+	SubmoduleRecursive
+)
+
 type BuildRuntimeState string
  
 const (
@@ -402,6 +411,24 @@ func (b *Build) GetGitStrategy() GitStrategy {
 	}
 }
  
+func (b *Build) GetSubmoduleStrategy() SubmoduleStrategy {
+	switch b.GetAllVariables().Get("GIT_SUBMODULE_STRATEGY") {
+	case "normal":
+		return SubmoduleNormal
+
+	case "recursive":
+		return SubmoduleRecursive
+
+	case "none", "":
+		// Default (legacy) behavior is to not update/init submodules
+		return SubmoduleNone
+
+	default:
+		// Will cause an error in AbstractShell) writeSubmoduleUpdateCmds
+		return SubmoduleInvalid
+	}
+}
+
 func (b *Build) IsDebugTraceEnabled() bool {
 	trace, err := strconv.ParseBool(b.GetAllVariables().Get("CI_DEBUG_TRACE"))
 	if err != nil {

--- a/dockerfiles/build/Dockerfile.arm
+++ b/dockerfiles/build/Dockerfile.arm
 FROM scaleway/alpine
  
-RUN apk add --update bash ca-certificates git
+RUN apk add --update bash ca-certificates git perl
 COPY ./ /usr/bin
--- a/dockerfiles/build/Dockerfile.x86_64
+++ b/dockerfiles/build/Dockerfile.x86_64
 FROM alpine
  
-RUN apk add --update bash ca-certificates git
+RUN apk add --update bash ca-certificates git perl
  
 COPY ./ /usr/bin
--- a/executors/shell/executor_shell_test.go
+++ b/executors/shell/executor_shell_test.go
@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"testing"
 	"time"
  
@@ -241,6 +242,96 @@ func TestBuildWithGitStrategyClone(t *testing.T) {
 	})
 }
  
+func TestBuildWithGitSubmoduleStrategyNone(t *testing.T) {
+	for _, strategy := range []string{"none", ""} {
+		t.Run("strategy "+strategy, func(t *testing.T) {
+			onEachShell(t, func(t *testing.T, shell string) {
+				successfulBuild, err := common.GetSuccessfulBuild()
+				assert.NoError(t, err)
+				build, cleanup := newBuild(t, successfulBuild, shell)
+				defer cleanup()
+
+				build.Variables = append(build.Variables, common.BuildVariable{Key: "GIT_SUBMODULE_STRATEGY", Value: "none"})
+
+				out, err := runBuildReturningOutput(t, build)
+				assert.NoError(t, err)
+				assert.Contains(t, out, "Skipping Git submodules setup")
+				assert.NotContains(t, out, "Updating/initializing submodules...")
+				assert.NotContains(t, out, "Updating/initializing submodules recursively...")
+
+				_, err = os.Stat(filepath.Join(build.BuildDir, "gitlab-grack", ".git"))
+				assert.Error(t, err, "Submodule not should have been initialized")
+
+				_, err = os.Stat(filepath.Join(build.BuildDir, "gitlab-grack", "tests", "example", ".git"))
+				assert.Error(t, err, "The submodule's submodule should not have been initialized")
+			})
+		})
+	}
+}
+
+func TestBuildWithGitSubmoduleStrategyNormal(t *testing.T) {
+	onEachShell(t, func(t *testing.T, shell string) {
+		successfulBuild, err := common.GetSuccessfulBuild()
+		assert.NoError(t, err)
+		build, cleanup := newBuild(t, successfulBuild, shell)
+		defer cleanup()
+
+		build.Variables = append(build.Variables, common.BuildVariable{Key: "GIT_SUBMODULE_STRATEGY", Value: "normal"})
+
+		out, err := runBuildReturningOutput(t, build)
+		assert.NoError(t, err)
+		assert.NotContains(t, out, "Skipping Git submodules setup")
+		assert.Contains(t, out, "Updating/initializing submodules...")
+		assert.NotContains(t, out, "Updating/initializing submodules recursively...")
+
+		_, err = os.Stat(filepath.Join(build.BuildDir, "gitlab-grack", ".git"))
+		assert.NoError(t, err, "Submodule should have been initialized")
+
+		_, err = os.Stat(filepath.Join(build.BuildDir, "gitlab-grack", "tests", "example", ".git"))
+		assert.Error(t, err, "The submodule's submodule should not have been initialized")
+	})
+}
+
+func TestBuildWithGitSubmoduleStrategyRecursive(t *testing.T) {
+	onEachShell(t, func(t *testing.T, shell string) {
+		successfulBuild, err := common.GetSuccessfulBuild()
+		assert.NoError(t, err)
+		build, cleanup := newBuild(t, successfulBuild, shell)
+		defer cleanup()
+
+		build.Variables = append(build.Variables, common.BuildVariable{Key: "GIT_SUBMODULE_STRATEGY", Value: "recursive"})
+
+		out, err := runBuildReturningOutput(t, build)
+		assert.NoError(t, err)
+		assert.NotContains(t, out, "Skipping Git submodules setup")
+		assert.NotContains(t, out, "Updating/initializing submodules...")
+		assert.Contains(t, out, "Updating/initializing submodules recursively...")
+
+		_, err = os.Stat(filepath.Join(build.BuildDir, "gitlab-grack", ".git"))
+		assert.NoError(t, err, "Submodule should have been initialized")
+
+		_, err = os.Stat(filepath.Join(build.BuildDir, "gitlab-grack", "tests", "example", ".git"))
+		assert.NoError(t, err, "The submodule's submodule should have been initialized")
+	})
+}
+
+func TestBuildWithGitSubmoduleStrategyInvalid(t *testing.T) {
+	onEachShell(t, func(t *testing.T, shell string) {
+		successfulBuild, err := common.GetSuccessfulBuild()
+		assert.NoError(t, err)
+		build, cleanup := newBuild(t, successfulBuild, shell)
+		defer cleanup()
+
+		build.Variables = append(build.Variables, common.BuildVariable{Key: "GIT_SUBMODULE_STRATEGY", Value: "invalid"})
+
+		out, err := runBuildReturningOutput(t, build)
+		assert.EqualError(t, err, "unknown GIT_SUBMODULE_STRATEGY")
+		assert.NotContains(t, out, "Skipping Git submodules setup")
+		assert.NotContains(t, out, "Updating/initializing submodules...")
+		assert.NotContains(t, out, "Updating/initializing submodules recursively...")
+	})
+}
+
 func TestBuildWithDebugTrace(t *testing.T) {
 	onEachShell(t, func(t *testing.T, shell string) {
 		successfulBuild, err := common.GetSuccessfulBuild()

--- a/shells/abstract.go
+++ b/shells/abstract.go
@@ -103,6 +103,28 @@ func (b *AbstractShell) writeCheckoutCmd(w ShellWriter, build *common.Build) {
 	w.Command("git", "checkout", "-f", "-q", build.Sha)
 }
  
+func (b *AbstractShell) writeSubmoduleUpdateCmd(w ShellWriter, build *common.Build, recursive bool) {
+	if recursive {
+		w.Notice("Updating/initializing submodules recursively...")
+	} else {
+		w.Notice("Updating/initializing submodules...")
+	}
+
+	// Sync .git/config to .gitmodules in case URL changes (e.g. new build token)
+	args := []string{"submodule", "sync"}
+	if recursive {
+		args = append(args, "--recursive")
+	}
+	w.Command("git", args...)
+
+	// Update / initialize submodules
+	args = []string{"submodule", "update", "--init"}
+	if recursive {
+		args = append(args, "--recursive")
+	}
+	w.Command("git", args...)
+}
+
 func (b *AbstractShell) cacheFile(build *common.Build, userKey string) (key, file string) {
 	if build.CacheDir == "" {
 		return
@@ -233,21 +255,12 @@ func (b *AbstractShell) writePrepareScript(w ShellWriter, info common.ShellScrip
 	return nil
 }
  
-func (b *AbstractShell) writeGetSourcesScript(w ShellWriter, info common.ShellScriptInfo) (err error) {
-	b.writeExports(w, info)
-
+func (b *AbstractShell) writeCloneFetchCmds(w ShellWriter, info common.ShellScriptInfo) (err error) {
 	build := info.Build
 	projectDir := build.FullProjectDir()
 	gitDir := path.Join(build.FullProjectDir(), ".git")
-	strategy := info.Build.GetGitStrategy()
-
-	b.writeTLSCAInfo(w, info.Build, "GIT_SSL_CAINFO")
-
-	if info.PreCloneScript != "" && strategy != common.GitNone {
-		b.writeCommands(w, info.PreCloneScript)
-	}
  
-	switch strategy {
+	switch info.Build.GetGitStrategy() {
 	case common.GitFetch:
 		b.writeFetchCmd(w, build, projectDir, gitDir)
 		b.writeCheckoutCmd(w, build)
@@ -263,6 +276,46 @@ func (b *AbstractShell) writeGetSourcesScript(w ShellWriter, info common.ShellSc
 	default:
 		return errors.New("unknown GIT_STRATEGY")
 	}
+
+	return nil
+}
+
+func (b *AbstractShell) writeSubmoduleUpdateCmds(w ShellWriter, info common.ShellScriptInfo) (err error) {
+	build := info.Build
+
+	switch build.GetSubmoduleStrategy() {
+	case common.SubmoduleNormal:
+		b.writeSubmoduleUpdateCmd(w, build, false)
+
+	case common.SubmoduleRecursive:
+		b.writeSubmoduleUpdateCmd(w, build, true)
+
+	case common.SubmoduleNone:
+		w.Notice("Skipping Git submodules setup")
+
+	default:
+		return errors.New("unknown GIT_SUBMODULE_STRATEGY")
+	}
+
+	return nil
+}
+
+func (b *AbstractShell) writeGetSourcesScript(w ShellWriter, info common.ShellScriptInfo) (err error) {
+	b.writeExports(w, info)
+	b.writeTLSCAInfo(w, info.Build, "GIT_SSL_CAINFO")
+
+	if info.PreCloneScript != "" && info.Build.GetGitStrategy() != common.GitNone {
+		b.writeCommands(w, info.PreCloneScript)
+	}
+
+	if err := b.writeCloneFetchCmds(w, info); err != nil {
+		return err
+	}
+
+	if err = b.writeSubmoduleUpdateCmds(w, info); err != nil {
+		return err
+	}
+
 	return nil
 }