Fill out the first integration test

.gitignore

 /es-git-go
 /.GOPATH
 /bin
+/tmp
+/vendor/github.com/stretchr/testify
+/vendor/github.com/wadey/gocovmerge
+/vendor/golang.org/x/tools
+/vendor/manifest

Makefile

@@ -29,7 +29,7 @@ build: .GOPATH/.ok
 .PHONY: clean test list cover format
 clean:
-	$Q rm -rf bin .GOPATH
+	$Q rm -rf bin .GOPATH tmp
 test: .GOPATH/.ok
 	$Q go test $(if $V,-v) -i -race $(allpackages) # install -race libs to speed up next run
@@ -82,6 +82,8 @@ setup: clean .GOPATH/.ok
 	- ./bin/gvt fetch golang.org/x/tools/cmd/goimports
 	- ./bin/gvt fetch github.com/wadey/gocovmerge
 	- ./bin/gvt fetch github.com/stretchr/testify/assert
+	- mkdir tmp
+	- git clone --bare https://gitlab.com/gitlab-org/gitlab-test.git tmp/gitlab-test.git
 VERSION          := $(shell git describe --tags --always --dirty="-dev")
 DATE             := $(shell date -u '+%Y-%m-%d-%H%M UTC')

elastic/client.go

@@ -27,7 +27,7 @@ const (
 type Client struct {
 	IndexName string
 	ProjectID string
-	client    *elastic.Client
+	Client    *elastic.Client
 	bulk      *elastic.BulkProcessor
 }
@@ -95,7 +95,7 @@ func NewClient(config *Config) (*Client, error) {
 	return &Client{
 		IndexName: config.IndexName,
 		ProjectID: config.ProjectID,
-		client:    client,
+		Client:    client,
 		bulk:      bulk,
 	}, nil
 }
@@ -119,6 +119,24 @@ func (c *Client) Index(id string, thing interface{}) {
 	c.bulk.Add(req)
 }
+// We only really use this for tests
+func (c *Client) Get(id string) (*elastic.GetResult, error) {
+	return c.Client.Get().
+		Index(c.IndexName).
+		Type("repository").
+		Id(id).
+		Routing(c.ProjectID).
+		Do(context.TODO())
+}
+
+func (c *Client) GetCommit(id string) (*elastic.GetResult, error) {
+	return c.Get(c.ProjectID + "_" + id)
+}
+
+func (c *Client) GetBlob(path string) (*elastic.GetResult, error) {
+	return c.Get(c.ProjectID + "_" + path)
+}
+
 func (c *Client) Remove(id string) {
 	req := elastic.NewBulkDeleteRequest().
 		Index(c.IndexName).

elastic/index.go

@@ -647,7 +647,7 @@ var indexMapping = `
 // CreateIndex creates an index matching that created by gitlab-elasticsearch-git v1.1.1
 func (c *Client) CreateIndex() error {
-	createIndex, err := c.client.CreateIndex(c.IndexName).BodyString(indexMapping).Do(context.Background())
+	createIndex, err := c.Client.CreateIndex(c.IndexName).BodyString(indexMapping).Do(context.Background())
 	if err != nil {
 		return err
 	}
@@ -660,7 +660,7 @@ func (c *Client) CreateIndex() error {
 }
 func (c *Client) DeleteIndex() error {
-	deleteIndex, err := c.client.DeleteIndex(c.IndexName).Do(context.Background())
+	deleteIndex, err := c.Client.DeleteIndex(c.IndexName).Do(context.Background())
 	if err != nil {
 		return err
 	}

integration_test.go

 package main_test
 import (
+	"encoding/json"
 	"flag"
 	"fmt"
 	"os"
+	"os/exec"
 	"testing"
 	"time"
@@ -13,11 +15,13 @@ import (
 )
 var (
-	binary = flag.String("binary", "./bin/es-git-go", "Path to `es-git-go` binary for integration tests")
+	binary   = flag.String("binary", "./bin/es-git-go", "Path to `es-git-go` binary for integration tests")
+	testRepo = flag.String("test-repo", "./tmp/gitlab-test.git", "Path to `gitlab-test` repository for integration tests")
 )
 const (
 	projectID = "667"
+	headSHA   = "b83d6e391c22777fca1ed3012fce84f633d7fed0"
 )
 func checkDeps(t *testing.T) {
@@ -35,9 +39,14 @@ func checkDeps(t *testing.T) {
 		t.Log("No binary found at ", *binary)
 		t.Skip()
 	}
+
+	if _, err := os.Stat(*testRepo); err != nil {
+		t.Log("No test repo found at ", *testRepo)
+		t.Skip()
+	}
 }
-func buildIndex(t *testing.T) (string, func()) {
+func buildIndex(t *testing.T) (*elastic.Client, func()) {
 	railsEnv := fmt.Sprintf("test-%d", time.Now().Unix())
 	os.Setenv("RAILS_ENV", railsEnv)
@@ -46,16 +55,100 @@ func buildIndex(t *testing.T) (string, func()) {
 	assert.NoError(t, client.CreateIndex())
-	return railsEnv, func() {
+	return client, func() {
 		client.DeleteIndex()
 	}
 }
-func TestIndexingSelf(t *testing.T) {
+func run(from, to string) error {
+	cmd := exec.Command(*binary, projectID, *testRepo)
+	cmd.Env = os.Environ()
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+
+	if from != "" {
+		cmd.Env = append(cmd.Env, "ID_FROM="+from)
+	}
+
+	if to != "" {
+		cmd.Env = append(cmd.Env, "ID_TO="+to)
+	}
+
+	return cmd.Run()
+}
+
+func TestIndexingGitlabTest(t *testing.T) {
 	checkDeps(t)
-	indexName, td := buildIndex(t)
+	c, td := buildIndex(t)
 	defer td()
-	// We're going to index our own git repository, because why not?
-	_ = indexName
+	assert.NoError(t, run("", headSHA))
+
+	// Check the indexing of a commit
+	commit, err := c.GetCommit(headSHA)
+	assert.NoError(t, err)
+	assert.True(t, commit.Found)
+	assert.Equal(t, "repository", commit.Type)
+	assert.Equal(t, projectID+"_"+headSHA, commit.Id)
+	assert.Equal(t, projectID, commit.Routing)
+	assert.Equal(t, projectID, commit.Parent)
+
+	doc := make(map[string]map[string]interface{})
+	assert.NoError(t, json.Unmarshal(*commit.Source, &doc))
+
+	commitDoc, ok := doc["commit"]
+	assert.True(t, ok)
+	assert.Equal(
+		t,
+		map[string]interface{}{
+			"type": "commit",
+			"sha":  headSHA,
+			"author": map[string]interface{}{
+				"email": "job@gitlab.com",
+				"name":  "Job van der Voort",
+				"time":  "20160927T153746+0100",
+			},
+			"committer": map[string]interface{}{
+				"email": "job@gitlab.com",
+				"name":  "Job van der Voort",
+				"time":  "20160927T153746+0100",
+			},
+			"rid":     projectID,
+			"message": "Merge branch 'branch-merged' into 'master'\r\n\r\nadds bar folder and branch-test text file to check Repository merged_to_root_ref method\r\n\r\n\r\n\r\nSee merge request !12",
+		},
+		commitDoc,
+	)
+
+	// Check the indexing of a text blob
+	blob, err := c.GetBlob("README.md")
+	assert.NoError(t, err)
+	assert.True(t, blob.Found)
+	assert.Equal(t, "repository", blob.Type)
+	assert.Equal(t, projectID+"_README.md", blob.Id)
+	assert.Equal(t, projectID, blob.Routing)
+	assert.Equal(t, projectID, blob.Parent)
+
+	doc = make(map[string]map[string]interface{})
+	assert.NoError(t, json.Unmarshal(*blob.Source, &doc))
+
+	blobDoc, ok := doc["blob"]
+	assert.True(t, ok)
+	assert.Equal(
+		t,
+		map[string]interface{}{
+			"type":       "blob",
+			"language":   "Markdown",
+			"path":       "README.md",
+			"file_name":  "README.md",
+			"oid":        "faaf198af3a36dbf41961466703cc1d47c61d051",
+			"rid":        projectID,
+			"commit_sha": headSHA,
+			"content":    "testme\n======\n\nSample repo for testing gitlab features\n",
+		},
+		blobDoc,
+	)
+
+	// Check that a binary blob isn't indexed
+	_, err = c.GetBlob("Gemfile.zip")
+	assert.Error(t, err)
 }