Skip to content
Snippets Groups Projects
Commit eecba7d6 authored by Nick Thomas's avatar Nick Thomas
Browse files

Numerous changes to get the first integration test ready

parent 64244977
No related branches found
No related tags found
1 merge request!1Initial implementation of an elasticsearch indexer in Go
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
+ 5
− 0
View file @ eecba7d6
.test: &test
services:
- elasticsearch:5.1
variables:
V: "1"
ELASTIC_CONNECTION_INFO: '{"url":["http://elasticsearch:9200"]}'
stage: test
script:
- make setup
Makefile
+ 2
− 1
View file @ eecba7d6
@@ -36,7 +36,7 @@ test: .GOPATH/.ok
$Q go test $(if $V,-v) -i -race $(allpackages) # install -race libs to speed up next run
ifndef CI
$Q go vet $(allpackages)
$Q GODEBUG=cgocheck=2 go test -race $(allpackages)
$Q GODEBUG=cgocheck=2 go test $(if $V,-v) -race $(allpackages)
else
$Q ( go vet $(allpackages); echo $$? ) | \
tee .GOPATH/test/vet.txt | sed '$$ d'; exit $$(tail -1 .GOPATH/test/vet.txt)
@@ -82,6 +82,7 @@ setup: clean .GOPATH/.ok
go get -u github.com/FiloSottile/gvt
- ./bin/gvt fetch golang.org/x/tools/cmd/goimports
- ./bin/gvt fetch github.com/wadey/gocovmerge
- ./bin/gvt fetch github.com/stretchr/testify/assert
 
VERSION := $(shell git describe --tags --always --dirty="-dev")
DATE := $(shell date -u '+%Y-%m-%d-%H%M UTC')
elastic/client.go
+ 46
− 4
View file @ eecba7d6
@@ -2,6 +2,8 @@ package elastic
 
import (
"context"
"fmt"
"os"
"strings"
 
"github.com/aws/aws-sdk-go/aws/credentials"
@@ -14,8 +16,9 @@ var (
// TODO: make this configurable / detectable.
// Limiting to 10MiB lets us work on small AWS clusters, but unnecessarily
// increases round trips in larger or non-AWS clusters
MaxBulkSize = 10 * 1024 * 1024
BulkWorkers = 2
MaxBulkSize = 10 * 1024 * 1024
BulkWorkers = 2
timeoutError = fmt.Errorf("Timeout")
)
 
type Client struct {
@@ -23,8 +26,17 @@ type Client struct {
bulk *elastic.BulkProcessor
}
 
func (c *Client) Flush() error {
return c.bulk.Flush()
// FromEnv creates an Elasticsearch client from the `ELASTIC_CONNECTION_INFO`
// environment variable
func FromEnv() (*Client, error) {
data := strings.NewReader(os.Getenv("ELASTIC_CONNECTION_INFO"))
config, err := ReadConfig(data)
if err != nil {
return nil, fmt.Errorf("Couldn't parse ELASTIC_CONNECTION_INFO: %s", err)
}
return NewClient(config)
}
 
func NewClient(config *Config) (*Client, error) {
@@ -68,3 +80,33 @@ func NewClient(config *Config) (*Client, error) {
 
return &Client{client: client, bulk: bulk}, nil
}
func (c *Client) Flush() error {
return c.bulk.Flush()
}
func (c *Client) CreateIndex(indexName, mapping string) error {
createIndex, err := c.client.CreateIndex(indexName).BodyString(mapping).Do(context.Background())
if err != nil {
return err
}
if !createIndex.Acknowledged {
return timeoutError
}
return nil
}
func (c *Client) DeleteIndex(indexName string) error {
deleteIndex, err := c.client.DeleteIndex(indexName).Do(context.Background())
if err != nil {
return err
}
if !deleteIndex.Acknowledged {
return timeoutError
}
return nil
}
elastic/elastic.go
+ 0
− 1
View file @ eecba7d6
@@ -3,7 +3,6 @@ package elastic
import (
"encoding/json"
"io"
"io/ioutil"
)
 
type Config struct {
indexer/create.go 0 → 100644
+ 521
− 0
View file @ eecba7d6
package indexer
var indexMapping = `
{
"mappings": {
"snippet": {
"properties": {
"author_id": {
"type": "integer"
},
"content": {
"type": "text",
"index_options": "offsets"
},
"created_at": {
"type": "date"
},
"file_name": {
"type": "text",
"index_options": "offsets"
},
"id": {
"type": "integer"
},
"project_id": {
"type": "integer"
},
"state": {
"type": "text"
},
"title": {
"type": "text",
"index_options": "offsets"
},
"updated_at": {
"type": "date"
},
"visibility_level": {
"type": "integer"
}
}
},
"note": {
"_parent": {
"type": "project"
},
"_routing": {
"required": true
},
"properties": {
"created_at": {
"type": "date"
},
"id": {
"type": "integer"
},
"issue": {
"properties": {
"assignee_id": {
"type": "integer"
},
"author_id": {
"type": "integer"
},
"confidential": {
"type": "boolean"
}
}
},
"note": {
"type": "text",
"index_options": "offsets"
},
"project_id": {
"type": "integer"
},
"updated_at": {
"type": "date"
}
}
},
"issue": {
"_parent": {
"type": "project"
},
"_routing": {
"required": true
},
"properties": {
"assignee_id": {
"type": "integer"
},
"author_id": {
"type": "integer"
},
"confidential": {
"type": "boolean"
},
"created_at": {
"type": "date"
},
"description": {
"type": "text",
"index_options": "offsets"
},
"id": {
"type": "integer"
},
"iid": {
"type": "integer"
},
"project_id": {
"type": "integer"
},
"state": {
"type": "text"
},
"title": {
"type": "text",
"index_options": "offsets"
},
"updated_at": {
"type": "date"
}
}
},
"project": {
"properties": {
"archived": {
"type": "boolean"
},
"created_at": {
"type": "date"
},
"description": {
"type": "text",
"index_options": "offsets"
},
"id": {
"type": "integer"
},
"last_activity_at": {
"type": "date"
},
"last_pushed_at": {
"type": "date"
},
"name": {
"type": "text",
"index_options": "offsets"
},
"name_with_namespace": {
"type": "text",
"index_options": "offsets",
"analyzer": "my_ngram_analyzer"
},
"namespace_id": {
"type": "integer"
},
"path": {
"type": "text",
"index_options": "offsets"
},
"path_with_namespace": {
"type": "text",
"index_options": "offsets"
},
"updated_at": {
"type": "date"
},
"visibility_level": {
"type": "integer"
}
}
},
"project_wiki": {
"_parent": {
"type": "project"
},
"_routing": {
"required": true
},
"properties": {
"blob": {
"properties": {
"commit_sha": {
"type": "text",
"index_options": "offsets",
"analyzer": "sha_analyzer"
},
"content": {
"type": "text",
"index_options": "offsets",
"analyzer": "code_analyzer",
"search_analyzer": "code_search_analyzer"
},
"file_name": {
"type": "text",
"analyzer": "code_analyzer",
"search_analyzer": "code_search_analyzer"
},
"id": {
"type": "text",
"index_options": "offsets",
"analyzer": "sha_analyzer"
},
"language": {
"type": "keyword"
},
"oid": {
"type": "text",
"index_options": "offsets",
"analyzer": "sha_analyzer"
},
"path": {
"type": "text",
"analyzer": "path_analyzer"
},
"rid": {
"type": "keyword"
}
}
},
"commit": {
"properties": {
"author": {
"properties": {
"email": {
"type": "text",
"index_options": "offsets"
},
"name": {
"type": "text",
"index_options": "offsets"
},
"time": {
"type": "date",
"format": "basic_date_time_no_millis"
}
}
},
"commiter": {
"properties": {
"email": {
"type": "text",
"index_options": "offsets"
},
"name": {
"type": "text",
"index_options": "offsets"
},
"time": {
"type": "date",
"format": "basic_date_time_no_millis"
}
}
},
"id": {
"type": "text",
"index_options": "offsets",
"analyzer": "sha_analyzer"
},
"message": {
"type": "text",
"index_options": "offsets"
},
"rid": {
"type": "keyword"
},
"sha": {
"type": "text",
"index_options": "offsets",
"analyzer": "sha_analyzer"
}
}
}
}
},
"merge_request": {
"_parent": {
"type": "project"
},
"_routing": {
"required": true
},
"properties": {
"author_id": {
"type": "integer"
},
"created_at": {
"type": "date"
},
"description": {
"type": "text",
"index_options": "offsets"
},
"id": {
"type": "integer"
},
"iid": {
"type": "integer"
},
"merge_status": {
"type": "text"
},
"source_branch": {
"type": "text",
"index_options": "offsets"
},
"source_project_id": {
"type": "integer"
},
"state": {
"type": "text"
},
"target_branch": {
"type": "text",
"index_options": "offsets"
},
"target_project_id": {
"type": "integer"
},
"title": {
"type": "text",
"index_options": "offsets"
},
"updated_at": {
"type": "date"
}
}
},
"milestone": {
"_parent": {
"type": "project"
},
"_routing": {
"required": true
},
"properties": {
"created_at": {
"type": "date"
},
"description": {
"type": "text",
"index_options": "offsets"
},
"id": {
"type": "integer"
},
"project_id": {
"type": "integer"
},
"title": {
"type": "text",
"index_options": "offsets"
},
"updated_at": {
"type": "date"
}
}
},
"repository": {
"_parent": {
"type": "project"
},
"_routing": {
"required": true
},
"properties": {
"blob": {
"properties": {
"commit_sha": {
"type": "text",
"index_options": "offsets",
"analyzer": "sha_analyzer"
},
"content": {
"type": "text",
"index_options": "offsets",
"analyzer": "code_analyzer",
"search_analyzer": "code_search_analyzer"
},
"file_name": {
"type": "text",
"analyzer": "code_analyzer",
"search_analyzer": "code_search_analyzer"
},
"id": {
"type": "text",
"index_options": "offsets",
"analyzer": "sha_analyzer"
},
"language": {
"type": "keyword"
},
"oid": {
"type": "text",
"index_options": "offsets",
"analyzer": "sha_analyzer"
},
"path": {
"type": "text",
"analyzer": "path_analyzer"
},
"rid": {
"type": "keyword"
},
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"commit": {
"properties": {
"author": {
"properties": {
"email": {
"type": "text",
"index_options": "offsets"
},
"name": {
"type": "text",
"index_options": "offsets"
},
"time": {
"type": "date",
"format": "basic_date_time_no_millis"
}
}
},
"commiter": {
"properties": {
"email": {
"type": "text",
"index_options": "offsets"
},
"name": {
"type": "text",
"index_options": "offsets"
},
"time": {
"type": "date",
"format": "basic_date_time_no_millis"
}
}
},
"committer": {
"properties": {
"email": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"time": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"id": {
"type": "text",
"index_options": "offsets",
"analyzer": "sha_analyzer"
},
"message": {
"type": "text",
"index_options": "offsets"
},
"rid": {
"type": "keyword"
},
"sha": {
"type": "text",
"index_options": "offsets",
"analyzer": "sha_analyzer"
},
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}
}
}
`
// Creates an index matching that created by gitlab-elasticsearch-git v1.1.1
func (i *Indexer) CreateIndex() error {
return i.Submitter.CreateIndex(i.IndexName, indexMapping)
}
indexer/indexer.go
+ 2
− 1
View file @ eecba7d6
@@ -10,6 +10,7 @@ import (
)
 
type Submitter interface {
CreateIndex(indexName, mapping string) error
Flush() error
}
 
@@ -49,7 +50,7 @@ func (i *Indexer) IndexCommits() error {
}
 
func (i *Indexer) IndexBlobs() error {
return i.Repo.EachFileChange(i.SubmitBlob, i.SubmitBLob, i.RemoveBlob)
return i.Repo.EachFileChange(i.SubmitBlob, i.SubmitBlob, i.RemoveBlob)
}
 
func (i *Indexer) Index() error {
integration_test.go 0 → 100644
+ 59
− 0
View file @ eecba7d6
package main_test
import (
"flag"
"fmt"
"os"
"testing"
"time"
"github.com/stretchr/testify/assert"
"gitlab.com/gitlab-org/es-git-go/elastic"
"gitlab.com/gitlab-org/es-git-go/indexer"
)
var (
binary = flag.String("binary", "./bin/es-git-go", "Path to `es-git-go` binary for integration tests")
)
func checkDeps(t *testing.T) {
if os.Getenv("ELASTIC_CONNECTION_INFO") == "" {
t.Log("ELASTIC_CONNECTION_INFO not set")
t.Skip()
}
if testing.Short() {
t.Log("Test run with -short, skipping integration test")
t.Skip()
}
if _, err := os.Stat(*binary); err != nil {
t.Log("No binary found at ", *binary)
t.Skip()
}
}
func buildIndex(t *testing.T) (string, func()) {
railsEnv := fmt.Sprintf("test-%d", time.Now().Unix())
indexName := fmt.Sprintf("gitlab-" + railsEnv)
client, err := elastic.FromEnv()
assert.NoError(t, err)
indexer := indexer.Indexer{IndexName: indexName, Submitter: client}
assert.NoError(t, indexer.CreateIndex())
return railsEnv, func() {
client.DeleteIndex(indexName)
}
}
func TestIndexingSelf(t *testing.T) {
checkDeps(t)
indexName, td := buildIndex(t)
defer td()
// We're going to index our own git repository, because why not?
_ = indexName
}
main.go
+ 2
− 8
View file @ eecba7d6
@@ -3,7 +3,6 @@ package main
import (
"log"
"os"
"strings"
 
"gitlab.com/gitlab-org/es-git-go/elastic"
"gitlab.com/gitlab-org/es-git-go/git"
@@ -15,14 +14,9 @@ func main() {
log.Fatalf("Usage: %s <project-id> <project-path>", os.Args[0])
}
 
esConfig, err := elastic.ReadConfig(strings.NewReader(os.Getenv("ELASTIC_CONNECTION_INFO")))
esClient, err := elastic.FromEnv()
if err != nil {
log.Fatalln("Couldn't parse ELASTIC_CONNECTION_INFO:", err)
}
esClient, err := elastic.NewClient(esConfig)
if err != nil {
log.Fatalln("Failed to connect to elasticsearch:", err)
log.Fatalln(err)
}
 
projectID := os.Args[1]
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment