add redis cluster to test setup

circle.yml

@@ -6,10 +6,11 @@ machine:
     - /usr/bin/redis-server --port 6380:
           background: true
   environment:
-    SRC_LOCATION:     "/home/ubuntu/.go_workspace/src/github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME"
-    COVERAGE_PROFILE: "/home/ubuntu/coverage.out"
-    GO_LDFLAGS:       '-extldflags "-static" -X main.VERSION=$CIRCLE_TAG -X main.COMMIT_SHA1=$CIRCLE_SHA1 -X main.BUILD_DATE=$(date +%F-%T)'
-    MY_GO_VERSION:    "1.9.2"
+    SRC_LOCATION:       "/home/ubuntu/.go_workspace/src/github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME"
+    COVERAGE_PROFILE:   "/home/ubuntu/coverage.out"
+    GO_LDFLAGS:         '-extldflags "-static" -X main.VERSION=$CIRCLE_TAG -X main.COMMIT_SHA1=$CIRCLE_SHA1 -X main.BUILD_DATE=$(date +%F-%T)'
+    MY_GO_VERSION:      "1.9.2"
+    REDIS_TEST_VERSION: "3.2.11"
 dependencies:
   pre:
@@ -17,16 +18,20 @@ dependencies:
     - rm -rf /home/ubuntu/.go_project
     - sudo service redis-server stop
     - >
-      cd ~ && if [ ! -d "redis-3.2.10" ]; then
-        wget http://download.redis.io/releases/redis-3.2.10.tar.gz
-        tar xzf redis-3.2.10.tar.gz
-        cd redis-3.2.10 && make;
+      cd ~ && if [ ! -d "redis-$REDIS_TEST_VERSION" ]; then
+        wget http://download.redis.io/releases/redis-$REDIS_TEST_VERSION.tar.gz
+        tar xzf redis-$REDIS_TEST_VERSION.tar.gz
+        cd redis-$REDIS_TEST_VERSION && make;
       fi
-    - cd ~/redis-3.2.10 && sudo make install
+    - cd ~/redis-$REDIS_TEST_VERSION && sudo make install
     - sudo sed -i 's/bin/local\/bin/g' /etc/init/redis-server.conf
     - sudo service redis-server start
+    #
+    # the next line will bring up a cluster of redis instances with slaves
+    # for addtl tests
+    - docker run -p 7000-7006:7000-7006 --name redis_cluster_test -d oliver006/redis_cluster_test
   cache_directories:
-    - ~/redis-3.2.10
+    - ~/redis-$REDIS_TEST_VERSION
   override:
     - mkdir -p "/home/ubuntu/.go_workspace/src/github.com/$CIRCLE_PROJECT_USERNAME"
     - ln -s $HOME/$CIRCLE_PROJECT_REPONAME $SRC_LOCATION

exporter/redis.go

@@ -570,6 +570,16 @@ func (e *Exporter) extractInfoMetrics(info, addr string, alias string, scrapes c
 	return nil
 }
+func doRedisCmd(c redis.Conn, cmd string, args ...interface{}) (reply interface{}, err error) {
+	log.Debugf("c.Do() - running command: %s %s", cmd, args)
+	defer log.Debugf("c.Do() - done")
+	res, err := c.Do(cmd, args...)
+	if err != nil {
+		log.Debugf("c.Do() - err: %s", err)
+	}
+	return res, err
+}
+
 func (e *Exporter) scrapeRedisHost(scrapes chan<- scrapeResult, addr string, idx int) error {
 	options := []redis.DialOption{
 		redis.DialConnectTimeout(5 * time.Second),
@@ -596,7 +606,7 @@ func (e *Exporter) scrapeRedisHost(scrapes chan<- scrapeResult, addr string, idx
 	}
 	if err != nil {
-		log.Printf("redis err: %s", err)
+		log.Debugf("aborting for addr: %s - redis err: %s", addr, err)
 		return err
 	}
@@ -615,7 +625,7 @@ func (e *Exporter) scrapeRedisHost(scrapes chan<- scrapeResult, addr string, idx
 		log.Debugf("Redis CONFIG err: %s", err)
 	}
-	info, err := redis.String(c.Do("INFO", "ALL"))
+	info, err := redis.String(doRedisCmd(c, "INFO", "ALL"))
 	if err == nil {
 		e.extractInfoMetrics(info, addr, e.redis.Aliases[idx], scrapes, dbCount, true)
 	} else {
@@ -623,8 +633,8 @@ func (e *Exporter) scrapeRedisHost(scrapes chan<- scrapeResult, addr string, idx
 		return err
 	}
-	if strings.Index(info, "cluster_enabled:1") != -1 {
-		info, err = redis.String(c.Do("CLUSTER", "INFO"))
+	if strings.Contains(info, "cluster_enabled:1") {
+		info, err = redis.String(doRedisCmd(c, "CLUSTER", "INFO"))
 		if err != nil {
 			log.Errorf("redis err: %s", err)
 		} else {
@@ -632,7 +642,7 @@ func (e *Exporter) scrapeRedisHost(scrapes chan<- scrapeResult, addr string, idx
 		}
 	}
-	if reply, err := c.Do("LATENCY", "LATEST"); err == nil {
+	if reply, err := doRedisCmd(c, "LATENCY", "LATEST"); err == nil {
 		var eventName string
 		var spikeLast, milliseconds, max int64
 		if tempVal, _ := reply.([]interface{}); len(tempVal) > 0 {
@@ -646,13 +656,14 @@ func (e *Exporter) scrapeRedisHost(scrapes chan<- scrapeResult, addr string, idx
 		}
 	}
+	log.Debugf("e.keys: %#v", e.keys)
 	for _, k := range e.keys {
-		if _, err := c.Do("SELECT", k.db); err != nil {
+		if _, err := doRedisCmd(c, "SELECT", k.db); err != nil {
 			continue
 		}
 		obtainedKeys := []string{}
-		if tempVal, err := redis.Strings(c.Do("KEYS", k.key)); err == nil && tempVal != nil {
+		if tempVal, err := redis.Strings(doRedisCmd(c, "KEYS", k.key)); err == nil && tempVal != nil {
 			for _, tempKey := range tempVal {
 				log.Debugf("Append result: %s", tempKey)
 				obtainedKeys = append(obtainedKeys, tempKey)
@@ -662,7 +673,7 @@ func (e *Exporter) scrapeRedisHost(scrapes chan<- scrapeResult, addr string, idx
 		for _, key := range obtainedKeys {
 			dbLabel := "db" + k.db
 			keyLabel := key
-			if tempVal, err := c.Do("GET", key); err == nil && tempVal != nil {
+			if tempVal, err := doRedisCmd(c, "GET", key); err == nil && tempVal != nil {
 				if val, err := strconv.ParseFloat(fmt.Sprintf("%s", tempVal), 64); err == nil {
 					e.keyValues.WithLabelValues(addr, e.redis.Aliases[idx], dbLabel, keyLabel).Set(val)
 				}
@@ -676,7 +687,7 @@ func (e *Exporter) scrapeRedisHost(scrapes chan<- scrapeResult, addr string, idx
 				"PFCOUNT",
 				"STRLEN",
 			} {
-				if tempVal, err := c.Do(op, key); err == nil && tempVal != nil {
+				if tempVal, err := doRedisCmd(c, op, key); err == nil && tempVal != nil {
 					e.keySizes.WithLabelValues(addr, e.redis.Aliases[idx], dbLabel, keyLabel).Set(float64(tempVal.(int64)))
 					break
 				}
@@ -684,18 +695,18 @@ func (e *Exporter) scrapeRedisHost(scrapes chan<- scrapeResult, addr string, idx
 		}
 	}
+	log.Debugf("scrapeRedisHost() done")
 	return nil
 }
 func (e *Exporter) scrape(scrapes chan<- scrapeResult) {
-
 	defer close(scrapes)
+
 	now := time.Now().UnixNano()
 	e.totalScrapes.Inc()
 	errorCount := 0
 	for idx, addr := range e.redis.Addrs {
-
 		var up float64 = 1
 		if err := e.scrapeRedisHost(scrapes, addr, idx); err != nil {
 			errorCount++

exporter/redis_test.go

@@ -13,10 +13,10 @@ import (
 	"flag"
 	"fmt"
 	"io/ioutil"
-	"log"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
+	"os"
 	"strings"
 	"testing"
 	"time"
@@ -25,6 +25,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	dto "github.com/prometheus/client_model/go"
+	log "github.com/sirupsen/logrus"
 )
 const (
@@ -100,25 +101,21 @@ func resetLatency(t *testing.T, addr string) error {
 	return nil
 }
-func getMetrics(t *testing.T) []byte {
-	url := TestServerURL + "/metrics"
-	log.Printf("Getting metrics from: %s", url)
+func downloadUrl(t *testing.T, url string) []byte {
+	log.Debugf("downloadURL() %s", url)
 	resp, err := http.Get(url)
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer resp.Body.Close()
-
 	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		t.Fatal(err)
 	}
-
 	return body
 }
 func TestLatencySpike(t *testing.T) {
-
 	e, _ := NewRedisExporter(defaultRedisHost, "test", "")
 	setupLatency(t, defaultRedisHost.Addrs[0])
@@ -230,7 +227,6 @@ func deleteKeysFromDB(t *testing.T, addr string) error {
 	}
 	c.Do("DEL", TestSetName)
-
 	return nil
 }
@@ -574,13 +570,16 @@ func TestCommandStats(t *testing.T) {
 }
 func TestHTTPEndpoint(t *testing.T) {
+	ts := httptest.NewServer(promhttp.Handler())
+	defer ts.Close()
+
 	e, _ := NewRedisExporter(defaultRedisHost, "test", dbNumStrFull+"="+url.QueryEscape(keys[0]))
 	setupDBKeys(t, defaultRedisHost.Addrs[0])
 	defer deleteKeysFromDB(t, defaultRedisHost.Addrs[0])
 	prometheus.Register(e)
-	body := getMetrics(t)
+	body := downloadUrl(t, ts.URL+"/metrics")
 	tests := []string{
 		// metrics
@@ -603,12 +602,10 @@ func TestHTTPEndpoint(t *testing.T) {
 }
 func TestNonExistingHost(t *testing.T) {
-
 	rr := RedisHost{Addrs: []string{"unix:///tmp/doesnt.exist"}, Aliases: []string{""}}
 	e, _ := NewRedisExporter(rr, "test", "")
 	chM := make(chan prometheus.Metric)
-
 	go func() {
 		e.Collect(chM)
 		close(chM)
@@ -757,6 +754,9 @@ func TestSanitizeMetricName(t *testing.T) {
 }
 func TestKeysReset(t *testing.T) {
+	ts := httptest.NewServer(promhttp.Handler())
+	defer ts.Close()
+
 	e, _ := NewRedisExporter(defaultRedisHost, "test", dbNumStrFull+"="+keys[0])
 	setupDBKeys(t, defaultRedisHost.Addrs[0])
@@ -764,13 +764,13 @@ func TestKeysReset(t *testing.T) {
 	prometheus.Register(e)
-	chM := make(chan prometheus.Metric)
+	chM := make(chan prometheus.Metric, 10000)
 	go func() {
 		e.Collect(chM)
 		close(chM)
 	}()
-	body := getMetrics(t)
+	body := downloadUrl(t, ts.URL+"/metrics")
 	if !bytes.Contains(body, []byte(keys[0])) {
 		t.Errorf("Did not found key %q\n%s", keys[0], body)
@@ -778,14 +778,182 @@ func TestKeysReset(t *testing.T) {
 	deleteKeysFromDB(t, defaultRedisHost.Addrs[0])
-	body = getMetrics(t)
+	body = downloadUrl(t, ts.URL+"/metrics")
 	if bytes.Contains(body, []byte(keys[0])) {
 		t.Errorf("Metric is present in metrics list %q\n%s", keys[0], body)
 	}
 }
+func TestClusterMaster(t *testing.T) {
+	if os.Getenv("TEST_REDIS_CLUSTER_MASTER_URI") == "" {
+		log.Println("TEST_REDIS_CLUSTER_MASTER_URI not set - skipping")
+		t.SkipNow()
+		return
+	}
+
+	ts := httptest.NewServer(promhttp.Handler())
+	defer ts.Close()
+
+	addr := "redis://" + os.Getenv("TEST_REDIS_CLUSTER_MASTER_URI")
+	host := RedisHost{Addrs: []string{addr}, Aliases: []string{"master"}}
+	e, _ := NewRedisExporter(host, "test", "")
+
+	setupDBKeys(t, defaultRedisHost.Addrs[0])
+	defer deleteKeysFromDB(t, defaultRedisHost.Addrs[0])
+	log.Println("wut")
+	prometheus.Register(e)
+
+	chM := make(chan prometheus.Metric, 10000)
+	go func() {
+		e.Collect(chM)
+		close(chM)
+	}()
+
+	body := downloadUrl(t, ts.URL+"/metrics")
+	if !bytes.Contains(body, []byte("test_instance_info")) {
+		t.Errorf("Did not found key %q\n%s", keys[0], body)
+	}
+}
+
+func TestPasswordProtectedInstance(t *testing.T) {
+	ts := httptest.NewServer(promhttp.Handler())
+	defer ts.Close()
+
+	testPwd := "p4$$w0rd"
+	host := defaultRedisHost
+	host.Passwords = []string{testPwd}
+
+	setupDBKeys(t, host.Addrs[0])
+
+	// set password for redis instance
+	c, err := redis.DialURL(host.Addrs[0])
+	if err != nil {
+		t.Errorf("couldn't setup redis, err: %s ", err)
+		return
+	}
+	defer c.Close()
+
+	if _, err = c.Do("CONFIG", "SET", "requirepass", testPwd); err != nil {
+		t.Fatalf("error setting password, err: %s", err)
+	}
+	c.Flush()
+
+	defer func() {
+		if _, err = c.Do("auth", testPwd); err != nil {
+			t.Fatalf("error unsetting password, err: %s", err)
+		}
+		if _, err = c.Do("CONFIG", "SET", "requirepass", ""); err != nil {
+			t.Fatalf("error unsetting password, err: %s", err)
+		}
+		deleteKeysFromDB(t, host.Addrs[0])
+	}()
+	e, _ := NewRedisExporter(host, "test", "")
+
+	prometheus.Register(e)
+
+	chM := make(chan prometheus.Metric, 10000)
+	go func() {
+		e.Collect(chM)
+		close(chM)
+	}()
+
+	body := downloadUrl(t, ts.URL+"/metrics")
+
+	if !bytes.Contains(body, []byte("test_up")) {
+		t.Errorf("error")
+	}
+}
+
+func TestPasswordInvalid(t *testing.T) {
+	ts := httptest.NewServer(promhttp.Handler())
+	defer ts.Close()
+
+	testPwd := "p4$$w0rd"
+	host := defaultRedisHost
+	host.Passwords = []string{"wrong_password"}
+
+	setupDBKeys(t, host.Addrs[0])
+
+	// set password for redis instance
+	c, err := redis.DialURL(host.Addrs[0])
+	if err != nil {
+		t.Errorf("couldn't setup redis, err: %s ", err)
+		return
+	}
+	defer c.Close()
+
+	if _, err = c.Do("CONFIG", "SET", "requirepass", testPwd); err != nil {
+		t.Fatalf("error setting password, err: %s", err)
+	}
+	c.Flush()
+
+	defer func() {
+		if _, err = c.Do("auth", testPwd); err != nil {
+			t.Fatalf("error unsetting password, err: %s", err)
+		}
+		if _, err = c.Do("CONFIG", "SET", "requirepass", ""); err != nil {
+			t.Fatalf("error unsetting password, err: %s", err)
+		}
+		deleteKeysFromDB(t, host.Addrs[0])
+	}()
+	e, _ := NewRedisExporter(host, "test", "")
+
+	prometheus.Register(e)
+
+	chM := make(chan prometheus.Metric, 10000)
+	go func() {
+		e.Collect(chM)
+		close(chM)
+	}()
+
+	body := downloadUrl(t, ts.URL+"/metrics")
+	log.Println(string(body))
+	if !bytes.Contains(body, []byte("test_exporter_last_scrape_error 1")) {
+		t.Errorf(`error, expected string "test_exporter_last_scrape_error 1" in body`)
+	}
+}
+
+func TestClusterSlave(t *testing.T) {
+	if os.Getenv("TEST_REDIS_CLUSTER_SLAVE_URI") == "" {
+		log.Println("TEST_REDIS_CLUSTER_SLAVE_URI not set - skipping")
+		t.SkipNow()
+		return
+	}
+
+	ts := httptest.NewServer(promhttp.Handler())
+	defer ts.Close()
+
+	addr := "redis://" + os.Getenv("TEST_REDIS_CLUSTER_SLAVE_URI")
+	host := RedisHost{Addrs: []string{addr}, Aliases: []string{"slave"}}
+	e, _ := NewRedisExporter(host, "test", "")
+
+	setupDBKeys(t, defaultRedisHost.Addrs[0])
+	defer deleteKeysFromDB(t, defaultRedisHost.Addrs[0])
+	log.Println("wut")
+	prometheus.Register(e)
+
+	chM := make(chan prometheus.Metric, 10000)
+	go func() {
+		e.Collect(chM)
+		close(chM)
+	}()
+
+	body := downloadUrl(t, ts.URL+"/metrics")
+	if !bytes.Contains(body, []byte("test_instance_info")) {
+		t.Errorf("Did not found key %q\n%s", keys[0], body)
+	}
+}
+
 func init() {
+	ll := strings.ToLower(os.Getenv("LOG_LEVEL"))
+	if pl, err := log.ParseLevel(ll); err == nil {
+		log.Printf("Setting log level to: %s", ll)
+		log.SetLevel(pl)
+	} else {
+		log.SetLevel(log.InfoLevel)
+	}
+
 	for _, n := range []string{"john", "paul", "ringo", "george"} {
 		key := fmt.Sprintf("key_%s_%d", n, ts)
 		keys = append(keys, key)
@@ -812,6 +980,4 @@ func init() {
 	log.Printf("Using redis addrs: %#v", addrs)
 	defaultRedisHost = RedisHost{Addrs: []string{"redis://" + *redisAddr}, Aliases: aliases}
-
-	TestServerURL = httptest.NewServer(promhttp.Handler()).URL
 }

main.go

@@ -104,7 +104,7 @@ func main() {
 	} else {
 		prometheus.MustRegister(exp)
 		prometheus.MustRegister(buildInfo)
-		http.Handle(*metricPath, prometheus.Handler())
+		http.Handle(*metricPath, promhttp.Handler())
 	}
 	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {