Add integration tests for charset conversions

integration_test.go

@@ -94,6 +94,32 @@ func TestIndexingRemovesFiles(t *testing.T) {
 	assert.Error(t, err)
 }
+// Go source is defined to be UTF-8 encoded, so literals here are UTF-8
+func TestIndexingTranscodesToUTF8(t *testing.T) {
+	checkDeps(t)
+	c, td := buildIndex(t)
+	defer td()
+
+	assert.NoError(t, run("", headSHA))
+
+	for _, tc := range []struct{
+			path string
+			expected string
+		} {
+			{"encoding/iso8859.txt", "狞\n"}, // GB18030
+			{"encoding/test.txt", "これはテストです。\nこれもマージして下さい。\n\nAdd excel file.\nDelete excel file."}, // SHIFT_JIS
+		} {
+
+		blob, err := c.GetBlob(tc.path)
+		assert.NoError(t, err)
+
+		blobDoc := make(map[string]*indexer.Blob)
+		assert.NoError(t, json.Unmarshal(*blob.Source, &blobDoc))
+
+		assert.Equal(t, tc.expected, blobDoc["blob"].Content)
+	}
+}
+
 func TestIndexingGitlabTest(t *testing.T) {
 	checkDeps(t)
 	c, td := buildIndex(t)