Merge branch 'cabeca-master'

Gemfile

@@ -47,6 +47,9 @@ gem 'will_paginate', '~> 3.0'
 # State machine
 gem 'state_machine'
+# Encoding detection
+gem 'charlock_holmes'
+
 # Other
 gem 'rake'
 gem 'foreman'

Gemfile.lock

@@ -44,6 +44,7 @@ GEM
     celluloid (0.12.4)
       facter (>= 1.6.12)
       timers (>= 1.0.0)
+    charlock_holmes (0.6.9)
     childprocess (0.3.5)
       ffi (~> 1.0, >= 1.0.6)
     chronic (0.9.0)
@@ -244,6 +245,7 @@ DEPENDENCIES
   annotate
   bootstrap-sass
   capybara
+  charlock_holmes
   childprocess
   coffee-rails (~> 3.2.1)
   devise

app/models/build.rb

@@ -73,7 +73,8 @@ class Build < ActiveRecord::Base
   def write_trace(trace)
     self.reload
-    update_attributes(trace: trace)
+    sanitized_output = sanitize_build_output(trace)
+    update_attributes(trace: sanitized_output)
   end
   def short_before_sha
@@ -92,12 +93,24 @@ class Build < ActiveRecord::Base
     end
   end
+  def sanitize_build_output(output)
+    GitlabCi::Encode.encode!(output)
+  end
+
+  def read_tmp_file
+    content = if tmp_file && File.readable?(tmp_file)
+                File.read(tmp_file)
+              end
+
+    content ||= ''
+  end
   def compose_output
     output = trace
-    if running? && tmp_file && File.exists?(tmp_file)
-      output << File.read(tmp_file)
+    if running?
+      sanitized_output = sanitize_build_output(read_tmp_file)
+      output << sanitized_output if sanitized_output.present?
     end
     output

config/initializers/encoding_detection.rb

+require 'charlock_holmes/string'
\ No newline at end of file

doc/installation.md

@@ -10,7 +10,7 @@ Create a user for GitLab:
     sudo apt-get update
     sudo apt-get upgrade
-    sudo apt-get install -y wget curl gcc checkinstall libxml2-dev libxslt-dev libcurl4-openssl-dev libreadline6-dev libc6-dev libssl-dev libmysql++-dev make build-essential zlib1g-dev openssh-server git-core libyaml-dev postfix libpq-dev
+    sudo apt-get install -y wget curl gcc checkinstall libxml2-dev libxslt-dev libcurl4-openssl-dev libreadline6-dev libc6-dev libssl-dev libmysql++-dev make build-essential zlib1g-dev openssh-server git-core libyaml-dev postfix libpq-dev libicu-dev
     sudo apt-get install redis-server 
 ## 2. Install Ruby (RVM) for gitlab_ci

lib/gitlab_ci/encode.rb

+module GitlabCi
+  module Encode
+    extend self
+
+    def encode!(message)
+      return nil unless message.respond_to? :force_encoding
+
+      # if message is utf-8 encoding, just return it
+      message.force_encoding("UTF-8")
+      return message if message.valid_encoding?
+
+      # return message if message type is binary
+      detect = CharlockHolmes::EncodingDetector.detect(message)
+      return message if detect[:type] == :binary
+
+      # if message is not utf-8 encoding, convert it
+      if detect[:encoding]
+        message.force_encoding(detect[:encoding])
+        message.encode!("UTF-8", detect[:encoding], undef: :replace, replace: "", invalid: :replace)
+      end
+
+      # ensure message encoding is utf8
+      message.valid_encoding? ? message : raise
+
+      # Prevent app from crash cause of encoding errors
+    rescue
+      encoding = detect ? detect[:encoding] : "unknown"
+      "--broken encoding: #{encoding}"
+    end
+  end
+end