Merge branch 'master' of gitlab.com:gitlab-org/gitlab-ci

.gitignore

@@ -17,3 +17,4 @@ tmp/*
 /db/*.sqlite3
 /log/*.log
 /.idea
+/builds/*

CHANGELOG

@@ -9,9 +9,11 @@ v7.13.0
   - Redirect back after authorization
   - Change favicon
   - Refactoring: Get rid of private_token usage in the frontend.
+  - Allow to specify allow_failure for job
 v7.12.2
   - Revert: Runner without tag should pick builds without tag only
+  - Build traces is stored in the file instead of database
 v7.12.1
   - Runner without tag should pick builds without tag only

app/models/build.rb

@@ -2,22 +2,23 @@
 #
 # Table name: builds
 #
-#  id          :integer          not null, primary key
-#  project_id  :integer
-#  status      :string(255)
-#  finished_at :datetime
-#  trace       :text
-#  created_at  :datetime
-#  updated_at  :datetime
-#  started_at  :datetime
-#  runner_id   :integer
-#  commit_id   :integer
-#  coverage    :float
-#  commands    :text
-#  options     :text
-#  job_id      :integer
-#  name        :string(255)
-#  deploy      :boolean          default(FALSE)
+#  id            :integer          not null, primary key
+#  project_id    :integer
+#  status        :string(255)
+#  finished_at   :datetime
+#  trace         :text
+#  created_at    :datetime
+#  updated_at    :datetime
+#  started_at    :datetime
+#  runner_id     :integer
+#  commit_id     :integer
+#  coverage      :float
+#  commands      :text
+#  job_id        :integer
+#  name          :string(255)
+#  deploy        :boolean          default(FALSE)
+#  options       :text
+#  allow_failure :boolean          default(FALSE), not null
 #
 class Build < ActiveRecord::Base
@@ -75,6 +76,7 @@ class Build < ActiveRecord::Base
       new_build.commit_id = build.commit_id
       new_build.project_id = build.project_id
       new_build.name = build.name
+      new_build.allow_failure = build.allow_failure
       new_build.save
       new_build
     end
@@ -153,6 +155,10 @@ class Build < ActiveRecord::Base
     canceled? || success? || failed?
   end
+  def ignored?
+    failed? && allow_failure?
+  end
+
   def timeout
     project.timeout
   end
@@ -210,4 +216,37 @@ class Build < ActiveRecord::Base
       # so we just silentrly ignore error for now
     end
   end
+
+  def trace
+    if File.exist?(path_to_trace)
+      File.read(path_to_trace)
+    else
+      # backward compatibility
+      read_attribute :trace
+    end
+  end
+
+  def trace=(trace)
+    unless Dir.exists? dir_to_trace
+      FileUtils.mkdir_p dir_to_trace
+    end
+
+    File.write(path_to_trace, trace)
+  end
+
+  def dir_to_trace
+    Rails.root.join(
+      root_dir_to_trace,
+      created_at.utc.strftime("%Y_%m"),
+      project.id.to_s
+    )
+  end
+
+  def root_dir_to_trace
+    "builds"
+  end
+
+  def path_to_trace
+    "#{dir_to_trace}/#{id}.log"
+  end
 end

app/models/commit.rb

@@ -111,7 +111,8 @@ class Commit < ActiveRecord::Base
         name: build_attrs[:name],
         commands: build_attrs[:script],
         tag_list: build_attrs[:tags],
-        options: build_attrs[:options]
+        options: build_attrs[:options],
+        allow_failure: build_attrs[:allow_failure]
       })
     end
   end
@@ -149,6 +150,7 @@ class Commit < ActiveRecord::Base
         commands: build_attrs[:script],
         tag_list: build_attrs[:tags],
         options: build_attrs[:options],
+        allow_failure: build_attrs[:allow_failure],
         deploy: true
       })
     end
@@ -186,7 +188,7 @@ class Commit < ActiveRecord::Base
   def success?
     builds_without_retry.all? do |build|
-      build.success?
+      build.success? || build.ignored?
     end
   end

app/views/builds/_build.html.haml

@@ -17,11 +17,13 @@
       #{build.short_sha}
   %td
     = build.name
-    - if build.tags.any?
-      - build.tag_list.each do |tag|
-        %span.label.label-primary
-          = tag
-
+    .pull-right
+      - if build.tags.any?
+        - build.tag_list.each do |tag|
+          %span.label.label-primary
+            = tag
+      - if build.allow_failure
+        %span.label.label-danger allowed to fail
   %td.duration
     - if build.duration

builds/.gitkeep

+.gitkeep
\ No newline at end of file

db/migrate/20150707134456_add_allow_failure_to_builds.rb

+class AddAllowFailureToBuilds < ActiveRecord::Migration
+  def change
+    add_column :builds, :allow_failure, :boolean, default: false, null: false
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema.define(version: 20150706103229) do
+ActiveRecord::Schema.define(version: 20150707134456) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -30,8 +30,9 @@ ActiveRecord::Schema.define(version: 20150706103229) do
     t.text     "commands"
     t.integer  "job_id"
     t.string   "name"
-    t.boolean  "deploy",      default: false
+    t.boolean  "deploy",        default: false
     t.text     "options"
+    t.boolean  "allow_failure", default: false, null: false
   end
   add_index "builds", ["commit_id"], name: "index_builds_on_commit_id", using: :btree

doc/install/installation.md

@@ -129,6 +129,9 @@ We recommend PostgreSQL but you can also use MySQL
     sudo -u gitlab_ci -H mkdir -p tmp/pids/
     sudo chmod -R u+rwX  tmp/pids/
+    # Make sure GitLab CI can write to the builds/ directory
+    sudo chmod -R u+rwX  builds
+
 ### Install gems
     # For MySQL (note, the option says "without ... postgres")

doc/raketasks/backup_restore.md

@@ -2,7 +2,7 @@
 ## Create a backup of the GitLab CI
-A backup creates an archive file that contains the database.
+A backup creates an archive file that contains the database and builds files.
 This archive will be saved in backup_path (see `config/application.yml`).
 The filename will be `[TIMESTAMP]_gitlab_ci_backup.tar.gz`. This timestamp can be used to restore an specific backup.
 You can only restore a backup to exactly the same version of GitLab CI that you created it on, for example 7.10.1.
@@ -24,6 +24,8 @@ Example output:
 Dumping database ...
 Dumping PostgreSQL database gitlab_ci_development ... [DONE]
 done
+Dumping builds ...
+done
 Creating backup archive: 1430930060_gitlab_ci_backup.tar.gz ... done
 Uploading backup archive to remote storage  ... skipped
 Deleting tmp directories ... done

doc/update/7.12-to-7.13.md

+# Update from 7.12 to 7.13
+
+## Notice
+
+__GitLab CI 7.13 requires GitLab 7.12 or higher and GitLab Multi Runner 0.4.0 or higher
+
+### 1. Stop CI server
+
+    sudo service gitlab_ci stop
+
+### 2. Switch to your gitlab_ci user
+
+```
+sudo su gitlab_ci
+cd /home/gitlab_ci/gitlab-ci
+```
+
+### 3. Get latest code
+
+```
+git fetch
+git checkout 7-13-stable
+```
+
+### 4. Make sure GitLab CI can write to the builds/ directory
+
+```
+sudo chmod -R u+rwX  builds
+```
+
+### 5. Install libs, migrations etc
+
+
+```
+# Install nodejs dependency:
+sudo apt-get install nodejs
+
+# For MySQL users
+bundle install --without postgres development test --deployment
+
+# For Postgres users
+bundle install --without mysql development test --deployment
+
+# Run migrations
+bundle exec rake db:migrate RAILS_ENV=production
+```
+
+
+### 5. Start web application
+
+    sudo service gitlab_ci start

lib/backup/builds.rb

+module Backup
+  class Builds
+    attr_reader :app_builds_dir, :backup_builds_dir, :backup_dir
+
+    def initialize
+      @app_builds_dir = File.realpath(Rails.root.join('builds'))
+      @backup_dir = GitlabCi.config.backup.path
+      @backup_builds_dir = File.join(GitlabCi.config.backup.path, 'builds')
+    end
+
+    # Copy builds from builds directory to backup/builds
+    def dump
+      FileUtils.mkdir_p(backup_builds_dir)
+      FileUtils.cp_r(app_builds_dir, backup_dir)
+    end
+
+    def restore
+      backup_existing_builds_dir
+
+      FileUtils.cp_r(backup_builds_dir, app_builds_dir)
+    end
+
+    def backup_existing_builds_dir
+      timestamped_builds_path = File.join(app_builds_dir, '..', "builds.#{Time.now.to_i}")
+      if File.exists?(app_builds_dir)
+        FileUtils.mv(app_builds_dir, File.expand_path(timestamped_builds_path))
+      end
+    end
+  end
+end

lib/backup/manager.rb

@@ -15,7 +15,7 @@ module Backup
           file << s.to_yaml.gsub(/^---\n/,'')
         end
-        FileUtils.chmod(0700, "db")
+        FileUtils.chmod(0700, ["db", "builds"])
         # create archive
         $progress.print "Creating backup archive: #{tar_file} ... "
@@ -146,7 +146,7 @@ module Backup
     private
     def backup_contents
-      ["db", "backup_information.yml"]
+      ["db", "builds", "backup_information.yml"]
     end
     def settings

lib/gitlab_ci_yaml_processor.rb

@@ -84,6 +84,7 @@ class GitlabCiYamlProcessor
       name: name,
       only: job[:only],
       except: job[:except],
+      allow_failure: job[:allow_failure] || false,
       options: {
         image: job[:image] || @image,
         services: job[:services] || @services
@@ -133,7 +134,7 @@ class GitlabCiYamlProcessor
   def validate_job!(name, job)
     job.keys.each do |key|
-      unless [:tags, :script, :only, :except, :type, :image, :services].include? key
+      unless [:tags, :script, :only, :except, :type, :image, :services, :allow_failure].include? key
         raise ValidationError, "#{name}: unknown parameter #{key}"
       end
     end
@@ -159,5 +160,9 @@ class GitlabCiYamlProcessor
     if job[:except] && !job[:except].is_a?(Array)
       raise ValidationError, "#{name}: except parameter should be an array"
     end
+
+    if job[:allow_failure] && !job[:allow_failure].in?([true, false])
+      raise ValidationError, "#{name}: allow_failure parameter should be an boolean"
+    end
   end
 end

lib/tasks/backup.rake

@@ -5,10 +5,13 @@ namespace :backup do
     configure_cron_mode
     $progress.puts "Dumping database ... ".blue
-
     Backup::Database.new.dump
     $progress.puts "done".green
+    $progress.puts "Dumping builds ... ".blue
+    Backup::Builds.new.dump
+    $progress.puts "done".green
+
     backup = Backup::Manager.new
     backup.pack
     backup.cleanup
@@ -26,6 +29,10 @@ namespace :backup do
     Backup::Database.new.restore
     $progress.puts "done".green
+    $progress.puts "Restoring builds ... ".blue
+    Backup::Builds.new.restore
+    $progress.puts "done".green
+
     backup.cleanup
   end

spec/factories/builds.rb

@@ -2,22 +2,23 @@
 #
 # Table name: builds
 #
-#  id          :integer          not null, primary key
-#  project_id  :integer
-#  status      :string(255)
-#  finished_at :datetime
-#  trace       :text
-#  created_at  :datetime
-#  updated_at  :datetime
-#  started_at  :datetime
-#  runner_id   :integer
-#  commit_id   :integer
-#  coverage    :float
-#  commands    :text
-#  options     :text
-#  job_id      :integer
-#  name        :string(255)
-#  deploy      :boolean          default(FALSE)
+#  id            :integer          not null, primary key
+#  project_id    :integer
+#  status        :string(255)
+#  finished_at   :datetime
+#  trace         :text
+#  created_at    :datetime
+#  updated_at    :datetime
+#  started_at    :datetime
+#  runner_id     :integer
+#  commit_id     :integer
+#  coverage      :float
+#  commands      :text
+#  job_id        :integer
+#  name          :string(255)
+#  deploy        :boolean          default(FALSE)
+#  options       :text
+#  allow_failure :boolean          default(FALSE), not null
 #
 # Read about factories at https://github.com/thoughtbot/factory_girl

spec/factories/commits.rb

@@ -49,5 +49,17 @@ FactoryGirl.define do
         ci_yaml_file: File.read(Rails.root.join('spec/support/gitlab_stubs/gitlab_ci.yml'))
       }
     end
+
+    factory :commit_with_one_job do
+      after(:create) do |commit, evaluator|
+        commit.push_data[:ci_yaml_file] = YAML.dump({rspec: { script: "ls" }})
+      end
+    end
+
+    factory :commit_with_two_jobs do
+      after(:create) do |commit, evaluator|
+        commit.push_data[:ci_yaml_file] = YAML.dump({rspec: { script: "ls" }, spinach: { script: "ls" }})
+      end
+    end
   end
 end

spec/lib/gitlab_ci_yaml_processor_spec.rb

@@ -18,7 +18,8 @@ describe GitlabCiYamlProcessor do
         only: nil,
         script: "pwd\nrspec",
         tags: [],
-        options: {}
+        options: {},
+        allow_failure: false
       }
     end
@@ -71,7 +72,7 @@ describe GitlabCiYamlProcessor do
     it "returns builds if no branch specified" do
       config = YAML.dump({
         before_script: ["pwd"],
-        rspec: {script: "rspec", type: "deploy"}
+        rspec: {script: "rspec", type: "deploy", allow_failure: true}
       })
       config_processor = GitlabCiYamlProcessor.new(config)
@@ -83,7 +84,8 @@ describe GitlabCiYamlProcessor do
         only: nil,
         script: "pwd\nrspec",
         tags: [],
-        options: {}
+        options: {},
+        allow_failure: true
       }
     end
@@ -153,7 +155,8 @@ describe GitlabCiYamlProcessor do
         options: {
           image: "ruby:2.1",
           services: ["mysql"]
-        }
+        },
+        allow_failure: false
       }
     end
@@ -177,7 +180,8 @@ describe GitlabCiYamlProcessor do
         options: {
           image: "ruby:2.5",
           services: ["postgresql"]
-        }
+        },
+        allow_failure: false
       }
     end
   end
@@ -256,5 +260,12 @@ describe GitlabCiYamlProcessor do
         GitlabCiYamlProcessor.new(config)
       end.to raise_error(GitlabCiYamlProcessor::ValidationError, "Please define at least one job")
     end
+
+    it "returns errors if job allow_failure parameter is not an boolean" do
+      config = YAML.dump({rspec: {script: "test", allow_failure: "string"}})
+      expect do
+        GitlabCiYamlProcessor.new(config)
+      end.to raise_error(GitlabCiYamlProcessor::ValidationError, "rspec job: allow_failure parameter should be an boolean")
+    end
   end
 end
\ No newline at end of file

spec/models/build_spec.rb

@@ -2,22 +2,23 @@
 #
 # Table name: builds
 #
-#  id          :integer          not null, primary key
-#  project_id  :integer
-#  status      :string(255)
-#  finished_at :datetime
-#  trace       :text
-#  created_at  :datetime
-#  updated_at  :datetime
-#  started_at  :datetime
-#  runner_id   :integer
-#  commit_id   :integer
-#  coverage    :float
-#  commands    :text
-#  options     :text
-#  job_id      :integer
-#  name        :string(255)
-#  deploy      :boolean          default(FALSE)
+#  id            :integer          not null, primary key
+#  project_id    :integer
+#  status        :string(255)
+#  finished_at   :datetime
+#  trace         :text
+#  created_at    :datetime
+#  updated_at    :datetime
+#  started_at    :datetime
+#  runner_id     :integer
+#  commit_id     :integer
+#  coverage      :float
+#  commands      :text
+#  job_id        :integer
+#  name          :string(255)
+#  deploy        :boolean          default(FALSE)
+#  options       :text
+#  allow_failure :boolean          default(FALSE), not null
 #
 require 'spec_helper'
@@ -126,6 +127,42 @@ describe Build do
     end
   end
+  describe :ignored? do
+    subject { build.ignored? }
+
+    context 'if build is not allowed to fail' do
+      before { build.allow_failure = false }
+
+      context 'and build.status is success' do
+        before { build.status = 'success' }
+
+        it { should be_false }
+      end
+
+      context 'and build.status is failed' do
+        before { build.status = 'failed' }
+
+        it { should be_false }
+      end
+    end
+
+    context 'if build is allowed to fail' do
+      before { build.allow_failure = true }
+
+      context 'and build.status is success' do
+        before { build.status = 'success' }
+
+        it { should be_false }
+      end
+
+      context 'and build.status is failed' do
+        before { build.status = 'failed' }
+
+        it { should be_true }
+      end
+    end
+  end
+
   describe :trace do
     subject { build.trace_html }

spec/models/project_services/hip_chat_message_spec.rb

@@ -6,12 +6,7 @@ describe HipChatMessage do
   let(:project) { FactoryGirl.create(:project) }
   context "One build" do
-    let(:commit) do
-      commit = FactoryGirl.create(:commit, project: project)
-      commit.push_data[:ci_yaml_file] = YAML.dump({rspec: { script: 'pwd' }})
-      commit.save
-      commit
-    end
+    let(:commit) { FactoryGirl.create(:commit_with_one_job, project: project) }
     let(:build) do 
       commit.create_builds
@@ -42,7 +37,7 @@ describe HipChatMessage do
   end
   context "Several builds" do
-    let(:commit) {commit = FactoryGirl.create(:commit, project: project)}
+    let(:commit) { FactoryGirl.create(:commit_with_two_jobs, project: project) }
     let(:build) do 
       commit.builds.first