Allow to specify flexible list of types in yaml

```
types:
- test
- deploy
- notify

rspec:
  script: "rspec"

rubocop:
  script: "rubocop"

staging:
  type: deploy
  script: "echo deploy"
  only:
  - master

production:
  type: deploy
  script: "echo production"
  only:
  - tags

dockerhub:
  type: notify
  script: "curl http://docker/hub/web/hook"

downstream:
  type: notify
  script: "curl http://build/downstream/jobs"
```

This will trigger two test jobs in parallel, when finished it will trigged either staging or production, when finished it will trigger dockerhub and downstream in parallel.

app/controllers/lints_controller.rb

@@ -10,8 +10,8 @@ class LintsController < ApplicationController
       @error = "Please provide content of .gitlab-ci.yml"
     else
       @config_processor = GitlabCiYamlProcessor.new params[:content]
+      @types = @config_processor.types
       @builds = @config_processor.builds
-      @deploy_builds = @config_processor.deploy_builds
       @status = true
     end
   rescue GitlabCiYamlProcessor::ValidationError => e

app/models/build.rb

@@ -111,8 +111,8 @@ class Build < ActiveRecord::Base
         WebHookService.new.build_end(build)
       end
-      if build.commit.success? && !build.deploy?
-        build.commit.create_deploy_builds
+      if build.commit.success?
+        build.commit.create_next_builds
       end
       project.execute_services(build)

app/models/commit.rb

@@ -93,67 +93,67 @@ class Commit < ActiveRecord::Base
     recipients.uniq
   end
-  def create_builds
-    return if skip_ci?
-    begin
-      builds_for_ref = config_processor.builds_for_ref(ref, tag)
-    rescue GitlabCiYamlProcessor::ValidationError => e
-      save_yaml_error(e.message) and return
-    rescue Exception => e
-      logger.error e.message + "\n" + e.backtrace.join("\n")
-      save_yaml_error("Undefined yaml error") and return
-    end
-    builds_for_ref.each do |build_attrs|
+  def job_type
+    return unless config_processor
+    job_types = builds_without_retry.select(&:active?).map(&:job_type)
+    config_processor.types.find { |job_type| job_types.include? job_type }
+  end
+  def create_builds_for_type(job_type)
+    return if skip_ci?
+    return unless config_processor
+    builds_attrs = config_processor.builds_for_type_and_ref(job_type, ref, tag)
+    builds_attrs.map do |build_attrs|
       builds.create!({
         project: project,
         name: build_attrs[:name],
         commands: build_attrs[:script],
         tag_list: build_attrs[:tags],
         options: build_attrs[:options],
-        allow_failure: build_attrs[:allow_failure]
+        allow_failure: build_attrs[:allow_failure],
+        job_type: build_attrs[:type]
       })
     end
   end
+  def create_next_builds
+    return if skip_ci?
+    return unless config_processor
+
+    build_types = builds.group_by(&:job_type)
+
+    config_processor.types.any? do |job_type|
+      !build_types.include?(job_type) && create_builds_for_type(job_type).present?
+    end
+  end
+
+  def create_builds
+    return if skip_ci?
+    return unless config_processor
+
+    config_processor.types.any? do |job_type|
+      create_builds_for_type(job_type).present?
+    end
+  end
+
   def builds_without_retry
+    return unless config_processor
     @builds_without_retry ||=
       begin
+        job_types = config_processor.types
         grouped_builds = builds.group_by(&:name)
-        grouped_builds.map do |name, builds|
+        grouped_builds = grouped_builds.map do |name, builds|
           builds.sort_by(&:id).last
         end
+        grouped_builds.sort_by do |build|
+          [job_types.index(build.job_type), build.name]
+        end
       end
   end
   def retried_builds
-    @retried_builds ||= (builds - builds_without_retry)
-  end
-
-  def create_deploy_builds
-    return if skip_ci?
-
-    begin
-      deploy_builds_for_ref = config_processor.deploy_builds_for_ref(ref, tag)
-    rescue GitlabCiYamlProcessor::ValidationError => e
-      save_yaml_error(e.message) and return
-    rescue Exception => e
-      logger.error e.message + "\n" + e.backtrace.join("\n")
-      save_yaml_error("Undefined yaml error") and return
-    end
-
-    deploy_builds_for_ref.each do |build_attrs|
-      builds.create!({
-        project: project,
-        name: build_attrs[:name],
-        commands: build_attrs[:script],
-        tag_list: build_attrs[:tags],
-        options: build_attrs[:options],
-        allow_failure: build_attrs[:allow_failure],
-        deploy: true
-      })
-    end
+    @retried_builds ||= (builds.order(id: :desc) - builds_without_retry)
   end
   def status
@@ -225,6 +225,13 @@ class Commit < ActiveRecord::Base
   def config_processor
     @config_processor ||= GitlabCiYamlProcessor.new(push_data[:ci_yaml_file] || project.generated_yaml_config)
+  rescue GitlabCiYamlProcessor::ValidationError => e
+    save_yaml_error(e.message)
+    nil
+  rescue Exception => e
+    logger.error e.message + "\n" + e.backtrace.join("\n")
+    save_yaml_error("Undefined yaml error")
+    nil
   end
   def skip_ci?
@@ -235,6 +242,7 @@ class Commit < ActiveRecord::Base
   private
   def save_yaml_error(error)
+    return unless self.yaml_errors?
     self.yaml_errors = error
     save
   end

app/services/create_commit_service.rb

@@ -42,10 +42,6 @@ class CreateCommitService
     commit.create_builds unless commit.builds.any?
-    if commit.builds.empty?
-      commit.create_deploy_builds
-    end
-
     commit
   end
 end

app/views/builds/_build.html.haml

@@ -7,14 +7,8 @@
       %strong Build ##{build.id}
   %td
-    - if build.commit.tag?
-      Tag
-      ·
-      #{build.ref}
-    - else
-      Commit
-      ·
-      #{build.short_sha}
+    = build.job_type
+
   %td
     = build.name
     .pull-right

app/views/commits/_commit.html.haml

 %tr.build.alert{class: commit_status_alert_class(commit)}
   %td.status
     = commit.status
+    - if commit.running?
+      ·
+      = commit.job_type
+
   %td.build-link
     = link_to project_ref_commit_path(commit.project, commit.ref, commit.sha) do

app/views/commits/show.html.haml

@@ -68,7 +68,7 @@
     %tr
       %th Status
       %th Build ID
-      %th Trigger
+      %th Type
       %th Name
       %th Duration
       %th Finished at
@@ -86,7 +86,7 @@
       %tr
         %th Status
         %th Build ID
-        %th Trigger
+        %th Type
         %th Name
         %th Duration
         %th Finished at

app/views/lints/_create.html.haml

@@ -10,37 +10,23 @@
         %th Parameter
         %th Value
     %tbody
-      - @builds.each do |build|
-        %tr
-          %td Job - #{build[:name]}
-          %td
-            %pre
-              = simple_format build[:script]
-            
-            %b Tag list:
-            = build[:tags]
-            %br
-            %b Refs only:
-            = build[:only] && build[:only].join(", ")
-            %br
-            %b Refs except:
-            = build[:except] && build[:except].join(", ")
-      - @deploy_builds.each do |build|
-        %tr
-          %td Deploy Job - #{build[:name]}
-          %td
-            %pre
-              = simple_format build[:script]
-            
-            %b Tag list:
-            = build[:tags]
-            %br
-            %b Refs only:
-            = build[:only] && build[:only].join(", ")
-            %br
-            %b Refs except:
-            = build[:except] && build[:except].join(", ")
+      - @types.each do |type|
+        - @builds.select { |build| build[:type] == type }.each do |build|
+          %tr
+            %td #{type.capitalize} Job - #{build[:name]}
+            %td
+              %pre
+                = simple_format build[:script]
+              %br
+              %b Tag list:
+              = build[:tags]
+              %br
+              %b Refs only:
+              = build[:only] && build[:only].join(", ")
+              %br
+              %b Refs except:
+              = build[:except] && build[:except].join(", ")
 -else
   %p
@@ -49,5 +35,5 @@
     %i.icon-remove.incorrect-syntax
   %b Error:
   = @error
-          
+
     
\ No newline at end of file

db/migrate/20150710113836_add_job_type_to_builds.rb

+class AddJobTypeToBuilds < ActiveRecord::Migration
+  def change
+    add_column :builds, :job_type, :string
+  end
+end

db/migrate/20150710113851_migrate_deploy_to_job_type_for_builds.rb

+class MigrateDeployToJobTypeForBuilds < ActiveRecord::Migration
+  def up
+    execute("UPDATE builds SET job_type='test' WHERE NOT deploy")
+    execute("UPDATE builds SET job_type='deploy' WHERE deploy")
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema.define(version: 20150707134456) do
+ActiveRecord::Schema.define(version: 20150710113851) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -33,6 +33,7 @@ ActiveRecord::Schema.define(version: 20150707134456) do
     t.boolean  "deploy",        default: false
     t.text     "options"
     t.boolean  "allow_failure", default: false, null: false
+    t.string   "job_type"
   end
   add_index "builds", ["commit_id"], name: "index_builds_on_commit_id", using: :btree

lib/gitlab_ci_yaml_processor.rb

 class GitlabCiYamlProcessor
   class ValidationError < StandardError;end
+  DEFAULT_TYPES = %w(build test deploy)
+  DEFAULT_TYPE = 'test'
+  ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type]
+
   attr_reader :before_script, :image, :services
   def initialize(config)
@@ -17,12 +21,8 @@ class GitlabCiYamlProcessor
     validate!
   end
-  def deploy_builds_for_ref(ref, tag = false)
-    deploy_builds.select{|build| process?(build[:only], build[:except], ref, tag)}
-  end
-
-  def builds_for_ref(ref, tag = false)
-    builds.select{|build| process?(build[:only], build[:except], ref, tag)}
+  def builds_for_type_and_ref(type, ref, tag = false)
+    builds.select{|build| build[:type] == type && process?(build[:only], build[:except], ref, tag)}
   end
   def builds
@@ -31,10 +31,8 @@ class GitlabCiYamlProcessor
     end
   end
-  def deploy_builds
-    @deploy_jobs.map do |name, job|
-      build_job(name, job)
-    end
+  def types
+    @types || DEFAULT_TYPES
   end
   private
@@ -43,7 +41,8 @@ class GitlabCiYamlProcessor
     @before_script = @config[:before_script] || []
     @image = @config[:image]
     @services = @config[:services]
-    @config.except!(:before_script, :image, :services)
+    @types = @config[:types]
+    @config.except!(:before_script, :image, :services, :types)
     @config.each do |name, param|
       raise ValidationError, "Unknown parameter: #{name}" unless param.is_a?(Hash)
@@ -53,8 +52,10 @@ class GitlabCiYamlProcessor
       raise ValidationError, "Please define at least one job"
     end
-    @jobs = @config.select{|key, value| value[:type] != "deploy"}
-    @deploy_jobs = @config.select{|key, value| value[:type] == "deploy"}
+    @jobs = {}
+    @config.each do |key, job|
+      @jobs[key] = {type: DEFAULT_TYPE}.merge(job)
+    end
   end
   def process?(only_params, except_params, ref, tag)
@@ -79,6 +80,7 @@ class GitlabCiYamlProcessor
   def build_job(name, job)
     {
+      type: job[:type],
       script: "#{@before_script.join("\n")}\n#{normalize_script(job[:script])}",
       tags: job[:tags] || [],
       name: name,
@@ -121,12 +123,12 @@ class GitlabCiYamlProcessor
       raise ValidationError, "services should be an array of strings"
     end
-    @jobs.each do |name, job|
-      validate_job!("#{name} job", job)
+    unless @types.nil? || @types.is_a?(Array) && @types.all? {|type| type.is_a?(String)}
+      raise ValidationError, "types should be an array of strings"
     end
-    @deploy_jobs.each do |name, job|
-      validate_job!("#{name} deploy job", job)
+    @jobs.each do |name, job|
+      validate_job!("#{name} job", job)
     end
     true
@@ -134,11 +136,19 @@ class GitlabCiYamlProcessor
   def validate_job!(name, job)
     job.keys.each do |key|
-      unless [:tags, :script, :only, :except, :type, :image, :services, :allow_failure].include? key
+      unless ALLOWED_JOB_KEYS.include? key
         raise ValidationError, "#{name}: unknown parameter #{key}"
       end
     end
+    unless job[:type].is_a?(String)
+      raise ValidationError, "#{name}: type should be a string"
+    end
+
+    unless job[:type].in?(types)
+      raise ValidationError, "#{name}: type parameter should be #{types.join(", ")}"
+    end
+
     if job[:image] && !job[:image].is_a?(String)
       raise ValidationError, "#{name}: image should be a string"
     end

spec/lib/gitlab_ci_yaml_processor_spec.rb

@@ -3,6 +3,8 @@ require 'spec_helper'
 describe GitlabCiYamlProcessor do
   
   describe "#builds_for_ref" do
+    let (:type) { 'test' }
+
     it "returns builds if no branch specified" do
       config = YAML.dump({
         before_script: ["pwd"],
@@ -11,8 +13,9 @@ describe GitlabCiYamlProcessor do
       config_processor = GitlabCiYamlProcessor.new(config)
-      config_processor.builds_for_ref("master").size.should == 1
-      config_processor.builds_for_ref("master").first.should == {
+      config_processor.builds_for_type_and_ref(type, "master").size.should == 1
+      config_processor.builds_for_type_and_ref(type, "master").first.should == {
+        type: "test",
         except: nil,
         name: :rspec,
         only: nil,
@@ -31,7 +34,7 @@ describe GitlabCiYamlProcessor do
       config_processor = GitlabCiYamlProcessor.new(config)
-      config_processor.builds_for_ref("master").size.should == 0
+      config_processor.builds_for_type_and_ref(type, "master").size.should == 0
     end
     it "does not return builds if only has regexp with another branch" do
@@ -42,7 +45,7 @@ describe GitlabCiYamlProcessor do
       config_processor = GitlabCiYamlProcessor.new(config)
-      config_processor.builds_for_ref("master").size.should == 0
+      config_processor.builds_for_type_and_ref(type, "master").size.should == 0
     end
     it "returns builds if only has specified this branch" do
@@ -53,7 +56,7 @@ describe GitlabCiYamlProcessor do
       config_processor = GitlabCiYamlProcessor.new(config)
-      config_processor.builds_for_ref("master").size.should == 1
+      config_processor.builds_for_type_and_ref(type, "master").size.should == 1
     end
     it "does not build tags" do
@@ -64,73 +67,35 @@ describe GitlabCiYamlProcessor do
       config_processor = GitlabCiYamlProcessor.new(config)
-      config_processor.builds_for_ref("0-1", true).size.should == 0
+      config_processor.builds_for_type_and_ref(type, "0-1", true).size.should == 0
     end
-  end
-
-  describe "#deploy_builds_for_ref" do
-    it "returns builds if no branch specified" do
-      config = YAML.dump({
-        before_script: ["pwd"],
-        rspec: {script: "rspec", type: "deploy", allow_failure: true}
-      })
-
-      config_processor = GitlabCiYamlProcessor.new(config)
-      config_processor.deploy_builds_for_ref("master").size.should == 1
-      config_processor.deploy_builds_for_ref("master").first.should == {
-        except: nil,
-        name: :rspec,
-        only: nil,
-        script: "pwd\nrspec",
-        tags: [],
-        options: {},
-        allow_failure: true
-      }
-    end
-
-    it "does not return builds if only has another branch" do
-      config = YAML.dump({
-        before_script: ["pwd"],
-        rspec: {script: "rspec", type: "deploy", only: ["deploy"]}
-      })
-
-      config_processor = GitlabCiYamlProcessor.new(config)
-
-      config_processor.deploy_builds_for_ref("master").size.should == 0
-    end
-
-    it "does not return builds if only has regexp with another branch" do
+    it "returns builds if only has a list of branches including specified" do
       config = YAML.dump({
-        before_script: ["pwd"],
-        rspec: {script: "rspec", type: "deploy", only: ["/^deploy$/"]}
-      })
+                           before_script: ["pwd"],
+                           rspec: {script: "rspec", type: type, only: ["master", "deploy"]}
+                         })
       config_processor = GitlabCiYamlProcessor.new(config)
-      config_processor.deploy_builds_for_ref("master").size.should == 0
+      config_processor.builds_for_type_and_ref(type, "deploy").size.should == 1
     end
-    it "returns builds if only has specified this branch" do
-      config = YAML.dump({
-        before_script: ["pwd"],
-        rspec: {script: "rspec", type: "deploy", only: ["master"]}
-      })
-      config_processor = GitlabCiYamlProcessor.new(config)
-
-      config_processor.deploy_builds_for_ref("master").size.should == 1
-    end
-
-    it "returns builds if only has a list of branches including specified" do
+    it "returns build only for specified type" do
       config = YAML.dump({
-        before_script: ["pwd"],
-        rspec: {script: "rspec", type: "deploy", only: ["master", "deploy"]}
-      })
+                           before_script: ["pwd"],
+                           build: {script: "build", type: "build", only: ["master", "deploy"]},
+                           rspec: {script: "rspec", type: type, only: ["master", "deploy"]},
+                           staging: {script: "deploy", type: "deploy", only: ["master", "deploy"]},
+                           production: {script: "deploy", type: "deploy", only: ["master", "deploy"]},
+                         })
       config_processor = GitlabCiYamlProcessor.new(config)
-      config_processor.deploy_builds_for_ref("deploy").size.should == 1
+      config_processor.builds_for_type_and_ref("production", "deploy").size.should == 0
+      config_processor.builds_for_type_and_ref(type, "deploy").size.should == 1
+      config_processor.builds_for_type_and_ref("deploy", "deploy").size.should == 2
     end
   end
@@ -145,9 +110,10 @@ describe GitlabCiYamlProcessor do
       config_processor = GitlabCiYamlProcessor.new(config)
-      config_processor.builds_for_ref("master").size.should == 1
-      config_processor.builds_for_ref("master").first.should == {
+      config_processor.builds_for_type_and_ref("test", "master").size.should == 1
+      config_processor.builds_for_type_and_ref("test", "master").first.should == {
         except: nil,
+        type: "test",
         name: :rspec,
         only: nil,
         script: "pwd\nrspec",
@@ -170,9 +136,10 @@ describe GitlabCiYamlProcessor do
       config_processor = GitlabCiYamlProcessor.new(config)
-      config_processor.builds_for_ref("master").size.should == 1
-      config_processor.builds_for_ref("master").first.should == {
+      config_processor.builds_for_type_and_ref("test", "master").size.should == 1
+      config_processor.builds_for_type_and_ref("test", "master").first.should == {
         except: nil,
+        type: "test",
         name: :rspec,
         only: nil,
         script: "pwd\nrspec",
@@ -267,5 +234,40 @@ describe GitlabCiYamlProcessor do
         GitlabCiYamlProcessor.new(config)
       end.to raise_error(GitlabCiYamlProcessor::ValidationError, "rspec job: allow_failure parameter should be an boolean")
     end
+
+    it "returns errors if job type is not a string" do
+      config = YAML.dump({rspec: {script: "test", type: false, allow_failure: "string"}})
+      expect do
+        GitlabCiYamlProcessor.new(config)
+      end.to raise_error(GitlabCiYamlProcessor::ValidationError, "rspec job: type should be a string")
+    end
+
+    it "returns errors if job type is not a pre-defined type" do
+      config = YAML.dump({rspec: {script: "test", type: "acceptance", allow_failure: "string"}})
+      expect do
+        GitlabCiYamlProcessor.new(config)
+      end.to raise_error(GitlabCiYamlProcessor::ValidationError, "rspec job: type parameter should be build, test, deploy")
+    end
+
+    it "returns errors if job type is not a defined type" do
+      config = YAML.dump({types: ["build", "test"], rspec: {script: "test", type: "acceptance", allow_failure: "string"}})
+      expect do
+        GitlabCiYamlProcessor.new(config)
+      end.to raise_error(GitlabCiYamlProcessor::ValidationError, "rspec job: type parameter should be build, test")
+    end
+
+    it "returns errors if types is not an array" do
+      config = YAML.dump({types: "test", rspec: {script: "test"}})
+      expect do
+        GitlabCiYamlProcessor.new(config)
+      end.to raise_error(GitlabCiYamlProcessor::ValidationError, "types should be an array of strings")
+    end
+
+    it "returns errors if types is not an array of strings" do
+      config = YAML.dump({types: [true, "test"], rspec: {script: "test"}})
+      expect do
+        GitlabCiYamlProcessor.new(config)
+      end.to raise_error(GitlabCiYamlProcessor::ValidationError, "types should be an array of strings")
+    end
   end
 end
\ No newline at end of file

spec/models/commit_spec.rb

@@ -132,15 +132,24 @@ describe Commit do
     it { commit.sha.should start_with(subject) }
   end
-  describe "create_deploy_builds" do
-    it "creates deploy build" do
+  describe :create_next_builds do
+    it "creates builds for next type" do
       config_processor = GitlabCiYamlProcessor.new(gitlab_ci_yaml)
       commit.stub(:config_processor).and_return(config_processor)
-      commit.create_deploy_builds
+      commit.create_builds.should be_true
       commit.builds.reload
-
       commit.builds.size.should == 2
+
+      commit.create_next_builds.should be_true
+      commit.builds.reload
+      commit.builds.size.should == 4
+
+      commit.create_next_builds.should be_true
+      commit.builds.reload
+      commit.builds.size.should == 5
+
+      commit.create_next_builds.should be_false
     end
   end

spec/services/create_commit_service_spec.rb

@@ -23,32 +23,6 @@ describe CreateCommitService do
       it { commit.builds.first.should be_kind_of(Build) }
     end
-    context "deploy builds" do
-      it "calls create_deploy_builds if there are no builds" do
-        config = YAML.dump({production: {script: "ls", type: "deploy"}})
-        Commit.any_instance.should_receive(:create_deploy_builds)
-        service.execute(project,
-          ref: 'refs/heads/master',
-          before: '00000000',
-          after: '31das312',
-          ci_yaml_file: config,
-          commits: [ { message: "Message" } ]
-        )
-      end
-
-      it "does not call create_deploy_builds if there is build" do
-        config = YAML.dump({rspec: {script: "ls"},production: { script: "ls", type: "deploy"}})
-        Commit.any_instance.should_not_receive(:create_deploy_builds)
-        service.execute(project,
-          ref: 'refs/heads/master',
-          before: '00000000',
-          after: '31das312',
-          ci_yaml_file: config,
-          commits: [ { message: "Message" } ]
-        )
-      end
-    end
-
     context "skip tag if there is no build for it" do
       it "creates commit if there is appropriate job" do
         result = service.execute(project,

spec/support/gitlab_stubs/gitlab_ci.yml

@@ -7,6 +7,11 @@ before_script:
   - bundle install 
   - bundle exec rake db:create
+types:
+  - test
+  - deploy
+  - notify
+
 rspec:
   script: "rake spec"
   tags: 
@@ -43,4 +48,13 @@ production:
     - debian
   only:
     - master
-    - /^deploy-.*$/
\ No newline at end of file
+    - /^deploy-.*$/
+
+dockerhub:
+  type: notify
+  script: "curl http://dockerhub/URL"
+  tags:
+    - ruby
+    - postgres
+  only:
+    - branches