Implement Build Artifacts

- Offloads uploading to GitLab Workhorse
- Use /authorize request for fast uploading
- Added backup recipes for artifacts
- Support download acceleration using X-Sendfile

.gitignore

@@ -37,6 +37,9 @@ nohup.out
 public/assets/
 public/uploads.*
 public/uploads/
+shared/artifacts/
+shared/tmp/artifacts-uploads/
+shared/tmp/artifacts-cache/
 rails_best_practices_output.html
 /tags
 tmp/

CHANGELOG

@@ -95,6 +95,7 @@ v 8.1.0 (unreleased)
   - Show CI status on Your projects page and Starred projects page
   - Remove "Continuous Integration" page from dashboard
   - Add notes and SSL verification entries to hook APIs (Ben Boeckel)
+  - Added build artifacts
   - Fix grammar in admin area "labels" .nothing-here-block when no labels exist.
   - Move CI runners page to project settings area
   - Move CI variables page to project settings area

Gemfile

@@ -54,7 +54,7 @@ gem 'gollum-lib', '~> 4.0.2'
 gem "github-linguist", "~> 4.7.0", require: "linguist"
 # API
-gem 'grape',        '~> 0.6.1'
+gem 'grape',        '~> 0.13.0'
 gem 'grape-entity', '~> 0.4.2'
 gem 'rack-cors',    '~> 0.4.0', require: 'rack/cors'

Gemfile.lock

@@ -306,10 +306,10 @@ GEM
     gon (5.0.4)
       actionpack (>= 2.3.0)
       json
-    grape (0.6.1)
+    grape (0.13.0)
       activesupport
       builder
-      hashie (>= 1.2.0)
+      hashie (>= 2.1.0)
       multi_json (>= 1.3.2)
       multi_xml (>= 0.5.2)
       rack (>= 1.3.0)
@@ -829,7 +829,7 @@ DEPENDENCIES
   gitlab_omniauth-ldap (~> 1.2.1)
   gollum-lib (~> 4.0.2)
   gon (~> 5.0.0)
-  grape (~> 0.6.1)
+  grape (~> 0.13.0)
   grape-entity (~> 0.4.2)
   haml-rails (~> 0.9.0)
   hipchat (~> 1.5.0)

app/controllers/admin/application_settings_controller.rb

@@ -58,6 +58,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
       :admin_notification_email,
       :user_oauth_applications,
       :shared_runners_enabled,
+      :max_artifacts_size,
       restricted_visibility_levels: [],
       import_sources: []
     )

app/controllers/projects/builds_controller.rb

@@ -3,6 +3,7 @@ class Projects::BuildsController < Projects::ApplicationController
   before_action :build, except: [:index, :cancel_all]
   before_action :authorize_manage_builds!, except: [:index, :show, :status]
+  before_action :authorize_download_build_artifacts!, only: [:download]
   layout "project"
@@ -51,6 +52,18 @@ class Projects::BuildsController < Projects::ApplicationController
     redirect_to build_path(build)
   end
+  def download
+    unless artifacts_file.file_storage?
+      return redirect_to artifacts_file.url
+    end
+
+    unless artifacts_file.exists?
+      return not_found!
+    end
+
+    send_file artifacts_file.path, disposition: 'attachment'
+  end
+
   def status
     render json: @build.to_json(only: [:status, :id, :sha, :coverage], methods: :sha)
   end
@@ -67,6 +80,10 @@ class Projects::BuildsController < Projects::ApplicationController
     @build ||= ci_project.builds.unscoped.find_by!(id: params[:id])
   end
+  def artifacts_file
+    build.artifacts_file
+  end
+
   def build_path(build)
     namespace_project_build_path(build.gl_project.namespace, build.gl_project, build)
   end
@@ -76,4 +93,14 @@ class Projects::BuildsController < Projects::ApplicationController
       return page_404
     end
   end
+
+  def authorize_download_build_artifacts!
+    unless can?(current_user, :download_build_artifacts, @project)
+      if current_user.nil?
+        return authenticate_user!
+      else
+        return render_404
+      end
+    end
+  end
 end

app/models/ability.rb

@@ -154,6 +154,7 @@ class Ability
         :create_merge_request,
         :create_wiki,
         :manage_builds,
+        :download_build_artifacts,
         :push_code
       ]
     end

app/models/application_setting.rb

@@ -89,6 +89,7 @@ class ApplicationSetting < ActiveRecord::Base
       restricted_signup_domains: Settings.gitlab['restricted_signup_domains'],
       import_sources: ['github','bitbucket','gitlab','gitorious','google_code','fogbugz','git'],
       shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'],
+      max_artifacts_size: Settings.gitlab_ci['max_artifacts_size'],
     )
   end

app/models/ci/build.rb

@@ -39,6 +39,8 @@ module Ci
     scope :ignore_failures, ->() { where(allow_failure: false) }
     scope :similar, ->(build) { where(ref: build.ref, tag: build.tag, trigger_request_id: build.trigger_request_id) }
+    mount_uploader :artifacts_file, ArtifactUploader
+
     acts_as_taggable
     # To prevent db load megabytes of data from trace
@@ -217,6 +219,14 @@ module Ci
       "#{dir_to_trace}/#{id}.log"
     end
+    def token
+      project.token
+    end
+
+    def valid_token? token
+      project.valid_token? token
+    end
+
     def target_url
       Gitlab::Application.routes.url_helpers.
         namespace_project_build_url(gl_project.namespace, gl_project, self)
@@ -248,6 +258,13 @@ module Ci
       pending? && !any_runners_online?
     end
+    def download_url
+      if artifacts_file.exists?
+        Gitlab::Application.routes.url_helpers.
+          download_namespace_project_build_path(gl_project.namespace, gl_project, self)
+      end
+    end
+
     private
     def yaml_variables

app/models/commit_status.rb

@@ -92,4 +92,8 @@ class CommitStatus < ActiveRecord::Base
   def show_warning?
     false
   end
+
+  def download_url
+    nil
+  end
 end

app/uploaders/artifact_uploader.rb

+# encoding: utf-8
+class ArtifactUploader < CarrierWave::Uploader::Base
+  storage :file
+
+  attr_accessor :build, :field
+
+  def self.artifacts_path
+    File.expand_path('shared/artifacts/', Rails.root)
+  end
+
+  def self.artifacts_upload_path
+    File.expand_path('shared/tmp/artifacts-uploads/', Rails.root)
+  end
+
+  def self.artifacts_cache_path
+    File.expand_path('shared/tmp/artifacts-cache/', Rails.root)
+  end
+
+  def initialize(build, field)
+    @build, @field = build, field
+  end
+
+  def artifacts_path
+    File.join(build.created_at.utc.strftime('%Y_%m'), build.project.id.to_s, build.id.to_s)
+  end
+
+  def store_dir
+    File.join(ArtifactUploader.artifacts_path, artifacts_path)
+  end
+
+  def cache_dir
+    File.join(ArtifactUploader.artifacts_cache_path, artifacts_path)
+  end
+
+  def file_storage?
+    self.class.storage == CarrierWave::Storage::File
+  end
+
+  def exists?
+    file.try(:exists?)
+  end
+
+  def move_to_cache
+    true
+  end
+
+  def move_to_store
+    true
+  end
+end

app/views/admin/application_settings/_form.html.haml

@@ -139,5 +139,10 @@
             = f.check_box :shared_runners_enabled
             Enable shared runners for a new projects
+    .form-group
+      = f.label :max_artifacts_size, 'Maximum artifacts size (MB)', class: 'control-label col-sm-2'
+      .col-sm-10
+        = f.number_field :max_artifacts_size, class: 'form-control'
+
   .form-actions
     = f.submit 'Save', class: 'btn btn-primary'

app/views/projects/builds/show.html.haml

@@ -87,6 +87,9 @@
             Test coverage
           %h1 #{@build.coverage}%
+      - if current_user && can?(current_user, :download_build_artifacts, @project) && @build.download_url
+        .build-widget.center
+          = link_to "Download artifacts", @build.download_url, class: 'btn btn-sm btn-primary'
       .build-widget
         %h4.title

app/views/projects/commit_statuses/_commit_status.html.haml

@@ -61,6 +61,9 @@
   %td
     .pull-right
+      - if current_user && can?(current_user, :download_build_artifacts, @project) && commit_status.download_url
+        = link_to commit_status.download_url, title: 'Download artifacts' do
+          %i.fa.fa-download
       - if current_user && can?(current_user, :manage_builds, commit_status.gl_project)
         - if commit_status.active?
           - if commit_status.cancel_url

config/initializers/1_settings.rb

@@ -186,6 +186,7 @@ Settings.gitlab_ci['all_broken_builds']     = true if Settings.gitlab_ci['all_br
 Settings.gitlab_ci['add_pusher']            = false if Settings.gitlab_ci['add_pusher'].nil?
 Settings.gitlab_ci['url']                   ||= Settings.send(:build_gitlab_ci_url)
 Settings.gitlab_ci['builds_path']           = File.expand_path(Settings.gitlab_ci['builds_path'] || "builds/", Rails.root)
+Settings.gitlab_ci['max_artifacts_size']    ||= 100
 #
 # Reply by email

config/routes.rb

@@ -611,6 +611,7 @@ Gitlab::Application.routes.draw do
           member do
             get :status
             post :cancel
+            get :download
             post :retry
           end
         end

db/migrate/20151013092124_add_artifacts_file_to_builds.rb

+class AddArtifactsFileToBuilds < ActiveRecord::Migration
+  def change
+    add_column :ci_builds, :artifacts_file, :text
+  end
+end

db/migrate/20151109100728_add_max_artifacts_size_to_application_settings.rb

+class AddMaxArtifactsSizeToApplicationSettings < ActiveRecord::Migration
+  def change
+    add_column :application_settings, :max_artifacts_size, :integer, default: 100, null: false
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema.define(version: 20151105094515) do
+ActiveRecord::Schema.define(version: 20151109100728) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -48,6 +48,7 @@ ActiveRecord::Schema.define(version: 20151105094515) do
     t.text     "help_page_text"
     t.string   "admin_notification_email"
     t.boolean  "shared_runners_enabled",       default: true,  null: false
+    t.integer  "max_artifacts_size",           default: 100,   null: false
   end
   create_table "audit_events", force: true do |t|
@@ -108,6 +109,7 @@ ActiveRecord::Schema.define(version: 20151105094515) do
     t.string   "type"
     t.string   "target_url"
     t.string   "description"
+    t.text     "artifacts_file"
   end
   add_index "ci_builds", ["commit_id", "stage_idx", "created_at"], name: "index_ci_builds_on_commit_id_and_stage_idx_and_created_at", using: :btree

doc/ci/yaml/README.md

@@ -141,6 +141,7 @@ job_name:
 | tags          | optional | Defines a list of tags which are used to select runner |
 | allow_failure | optional | Allow build to fail. Failed build doesn't contribute to commit status |
 | when          | optional | Define when to run build. Can be `on_success`, `on_failure` or `always` |
+| artifacts     | optional | Define list build artifacts |
 ### script
 `script` is a shell script which is executed by runner. The shell script is prepended with `before_script`.
@@ -258,6 +259,20 @@ The above script will:
 1. Execute `cleanup_build` only when the `build` failed,
 2. Always execute `cleanup` as the last step in pipeline.
+### artifacts
+`artifacts` is used to specify list of files and directories which should be attached to build after success.
+
+```
+artifacts:
+- binaries/
+- .config
+```
+
+The above definition will archive all files in `binaries/` and `.config`.
+The artifacts will be send after the build success to GitLab and will be accessible in GitLab interface to download.
+
+This feature requires GitLab Runner v 0.7.0.
+
 ## Validate the .gitlab-ci.yml
 Each instance of GitLab CI has an embedded debug tool called Lint.
 You can find the link to the Lint in the project's settings page or use short url `/lint`.