Implement Build Artifacts

Use X-File header for fast artifacts upload - requires gitlab-workhorse extension

CHANGELOG

@@ -82,6 +82,7 @@ v 8.1.0 (unreleased)
   - Show CI status on Your projects page and Starred projects page
   - Remove "Continuous Integration" page from dashboard
   - Add notes and SSL verification entries to hook APIs (Ben Boeckel)
+  - Added build artifacts
   - Fix grammar in admin area "labels" .nothing-here-block when no labels exist.
   - Move CI runners page to project settings area
   - Move CI variables page to project settings area

Gemfile

@@ -57,7 +57,7 @@ gem 'gollum-lib', '~> 4.0.2'
 gem "gitlab-linguist", "~> 3.0.1", require: "linguist"
 # API
-gem 'grape',        '~> 0.6.1'
+gem 'grape',        '~> 0.13.0'
 gem 'grape-entity', '~> 0.4.2'
 gem 'rack-cors',    '~> 0.4.0', require: 'rack/cors'

Gemfile.lock

@@ -306,10 +306,10 @@ GEM
     gon (5.0.4)
       actionpack (>= 2.3.0)
       json
-    grape (0.6.1)
+    grape (0.13.0)
       activesupport
       builder
-      hashie (>= 1.2.0)
+      hashie (>= 2.1.0)
       multi_json (>= 1.3.2)
       multi_xml (>= 0.5.2)
       rack (>= 1.3.0)
@@ -834,7 +834,7 @@ DEPENDENCIES
   gitlab_omniauth-ldap (~> 1.2.1)
   gollum-lib (~> 4.0.2)
   gon (~> 5.0.0)
-  grape (~> 0.6.1)
+  grape (~> 0.13.0)
   grape-entity (~> 0.4.2)
   haml-rails (~> 0.9.0)
   hipchat (~> 1.5.0)

app/controllers/ci/admin/application_settings_controller.rb

@@ -25,6 +25,7 @@ module Ci
       params.require(:application_setting).permit(
         :all_broken_builds,
         :add_pusher,
+        :max_artifact_size,
       )
     end
   end

app/controllers/projects/builds_controller.rb

@@ -3,6 +3,7 @@ class Projects::BuildsController < Projects::ApplicationController
   before_action :build, except: [:index, :cancel_all]
   before_action :authorize_manage_builds!, except: [:index, :show, :status]
+  before_action :authorize_download_build_artifacts!, only: [:download]
   layout "project"
@@ -55,6 +56,18 @@ class Projects::BuildsController < Projects::ApplicationController
     end
   end
+  def download
+    unless build.artifact_file.file_storage?
+      return redirect_to build.artifact_file.url
+    end
+
+    unless build.artifact_file.exists?
+      return not_found!
+    end
+
+    send_file build.artifact_file.path, disposition: 'attachment'
+  end
+
   def status
     render json: @build.to_json(only: [:status, :id, :sha, :coverage], methods: :sha)
   end
@@ -80,4 +93,14 @@ class Projects::BuildsController < Projects::ApplicationController
       return page_404
     end
   end
+
+  def authorize_download_build_artifacts!
+    unless can?(current_user, :download_build_artifacts, @project)
+      if current_user.nil?
+        return authenticate_user!
+      else
+        return render_404
+      end
+    end
+  end
 end

app/models/ability.rb

@@ -154,6 +154,7 @@ class Ability
         :create_merge_request,
         :create_wiki,
         :manage_builds,
+        :download_build_artifacts,
         :push_code
       ]
     end

app/models/ci/application_setting.rb

@@ -21,6 +21,7 @@ module Ci
       create(
         all_broken_builds: Settings.gitlab_ci['all_broken_builds'],
         add_pusher: Settings.gitlab_ci['add_pusher'],
+        max_artifact_size: Settings.gitlab_ci['max_artifact_size'],
       )
     end
   end

app/models/ci/build.rb

@@ -27,6 +27,8 @@ module Ci
   class Build < CommitStatus
     LAZY_ATTRIBUTES = ['trace']
+    include Ci::ApiAccess
+
     belongs_to :runner, class_name: 'Ci::Runner'
     belongs_to :trigger_request, class_name: 'Ci::TriggerRequest'
@@ -39,6 +41,8 @@ module Ci
     scope :ignore_failures, ->() { where(allow_failure: false) }
     scope :similar, ->(build) { where(ref: build.ref, tag: build.tag, trigger_request_id: build.trigger_request_id) }
+    mount_uploader :artifact_file, ArtifactUploader
+
     acts_as_taggable
     # To prevent db load megabytes of data from trace
@@ -209,6 +213,10 @@ module Ci
       "#{dir_to_trace}/#{id}.log"
     end
+    def valid_token? token
+      project.valid_token? token
+    end
+
     def target_url
       Gitlab::Application.routes.url_helpers.
         namespace_project_build_url(gl_project.namespace, gl_project, self)
@@ -240,6 +248,17 @@ module Ci
       pending? && !any_runners_online?
     end
+    def download_url
+      if artifact_file.exists?
+        Gitlab::Application.routes.url_helpers.
+          download_namespace_project_build_path(gl_project.namespace, gl_project, self)
+      end
+    end
+
+    def artifacts_upload_url
+      build_artifact_upload_url(id, project.token)
+    end
+
     private
     def yaml_variables
@@ -273,6 +292,7 @@ module Ci
       variables << { key: :CI_BUILD_TAG, value: ref, public: true } if tag?
       variables << { key: :CI_BUILD_NAME, value: name, public: true }
       variables << { key: :CI_BUILD_STAGE, value: stage, public: true }
+      variables << { key: :CI_BUILD_ARTIFACTS_UPLOAD_URL, value: artifacts_upload_url, public: false }
       variables << { key: :CI_BUILD_TRIGGERED, value: 'true', public: true } if trigger_request
       variables
     end

app/models/commit_status.rb

@@ -92,4 +92,8 @@ class CommitStatus < ActiveRecord::Base
   def show_warning?
     false
   end
+
+  def download_url
+    nil
+  end
 end

app/uploaders/artifact_uploader.rb

+# encoding: utf-8
+class ArtifactUploader < CarrierWave::Uploader::Base
+  storage :file
+
+  attr_accessor :build, :field
+
+  def initialize(build, field)
+    @build, @field = build, field
+  end
+
+  def base_dir
+    Settings.gitlab_ci.artifacts_path
+  end
+
+  def artifacts_path
+    File.join(build.created_at.utc.strftime('%Y_%m'), build.project.id.to_s, build.id.to_s)
+  end
+
+  def store_dir
+    File.join(base_dir, artifacts_path)
+  end
+
+  def cache_dir
+    File.join(base_dir, 'tmp-cache', artifacts_path)
+  end
+
+  def file_storage?
+    self.class.storage == CarrierWave::Storage::File
+  end
+
+  def exists?
+    file.try(:exists?)
+  end
+
+  def move_to_cache
+    true
+  end
+
+  def move_to_store
+    true
+  end
+end

app/views/ci/admin/application_settings/_form.html.haml

@@ -20,5 +20,12 @@
             = f.check_box :add_pusher
             Add pusher to recipients list
+  %fieldset
+    %legend Build artifacts
+    .form-group
+      = f.label :max_artifact_size, 'Maximum artifact size (MB)', class: 'control-label col-sm-2'
+      .col-sm-10
+        = f.number_field :max_artifact_size, class: 'form-control'
+
   .form-actions
     = f.submit 'Save', class: 'btn btn-primary'

app/views/projects/builds/show.html.haml

@@ -84,6 +84,9 @@
             Test coverage
           %h1 #{@build.coverage}%
+      - if current_user && can?(current_user, :download_build_artifacts, @project) && @build.download_url
+        .build-widget.center
+          = link_to "Download artifacts", @build.download_url, class: 'btn btn-sm btn-primary'
       .build-widget
         %h4.title

app/views/projects/commit_statuses/_commit_status.html.haml

@@ -45,7 +45,10 @@
   %td
     .pull-right
-      - if current_user && can?(current_user, :manage_builds, commit_status.gl_project)
+      - if current_user && can?(current_user, :download_build_artifacts, @project) && commit_status.download_url
+        = link_to commit_status.download_url, title: 'Download artifacts' do
+          %i.fa.fa-download
+      - if current_user && can?(current_user, :manage_builds, @project)
         - if commit_status.cancel_url
           = link_to commit_status.cancel_url, title: 'Cancel' do
             %i.fa.fa-remove.cred

config/initializers/1_settings.rb

@@ -185,6 +185,9 @@ Settings.gitlab_ci['all_broken_builds']   = true if Settings.gitlab_ci['all_brok
 Settings.gitlab_ci['add_pusher']          = false if Settings.gitlab_ci['add_pusher'].nil?
 Settings.gitlab_ci['url']                 ||= Settings.send(:build_gitlab_ci_url)
 Settings.gitlab_ci['builds_path']         = File.expand_path(Settings.gitlab_ci['builds_path'] || "builds/", Rails.root)
+Settings.gitlab_ci['artifacts_path']      = File.expand_path('shared/artifacts/', Rails.root)
+Settings.gitlab_ci['max_artifact_size']   ||= 100
+
 #
 # Reply by email

config/routes.rb

@@ -594,6 +594,7 @@ Gitlab::Application.routes.draw do
           member do
             get :cancel
             get :status
+            get :download
             post :retry
           end
         end

db/migrate/20151013092124_add_artifacts_file_to_builds.rb

+class AddArtifactsFileToBuilds < ActiveRecord::Migration
+  def change
+    add_column :ci_builds, :artifact_file, :text
+  end
+end

db/migrate/20151015132538_add_max_artifact_size_to_ci_application_settings.rb

+class AddMaxArtifactSizeToCiApplicationSettings < ActiveRecord::Migration
+  def change
+    add_column :ci_application_settings, :max_artifact_size, :integer, default: 100, null: false
+  end
+end

db/schema.rb

@@ -79,6 +79,7 @@ ActiveRecord::Schema.define(version: 20151026182941) do
     t.boolean  "add_pusher"
     t.datetime "created_at"
     t.datetime "updated_at"
+    t.integer  "max_artifact_size", default: 100, null: false
   end
   create_table "ci_builds", force: true do |t|
@@ -107,6 +108,7 @@ ActiveRecord::Schema.define(version: 20151026182941) do
     t.string   "type"
     t.string   "target_url"
     t.string   "description"
+    t.text     "artifact_file"
   end
   add_index "ci_builds", ["commit_id", "stage_idx", "created_at"], name: "index_ci_builds_on_commit_id_and_stage_idx_and_created_at", using: :btree

doc/install/installation.md

@@ -246,6 +246,11 @@ We recommend using a PostgreSQL database. For MySQL check [MySQL setup guide](da
     # Change the permissions of the directory where CI build traces are stored
     sudo chmod -R u+rwX builds/
+    # Change the permissions of the directory where CI artifacts are stored
+    sudo chmod -R u+rwX artifacts/
+    sudo chmod -R ug+rwX artifacts/tmp-uploads
+    sudo chown -R git:www-data artifacts/tmp-uploads
+
     # Copy the example Unicorn config
     sudo -u git -H cp config/unicorn.rb.example config/unicorn.rb
@@ -392,7 +397,7 @@ Check if GitLab and its environment are configured correctly:
 ### Installation
-    sudo apt-get install -y nginx
+    sudo apt-get install -y nginx-extras
 ### Site Configuration

lib/api/helpers.rb

@@ -234,6 +234,10 @@ module API
       render_api_error!(message || '409 Conflict', 409)
     end
+    def file_to_large!
+      render_api_error!('413 Request Entity Too Large', 413)
+    end
+
     def render_validation_error!(model)
       if model.errors.any?
         render_api_error!(model.errors.messages || '400 Bad Request', 400)
@@ -282,6 +286,16 @@ module API
       end
     end
+    # file helpers
+
+    def present_file!(path, filename, content_type = 'application/octet-stream')
+      filename ||= File.basename(path)
+      header['Content-Disposition'] = "attachment; filename=#{filename}"
+      header['Content-Transfer-Encoding'] = 'binary'
+      content_type content_type
+      file FileStreamer.new(path)
+    end
+
     private
     def add_pagination_headers(paginated, per_page)