Add S3 artifacts handling

Former-commit-id: 8428b401

Gemfile

@@ -85,14 +85,14 @@ gem 'kaminari', '~> 0.17.0'
 gem 'hamlit', '~> 2.6.1'
 # Files attachments
-gem 'carrierwave', '~> 0.11.0'
+gem 'carrierwave', '~> 1.0'
 # Drag and Drop UI
 gem 'dropzonejs-rails', '~> 0.7.1'
 # for backups
-gem 'fog-aws', '~> 0.9'
-gem 'fog-core', '~> 1.40'
+gem 'fog-aws', '~> 1.3.0'
+gem 'fog-core', '~> 1.42.0'
 gem 'fog-google', '~> 0.5'
 gem 'fog-local', '~> 0.3'
 gem 'fog-openstack', '~> 0.1'

Gemfile.lock

@@ -105,12 +105,10 @@ GEM
     capybara-screenshot (1.0.14)
       capybara (>= 1.0, < 3)
       launchy
-    carrierwave (0.11.2)
-      activemodel (>= 3.2.0)
-      activesupport (>= 3.2.0)
-      json (>= 1.7)
+    carrierwave (1.0.0)
+      activemodel (>= 4.0.0)
+      activesupport (>= 4.0.0)
       mime-types (>= 1.16)
-      mimemagic (>= 0.3.0)
     cause (0.1)
     charlock_holmes (0.7.3)
     chronic (0.10.2)
@@ -210,7 +208,7 @@ GEM
     flowdock (0.7.1)
       httparty (~> 0.7)
       multi_json
-    fog-aws (0.11.0)
+    fog-aws (1.3.0)
       fog-core (~> 1.38)
       fog-json (~> 1.0)
       fog-xml (~> 0.1)
@@ -871,7 +869,7 @@ DEPENDENCIES
   bundler-audit (~> 0.5.0)
   capybara (~> 2.6.2)
   capybara-screenshot (~> 1.0.0)
-  carrierwave (~> 0.11.0)
+  carrierwave (~> 1.0)
   charlock_holmes (~> 0.7.3)
   chronic (~> 0.10.2)
   chronic_duration (~> 0.10.6)

app/models/ci/build.rb

@@ -401,6 +401,8 @@ module Ci
     def steps
       [Gitlab::Ci::Build::Step.from_commands(self),
        Gitlab::Ci::Build::Step.from_after_script(self)].compact
+    rescue
+       []
     end
     def image

app/uploaders/artifact_uploader.rb

@@ -4,29 +4,69 @@ class ArtifactUploader < GitlabUploader
   storage_options Gitlab.config.artifacts
   def self.artifacts_path
-    @storage_options.path
+    if object_store_options.enabled
+      ""
+    else
+      storage_options.path + "/"
+    end
+  end
+
+  def artifacts_path
+    self.class.artifacts_path
   end
   def self.artifacts_upload_path
-    File.join(self.artifacts_path, 'tmp/uploads/')
+    self.artifacts_path + 'tmp/uploads'
   end
   def self.artifacts_cache_path
-    File.join(self.artifacts_path, 'tmp/cache/')
+    self.artifacts_path + 'tmp/cache'
   end
   attr_accessor :job, :field
+  def self.object_store_options
+    Gitlab.config.artifacts.object_store
+  end
+
+  if object_store_options.enabled
+    storage :fog
+    #cache_storage :fog
+  else
+    storage :file
+    #cache_storage :file
+  end
+
   def initialize(job, field)
     @job, @field = job, field
   end
   def store_dir
-    File.join(self.class.artifacts_path, job.artifacts_path)
+    self.class.artifacts_path + job.artifacts_path
   end
   def cache_dir
-    File.join(self.class.artifacts_cache_path, job.artifacts_path)
+    self.class.artifacts_cache_path + job.artifacts_path
+  end
+
+  def fog_directory
+    return super unless use_object_store?
+
+    self.class.object_store_options.bucket
+  end
+
+  # Override the credentials
+  def fog_credentials
+    return super unless use_object_store?
+
+    {
+      provider:              object_store_options.provider,
+      aws_access_key_id:     object_store_options.access_key_id,
+      aws_secret_access_key: object_store_options.secret_access_key,
+      region:                object_store_options.region,
+      endpoint:              object_store_options.endpoint,
+      path_style:            true
+    }
   end
   def filename
@@ -36,4 +76,40 @@ class ArtifactUploader < GitlabUploader
   def exists?
     file.try(:exists?)
   end
+
+  def fog_public
+    false
+  end
+
+  def upload_authorize
+    result = { TempPath: ArtifactUploader.artifacts_upload_path }
+
+    if use_object_store?
+      path = File.join('tmp', 'cache', 'upload', SecureRandom.hex)
+      expire_at = ::Fog::Time.now + fog_authenticated_url_expiration
+      result[:UploadPath] = path
+      result[:UploadURL] = storage.connection.put_object_url(
+        fog_directory, path, expire_at)
+    end
+
+    result
+  end
+
+  def retrive_uploaded!(path)
+    CarrierWave::Storage::Fog::File.new(self, storage, path)
+  end
+
+  def upload_cache_path(path = nil)
+    File.join(cache_dir, path)
+  end
+
+  private
+
+  def object_store_options
+    self.class.object_store_options
+  end
+
+  def use_object_store?
+    object_store_options.enabled
+  end
 end

config/initializers/carrierwave.rb

@@ -2,16 +2,19 @@ CarrierWave::SanitizedFile.sanitize_regexp = /[^[:word:]\.\-\+]/
 aws_file = Rails.root.join('config', 'aws.yml')
+CarrierWave.configure do |config|
+  config.fog_provider = "fog/rackspace/storage"
+  config.fog_credentials = {
+    provider: 'AWS',                                        # required
+    aws_access_key_id:     'ddd',
+    aws_secret_access_key: 'ccc',
+  }
+end
+
 if File.exist?(aws_file)
   AWS_CONFIG = YAML.load(File.read(aws_file))[Rails.env]
   CarrierWave.configure do |config|
-    config.fog_credentials = {
-      provider: 'AWS',                                        # required
-      aws_access_key_id: AWS_CONFIG['access_key_id'],         # required
-      aws_secret_access_key: AWS_CONFIG['secret_access_key'], # required
-      region: AWS_CONFIG['region'],                           # optional, defaults to 'us-east-1'
-    }
     # required
     config.fog_directory = AWS_CONFIG['bucket']

lib/api/helpers.rb

@@ -283,7 +283,7 @@ module API
     # file helpers
-    def uploaded_file(field, uploads_path)
+    def uploaded_file(uploader, field)
       if params[field]
         bad_request!("#{field} is not a file") unless params[field].respond_to?(:filename)
         return params[field]
@@ -294,10 +294,17 @@ module API
       # sanitize file paths
       # this requires all paths to exist
       required_attributes! %W(#{field}.path)
-      uploads_path = File.realpath(uploads_path)
+      uploads_path = File.realpath(uploader.artifacts_path)
       file_path = File.realpath(params["#{field}.path"])
       bad_request!('Bad file path') unless file_path.start_with?(uploads_path)
+      # if file was uploaded, retrieve the cached filed
+      if params["#{field}.upload_path"]
+        file = uploader.retrive_uploaded!(params["#{field}.upload_path"])
+
+        return file
+      end
+
       UploadedFile.new(
         file_path,
         params["#{field}.name"],

lib/api/runner.rb

@@ -181,7 +181,7 @@ module API
         status 200
         content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE
-        Gitlab::Workhorse.artifact_upload_ok
+        job.artifacts_file.upload_authorize
       end
       desc 'Upload artifacts for job' do
@@ -199,6 +199,7 @@ module API
         optional :file, type: File, desc: %q(Artifact's file)
         optional 'file.path', type: String, desc: %q(path to locally stored body (generated by Workhorse))
         optional 'file.name', type: String, desc: %q(real filename as send in Content-Disposition (generated by Workhorse))
+        optional 'file.upload_path', type: String, desc: %q(virtual cache upload_path as send by authorize (generated by Workhorse))
         optional 'file.type', type: String, desc: %q(real content type as send in Content-Type (generated by Workhorse))
         optional 'metadata.path', type: String, desc: %q(path to locally stored body (generated by Workhorse))
         optional 'metadata.name', type: String, desc: %q(filename (generated by Workhorse))
@@ -210,9 +211,8 @@ module API
         job = authenticate_job!
         forbidden!('Job is not running!') unless job.running?
-        artifacts_upload_path = ArtifactUploader.artifacts_upload_path
-        artifacts = uploaded_file(:file, artifacts_upload_path)
-        metadata = uploaded_file(:metadata, artifacts_upload_path)
+        artifacts = uploaded_file(job.artifacts_file, :file)
+        metadata = uploaded_file(job.artifacts_metadata, :metadata)
         bad_request!('Missing artifacts file!') unless artifacts
         file_to_large! unless artifacts.size < max_artifacts_size
@@ -223,7 +223,8 @@ module API
           Gitlab::CurrentSettings.current_application_settings.default_artifacts_expire_in
         if job.save
-          present job, with: Entities::JobRequest::Response
+          status :created
+          #present job, with: Entities::JobRequest::Response
         else
           render_validation_error!(job)
         end

lib/ci/api/builds.rb

@@ -124,7 +124,7 @@ module Ci
           status 200
           content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE
-          Gitlab::Workhorse.artifact_upload_ok
+          build.artifacts_file.upload_authorize
         end
         # Upload artifacts to build - Runners only
@@ -153,9 +153,8 @@ module Ci
           build = authenticate_build!
           forbidden!('Build is not running!') unless build.running?
-          artifacts_upload_path = ArtifactUploader.artifacts_upload_path
-          artifacts = uploaded_file(:file, artifacts_upload_path)
-          metadata = uploaded_file(:metadata, artifacts_upload_path)
+          artifacts = uploaded_file(build.artifacts_file, :file)
+          metadata = uploaded_file(build.artifacts_metadata, :metadata)
           bad_request!('Missing artifacts file!') unless artifacts
           file_to_large! unless artifacts.size < max_artifacts_size