From bf7315cb13eab0cd1427efe29ce110a963b63c34 Mon Sep 17 00:00:00 2001
From: Jeroen Nijhof <jeroen@jeroennijhof.nl>
Date: Tue, 9 Jun 2015 17:56:37 +0200
Subject: [PATCH] update fog to 1.25.0 and add multipart upload support
---
Gemfile | 2 +-
Gemfile.lock | 82 +++++++++++++++++++++++++------
config/gitlab.yml.example | 4 ++
config/initializers/1_settings.rb | 1 +
lib/backup/manager.rb | 3 +-
5 files changed, 75 insertions(+), 17 deletions(-)
diff --git a/Gemfile b/Gemfile
index 0c1fff0bc14..6b33d7c6fd0 100644
--- a/Gemfile
+++ b/Gemfile
@@ -81,7 +81,7 @@ gem "carrierwave"
gem 'dropzonejs-rails'
# for aws storage
-gem "fog", "~> 1.14"
+gem "fog", "~> 1.25.0"
gem "unf"
# Authorization
diff --git a/Gemfile.lock b/Gemfile.lock
index 9d87de7d4e0..3bd2d1a835d 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,6 +1,7 @@
GEM
remote: https://rubygems.org/
specs:
+ CFPropertyList (2.3.1)
RedCloth (4.2.9)
ace-rails-ap (2.0.1)
actionmailer (4.1.9)
@@ -163,7 +164,7 @@ GEM
erubis (2.7.0)
escape_utils (0.2.4)
eventmachine (1.0.4)
- excon (0.32.1)
+ excon (0.45.3)
execjs (2.5.2)
expression_parser (0.9.0)
factory_girl (4.3.0)
@@ -178,29 +179,69 @@ GEM
fastercsv (1.5.5)
ffaker (2.0.0)
ffi (1.9.8)
- fog (1.21.0)
- fog-brightbox
- fog-core (~> 1.21, >= 1.21.1)
+ fission (0.5.0)
+ CFPropertyList (~> 2.2)
+ fog (1.25.0)
+ fog-brightbox (~> 0.4)
+ fog-core (~> 1.25)
fog-json
+ fog-profitbricks
+ fog-radosgw (>= 0.0.2)
+ fog-sakuracloud (>= 0.0.4)
+ fog-softlayer
+ fog-terremark
+ fog-vmfusion
+ fog-voxel
+ fog-xml (~> 0.1.1)
+ ipaddress (~> 0.5)
nokogiri (~> 1.5, >= 1.5.11)
- fog-brightbox (0.0.1)
- fog-core
+ opennebula
+ fog-brightbox (0.7.1)
+ fog-core (~> 1.22)
fog-json
- fog-core (1.21.1)
+ inflecto (~> 0.0.2)
+ fog-core (1.30.0)
builder
- excon (~> 0.32)
- formatador (~> 0.2.0)
+ excon (~> 0.45)
+ formatador (~> 0.2)
mime-types
net-scp (~> 1.1)
net-ssh (>= 2.1.3)
- fog-json (1.0.0)
- multi_json (~> 1.0)
+ fog-json (1.0.2)
+ fog-core (~> 1.0)
+ multi_json (~> 1.10)
+ fog-profitbricks (0.0.3)
+ fog-core
+ fog-xml
+ nokogiri
+ fog-radosgw (0.0.4)
+ fog-core (>= 1.21.0)
+ fog-json
+ fog-xml (>= 0.0.1)
+ fog-sakuracloud (1.0.1)
+ fog-core
+ fog-json
+ fog-softlayer (0.4.6)
+ fog-core
+ fog-json
+ fog-terremark (0.1.0)
+ fog-core
+ fog-xml
+ fog-vmfusion (0.1.0)
+ fission
+ fog-core
+ fog-voxel (0.1.0)
+ fog-core
+ fog-xml
+ fog-xml (0.1.2)
+ fog-core
+ nokogiri (~> 1.5, >= 1.5.11)
font-awesome-rails (4.2.0.0)
railties (>= 3.2, < 5.0)
foreman (0.63.0)
dotenv (>= 0.7)
thor (>= 0.13.6)
- formatador (0.2.4)
+ formatador (0.2.5)
gemnasium-gitlab-service (0.2.6)
rugged (~> 0.21)
gemojione (2.0.0)
@@ -287,6 +328,8 @@ GEM
i18n (0.7.0)
ice_cube (0.11.1)
ice_nine (0.10.0)
+ inflecto (0.0.2)
+ ipaddress (0.8.0)
jquery-atwho-rails (1.0.1)
jquery-rails (3.1.2)
railties (>= 3.0, < 5.0)
@@ -328,9 +371,9 @@ GEM
multipart-post (1.2.0)
mysql2 (0.3.16)
net-ldap (0.11)
- net-scp (1.1.2)
+ net-scp (1.2.1)
net-ssh (>= 2.6.5)
- net-ssh (2.8.0)
+ net-ssh (2.9.2)
newrelic_rpm (3.9.4.245)
nokogiri (1.6.6.2)
mini_portile (~> 0.6.0)
@@ -379,6 +422,10 @@ GEM
omniauth-twitter (1.0.1)
multi_json (~> 1.3)
omniauth-oauth (~> 1.0)
+ opennebula (4.12.1)
+ json
+ nokogiri
+ rbvmomi
org-ruby (0.9.12)
rubypants (~> 0.2)
orm_adapter (0.5.0)
@@ -446,6 +493,10 @@ GEM
rb-fsevent (0.9.4)
rb-inotify (0.9.5)
ffi (>= 0.5.0)
+ rbvmomi (1.8.2)
+ builder
+ nokogiri (>= 1.4.1)
+ trollop
rdoc (3.12.2)
json (~> 1.4)
redcarpet (3.2.3)
@@ -621,6 +672,7 @@ GEM
multi_json (~> 1.7)
twitter-stream (~> 0.1)
tins (0.13.1)
+ trollop (2.1.2)
turbolinks (2.5.3)
coffee-rails
twitter-stream (0.1.16)
@@ -704,7 +756,7 @@ DEPENDENCIES
enumerize
factory_girl_rails
ffaker (~> 2.0.0)
- fog (~> 1.14)
+ fog (~> 1.25.0)
font-awesome-rails (~> 4.2)
foreman
gemnasium-gitlab-service (~> 0.2)
diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example
index 787b3ccfc56..220a898cd61 100644
--- a/config/gitlab.yml.example
+++ b/config/gitlab.yml.example
@@ -252,6 +252,10 @@ production: &base
# aws_secret_access_key: 'secret123'
# # The remote 'directory' to store your backups. For S3, this would be the bucket name.
# remote_directory: 'my.s3.bucket'
+ # # Use multipart uploads when file size reaches 100MB, see
+ # # http://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html
+ # multipart_chunk_size: 104857600
+
## GitLab Shell settings
gitlab_shell:
diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb
index 1bd14a3a89f..c2c3c5bfde7 100644
--- a/config/initializers/1_settings.rb
+++ b/config/initializers/1_settings.rb
@@ -174,6 +174,7 @@ Settings.backup['upload'] ||= Settingslogic.new({ 'remote_directory' => nil, 'co
if Settings.backup['upload']['connection']
Settings.backup['upload']['connection'] = Hash[Settings.backup['upload']['connection'].map { |k, v| [k.to_sym, v] }]
end
+Settings.backup['upload']['multipart_chunk_size'] ||= 104857600
#
# Git
diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb
index b69aebf9fe1..b1596cc51b7 100644
--- a/lib/backup/manager.rb
+++ b/lib/backup/manager.rb
@@ -46,7 +46,8 @@ module Backup
connection = ::Fog::Storage.new(connection_settings)
directory = connection.directories.get(remote_directory)
- if directory.files.create(key: tar_file, body: File.open(tar_file), public: false)
+ if directory.files.create(key: tar_file, body: File.open(tar_file), public: false,
+ multipart_chunk_size: GitlabCi.config.backup.upload.multipart_chunk_size)
$progress.puts "done".green
else
puts "uploading backup to #{remote_directory} failed".red
--
GitLab