Merge branch '40781-os-to-ce' into 'master'

Bring Object Storage to CE

Closes #4171, #4163, #3370, #2841, and #29203

See merge request gitlab-org/gitlab-ce!17358

db/migrate/20170601163708_add_artifacts_store_to_ci_build.rb

+class AddArtifactsStoreToCiBuild < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column(:ci_builds, :artifacts_file_store, :integer)
+    add_column(:ci_builds, :artifacts_metadata_store, :integer)
+  end
+end

db/migrate/20170825015534_add_file_store_to_lfs_objects.rb

+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddFileStoreToLfsObjects < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  # When a migration requires downtime you **must** uncomment the following
+  # constant and define a short and easy to understand explanation as to why the
+  # migration requires downtime.
+  # DOWNTIME_REASON = ''
+
+  # When using the methods "add_concurrent_index", "remove_concurrent_index" or
+  # "add_column_with_default" you must disable the use of transactions
+  # as these methods can not run in an existing transaction.
+  # When using "add_concurrent_index" or "remove_concurrent_index" methods make sure
+  # that either of them is the _only_ method called in the migration,
+  # any other changes should go in a separate migration.
+  # This ensures that upon failure _only_ the index creation or removing fails
+  # and can be retried or reverted easily.
+  #
+  # To disable transactions uncomment the following line and remove these
+  # comments:
+  # disable_ddl_transaction!
+
+  def change
+    add_column(:lfs_objects, :file_store, :integer)
+  end
+end

db/migrate/20170918072949_add_file_store_job_artifacts.rb

+class AddFileStoreJobArtifacts < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  disable_ddl_transaction!
+  DOWNTIME = false
+
+  def change
+    add_column(:ci_job_artifacts, :file_store, :integer)
+  end
+end

db/migrate/20171214144320_add_store_column_to_uploads.rb

+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddStoreColumnToUploads < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column(:uploads, :store, :integer)
+  end
+end

db/schema.rb

@@ -307,6 +307,8 @@ ActiveRecord::Schema.define(version: 20180323150945) do
     t.integer "auto_canceled_by_id"
     t.boolean "retried"
     t.integer "stage_id"
+    t.integer "artifacts_file_store"
+    t.integer "artifacts_metadata_store"
     t.boolean "protected"
     t.integer "failure_reason"
   end
@@ -345,6 +347,7 @@ ActiveRecord::Schema.define(version: 20180323150945) do
     t.integer "project_id", null: false
     t.integer "job_id", null: false
     t.integer "file_type", null: false
+    t.integer "file_store"
     t.integer "size", limit: 8
     t.datetime_with_timezone "created_at", null: false
     t.datetime_with_timezone "updated_at", null: false
@@ -1009,6 +1012,7 @@ ActiveRecord::Schema.define(version: 20180323150945) do
     t.datetime "created_at"
     t.datetime "updated_at"
     t.string "file"
+    t.integer "file_store"
   end
   add_index "lfs_objects", ["oid"], name: "index_lfs_objects_on_oid", unique: true, using: :btree
@@ -1824,6 +1828,7 @@ ActiveRecord::Schema.define(version: 20180323150945) do
     t.datetime "created_at", null: false
     t.string "mount_point"
     t.string "secret"
+    t.integer "store"
   end
   add_index "uploads", ["checksum"], name: "index_uploads_on_checksum", using: :btree

doc/administration/job_artifacts.md

@@ -87,10 +87,124 @@ _The artifacts are stored by default in
 ### Using object storage
+>**Notes:**
+- [Introduced][ee-1762] in [GitLab Premium][eep] 9.4.
+- Since version 9.5, artifacts are [browsable], when object storage is enabled. 
+  9.4 lacks this feature.
 > Available in [GitLab Premium](https://about.gitlab.com/products/) and
 [GitLab.com Silver](https://about.gitlab.com/gitlab-com/).
-Use an [Object storage option][ee-os] like AWS S3 to store job artifacts.
+> Since version 10.6, available in [GitLab CE](https://about.gitlab.com/products/)
+
+If you don't want to use the local disk where GitLab is installed to store the
+artifacts, you can use an object storage like AWS S3 instead.
+This configuration relies on valid AWS credentials to be configured already.
+Use an [Object storage option][os] like AWS S3 to store job artifacts.
+
+### Object Storage Settings
+
+For source installations the following settings are nested under `artifacts:` and then `object_store:`. On omnibus installs they are prefixed by `artifacts_object_store_`.
+
+| Setting | Description | Default |
+|---------|-------------|---------|
+| `enabled` | Enable/disable object storage | `false` |
+| `remote_directory` | The bucket name where Artfacts will be stored| |
+| `background_upload` | Set to false to disable automatic upload. Option may be removed once upload is direct to S3 | `true` |
+| `proxy_download` | Set to true to enable proxying all files served. Option allows to reduce egress traffic as this allows clients to download directly from remote storage instead of proxying all data | `false` |
+| `connection` | Various connection options described below | |
+
+#### S3 compatible connection settings
+
+The connection settings match those provided by [Fog](https://github.com/fog), and are as follows:
+
+| Setting | Description | Default |
+|---------|-------------|---------|
+| `provider` | Always `AWS` for compatible hosts | AWS |
+| `aws_access_key_id` | AWS credentials, or compatible | |
+| `aws_secret_access_key` | AWS credentials, or compatible | |
+| `region` | AWS region | us-east-1 |
+| `host` | S3 compatible host for when not using AWS, e.g. `localhost` or `storage.example.com` | s3.amazonaws.com |
+| `endpoint` | Can be used when configuring an S3 compatible service such as [Minio](https://www.minio.io), by entering a URL such as `http://127.0.0.1:9000` | (optional) |
+| `path_style` | Set to true to use `host/bucket_name/object` style paths instead of `bucket_name.host/object`. Leave as false for AWS S3 | false |
+
+**In Omnibus installations:**
+
+_The artifacts are stored by default in
+`/var/opt/gitlab/gitlab-rails/shared/artifacts`._
+
+1. Edit `/etc/gitlab/gitlab.rb` and add the following lines by replacing with
+   the values you want:
+
+    ```ruby
+    gitlab_rails['artifacts_enabled'] = true
+    gitlab_rails['artifacts_object_store_enabled'] = true
+    gitlab_rails['artifacts_object_store_remote_directory'] = "artifacts"
+    gitlab_rails['artifacts_object_store_connection'] = {
+      'provider' => 'AWS',
+      'region' => 'eu-central-1',
+      'aws_access_key_id' => 'AWS_ACCESS_KEY_ID',
+      'aws_secret_access_key' => 'AWS_SECRET_ACCESS_KEY'
+    }
+    ```
+
+    NOTE: For GitLab 9.4+, if you are using AWS IAM profiles, be sure to omit the
+    AWS access key and secret acces key/value pairs. For example:
+
+    ```ruby
+    gitlab_rails['artifacts_object_store_connection'] = {
+      'provider' => 'AWS',
+      'region' => 'eu-central-1',
+      'use_iam_profile' => true
+    }
+    ```
+
+1. Save the file and [reconfigure GitLab][] for the changes to take effect.
+1. Migrate any existing local artifacts to the object storage:
+
+      ```bash
+      gitlab-rake gitlab:artifacts:migrate
+      ```
+
+      Currently this has to be executed manually and it will allow you to
+      migrate the existing artifacts to the object storage, but all new
+      artifacts will still be stored on the local disk. In the future
+      you will be given an option to define a default storage artifacts for all
+      new files.
+
+---
+
+**In installations from source:**
+
+_The artifacts are stored by default in
+`/home/git/gitlab/shared/artifacts`._
+
+1. Edit `/home/git/gitlab/config/gitlab.yml` and add or amend the following
+   lines:
+
+    ```yaml
+    artifacts:
+      enabled: true
+      object_store:
+        enabled: true
+        remote_directory: "artifacts" # The bucket name
+        connection:
+          provider: AWS # Only AWS supported at the moment
+          aws_access_key_id: AWS_ACESS_KEY_ID
+          aws_secret_access_key: AWS_SECRET_ACCESS_KEY
+          region: eu-central-1
+    ```
+
+1. Save the file and [restart GitLab][] for the changes to take effect.
+1. Migrate any existing local artifacts to the object storage:
+
+      ```bash
+      sudo -u git -H bundle exec rake gitlab:artifacts:migrate RAILS_ENV=production
+      ```
+      Currently this has to be executed manually and it will allow you to
+      migrate the existing artifacts to the object storage, but all new
+      artifacts will still be stored on the local disk. In the future
+      you will be given an option to define a default storage artifacts for all
+      new files.
 ## Expiring artifacts
@@ -194,7 +308,7 @@ When clicking on a specific file, [GitLab Workhorse] extracts it
 from the archive and the download begins. This implementation saves space,
 memory and disk I/O.
-[reconfigure gitlab]: restart_gitlab.md "How to restart GitLab"
-[restart gitlab]: restart_gitlab.md "How to restart GitLab"
+[reconfigure gitlab]: restart_gitlab.md#omnibus-gitlab-reconfigure "How to reconfigure Omnibus GitLab"
+[restart gitlab]: restart_gitlab.md#installations-from-source "How to restart GitLab"
 [gitlab workhorse]: https://gitlab.com/gitlab-org/gitlab-workhorse "GitLab Workhorse repository"
-[ee-os]: https://docs.gitlab.com/ee/administration/job_artifacts.html#using-object-storage
+[os]: https://docs.gitlab.com/administration/job_artifacts.html#using-object-storage

doc/administration/raketasks/uploads/migrate.md

+# Uploads Migrate Rake Task
+
+## Migrate to Object Storage
+
+After [configuring the object storage](../../uploads.md#using-object-storage) for GitLab's uploads, you may use this task to migrate existing uploads from the local storage to the remote storage.
+
+>**Note:**
+All of the processing will be done in a background worker and requires **no downtime**.
+
+This tasks uses 3 parameters to find uploads to migrate.
+
+>**Note:**
+These parameters are mainly internal to GitLab's structure, you may want to refer to the task list instead below.
+
+Parameter | Type | Description
+--------- | ---- | -----------
+`uploader_class` | string | Type of the uploader to migrate from
+`model_class` | string | Type of the model to migrate from
+`mount_point` | string/symbol | Name of the model's column on which the uploader is mounted on.
+
+This task also accepts some environment variables which you can use to override
+certain values:
+
+Variable | Type | Description
+-------- | ---- | -----------
+`BATCH`   | integer  | Specifies the size of the batch. Defaults to 200.
+
+** Omnibus Installation**
+
+```bash
+# gitlab-rake gitlab:uploads:migrate[uploader_class, model_class, mount_point]
+
+# Avatars
+gitlab-rake "gitlab:uploads:migrate[AvatarUploader, Project, :avatar]"
+gitlab-rake "gitlab:uploads:migrate[AvatarUploader, Group, :avatar]"
+gitlab-rake "gitlab:uploads:migrate[AvatarUploader, User, :avatar]"
+
+# Attachments
+gitlab-rake "gitlab:uploads:migrate[AttachmentUploader, Note, :attachment]"
+gitlab-rake "gitlab:uploads:migrate[AttachmentUploader, Appearance, :logo]"
+gitlab-rake "gitlab:uploads:migrate[AttachmentUploader, Appearance, :header_logo]"
+
+# Markdown
+gitlab-rake "gitlab:uploads:migrate[FileUploader, Project]"
+gitlab-rake "gitlab:uploads:migrate[PersonalFileUploader, Snippet]"
+gitlab-rake "gitlab:uploads:migrate[NamespaceFileUploader, Snippet]"
+gitlab-rake "gitlab:uploads:migrate[FileUploader, MergeRequest]"
+```
+
+**Source Installation**
+
+>**Note:**
+Use `RAILS_ENV=production` for every task.
+
+```bash
+# sudo -u git -H bundle exec rake gitlab:uploads:migrate
+
+# Avatars
+sudo -u git -H bundle exec rake "gitlab:uploads:migrate[AvatarUploader, Project, :avatar]"
+sudo -u git -H bundle exec rake "gitlab:uploads:migrate[AvatarUploader, Group, :avatar]"
+sudo -u git -H bundle exec rake "gitlab:uploads:migrate[AvatarUploader, User, :avatar]"
+
+# Attachments
+sudo -u git -H bundle exec rake "gitlab:uploads:migrate[AttachmentUploader, Note, :attachment]"
+sudo -u git -H bundle exec rake "gitlab:uploads:migrate[AttachmentUploader, Appearance, :logo]"
+sudo -u git -H bundle exec rake "gitlab:uploads:migrate[AttachmentUploader, Appearance, :header_logo]"
+
+# Markdown
+sudo -u git -H bundle exec rake "gitlab:uploads:migrate[FileUploader, Project]"
+sudo -u git -H bundle exec rake "gitlab:uploads:migrate[PersonalFileUploader, Snippet]"
+sudo -u git -H bundle exec rake "gitlab:uploads:migrate[NamespaceFileUploader, Snippet]"
+sudo -u git -H bundle exec rake "gitlab:uploads:migrate[FileUploader, MergeRequest]"
+
+```

doc/administration/uploads.md

+# Uploads administration
+
+>**Notes:**
+Uploads represent all user data that may be sent to GitLab as a single file. As an example, avatars and notes' attachments are uploads. Uploads are integral to GitLab functionality, and therefore cannot be disabled.
+
+### Using local storage
+
+>**Notes:**
+This is the default configuration
+
+To change the location where the uploads are stored locally, follow the steps
+below.
+
+---
+
+**In Omnibus installations:**
+
+>**Notes:**
+For historical reasons, uploads are stored into a base directory, which by default is `uploads/-/system`. It is strongly discouraged to change this configuration option on an existing GitLab installation.
+
+_The uploads are stored by default in `/var/opt/gitlab/gitlab-rails/public/uploads/-/system`._
+
+1. To change the storage path for example to `/mnt/storage/uploads`, edit
+   `/etc/gitlab/gitlab.rb` and add the following line:
+
+    ```ruby
+    gitlab_rails['uploads_storage_path'] = "/mnt/storage/"
+	gitlab_rails['uploads_base_dir'] = "uploads"
+    ```
+
+1. Save the file and [reconfigure GitLab][] for the changes to take effect.
+
+---
+
+**In installations from source:**
+
+_The uploads are stored by default in
+`/home/git/gitlab/public/uploads/-/system`._
+
+1. To change the storage path for example to `/mnt/storage/uploads`, edit
+   `/home/git/gitlab/config/gitlab.yml` and add or amend the following lines:
+
+    ```yaml
+	uploads:
+	  storage_path: /mnt/storage
+	  base_dir: uploads
+    ```
+
+1. Save the file and [restart GitLab][] for the changes to take effect.
+
+### Using object storage
+
+>**Notes:**
+- [Introduced][ee-3867] in [GitLab Enterprise Edition Premium][eep] 10.5.
+
+If you don't want to use the local disk where GitLab is installed to store the
+uploads, you can use an object storage provider like AWS S3 instead.
+This configuration relies on valid AWS credentials to be configured already.
+
+### Object Storage Settings
+
+For source installations the following settings are nested under `uploads:` and then `object_store:`. On omnibus installs they are prefixed by `uploads_object_store_`.
+
+| Setting | Description | Default |
+|---------|-------------|---------|
+| `enabled` | Enable/disable object storage | `false` |
+| `remote_directory` | The bucket name where Uploads will be stored| |
+| `background_upload` | Set to false to disable automatic upload. Option may be removed once upload is direct to S3 | `true` |
+| `proxy_download` | Set to true to enable proxying all files served. Option allows to reduce egress traffic as this allows clients to download directly from remote storage instead of proxying all data | `false` |
+| `connection` | Various connection options described below | |
+
+#### S3 compatible connection settings
+
+The connection settings match those provided by [Fog](https://github.com/fog), and are as follows:
+
+| Setting | Description | Default |
+|---------|-------------|---------|
+| `provider` | Always `AWS` for compatible hosts | AWS |
+| `aws_access_key_id` | AWS credentials, or compatible | |
+| `aws_secret_access_key` | AWS credentials, or compatible | |
+| `region` | AWS region | us-east-1 |
+| `host` | S3 compatible host for when not using AWS, e.g. `localhost` or `storage.example.com` | s3.amazonaws.com |
+| `endpoint` | Can be used when configuring an S3 compatible service such as [Minio](https://www.minio.io), by entering a URL such as `http://127.0.0.1:9000` | (optional) |
+| `path_style` | Set to true to use `host/bucket_name/object` style paths instead of `bucket_name.host/object`. Leave as false for AWS S3 | false |
+
+**In Omnibus installations:**
+
+_The uploads are stored by default in
+`/var/opt/gitlab/gitlab-rails/public/uploads/-/system`._
+
+1. Edit `/etc/gitlab/gitlab.rb` and add the following lines by replacing with
+   the values you want:
+
+    ```ruby
+    gitlab_rails['uploads_object_store_enabled'] = true
+    gitlab_rails['uploads_object_store_remote_directory'] = "uploads"
+    gitlab_rails['uploads_object_store_connection'] = {
+      'provider' => 'AWS',
+      'region' => 'eu-central-1',
+      'aws_access_key_id' => 'AWS_ACCESS_KEY_ID',
+      'aws_secret_access_key' => 'AWS_SECRET_ACCESS_KEY'
+    }
+    ```
+
+>**Note:**
+If you are using AWS IAM profiles, be sure to omit the AWS access key and secret acces key/value pairs.
+
+    ```ruby
+    gitlab_rails['uploads_object_store_connection'] = {
+      'provider' => 'AWS',
+      'region' => 'eu-central-1',
+      'use_iam_profile' => true
+    }
+    ```
+
+1. Save the file and [reconfigure GitLab][] for the changes to take effect.
+1. Migrate any existing local uploads to the object storage:
+
+>**Notes:**
+These task complies with the `BATCH` environment variable to process uploads in batch (200 by default). All of the processing will be done in a background worker and requires **no downtime**.
+
+      ```bash
+      # gitlab-rake gitlab:uploads:migrate[uploader_class, model_class, mount_point]
+	  
+	  # Avatars
+	  gitlab-rake "gitlab:uploads:migrate[AvatarUploader, Project, :avatar]"
+	  gitlab-rake "gitlab:uploads:migrate[AvatarUploader, Group, :avatar]"
+	  gitlab-rake "gitlab:uploads:migrate[AvatarUploader, User, :avatar]"
+
+      # Attachments
+	  gitlab-rake "gitlab:uploads:migrate[AttachmentUploader, Note, :attachment]"
+	  gitlab-rake "gitlab:uploads:migrate[AttachmentUploader, Appearance, :logo]"
+	  gitlab-rake "gitlab:uploads:migrate[AttachmentUploader, Appearance, :header_logo]"
+
+      # Markdown
+	  gitlab-rake "gitlab:uploads:migrate[FileUploader, Project]"
+	  gitlab-rake "gitlab:uploads:migrate[PersonalFileUploader, Snippet]"
+	  gitlab-rake "gitlab:uploads:migrate[NamespaceFileUploader, Snippet]"
+	  gitlab-rake "gitlab:uploads:migrate[FileUploader, MergeRequest]"
+      ```
+
+      Currently this has to be executed manually and it will allow you to
+      migrate the existing uploads to the object storage, but all new
+      uploads will still be stored on the local disk. In the future
+      you will be given an option to define a default storage for all
+      new files.
+
+---
+
+**In installations from source:**
+
+_The uploads are stored by default in
+`/home/git/gitlab/public/uploads/-/system`._
+
+1. Edit `/home/git/gitlab/config/gitlab.yml` and add or amend the following
+   lines:
+
+    ```yaml
+    uploads:
+      object_store:
+        enabled: true
+        remote_directory: "uploads" # The bucket name
+        connection:
+          provider: AWS # Only AWS supported at the moment
+          aws_access_key_id: AWS_ACESS_KEY_ID
+          aws_secret_access_key: AWS_SECRET_ACCESS_KEY
+          region: eu-central-1
+    ```
+
+1. Save the file and [restart GitLab][] for the changes to take effect.
+1. Migrate any existing local uploads to the object storage:
+
+>**Notes:**
+
+- These task comply with the `BATCH` environment variable to process uploads in batch (200 by default). All of the processing will be done in a background worker and requires **no downtime**.
+
+- To migrate in production use `RAILS_ENV=production` environment variable.
+
+      ```bash
+      # sudo -u git -H bundle exec rake gitlab:uploads:migrate
+	  
+	  # Avatars
+	  sudo -u git -H bundle exec rake "gitlab:uploads:migrate[AvatarUploader, Project, :avatar]"
+	  sudo -u git -H bundle exec rake "gitlab:uploads:migrate[AvatarUploader, Group, :avatar]"
+	  sudo -u git -H bundle exec rake "gitlab:uploads:migrate[AvatarUploader, User, :avatar]"
+
+      # Attachments
+	  sudo -u git -H bundle exec rake "gitlab:uploads:migrate[AttachmentUploader, Note, :attachment]"
+	  sudo -u git -H bundle exec rake "gitlab:uploads:migrate[AttachmentUploader, Appearance, :logo]"
+	  sudo -u git -H bundle exec rake "gitlab:uploads:migrate[AttachmentUploader, Appearance, :header_logo]"
+
+      # Markdown
+	  sudo -u git -H bundle exec rake "gitlab:uploads:migrate[FileUploader, Project]"
+	  sudo -u git -H bundle exec rake "gitlab:uploads:migrate[PersonalFileUploader, Snippet]"
+	  sudo -u git -H bundle exec rake "gitlab:uploads:migrate[NamespaceFileUploader, Snippet]"
+	  sudo -u git -H bundle exec rake "gitlab:uploads:migrate[FileUploader, MergeRequest]"
+	  
+      ```
+
+      Currently this has to be executed manually and it will allow you to
+      migrate the existing uploads to the object storage, but all new
+      uploads will still be stored on the local disk. In the future
+      you will be given an option to define a default storage for all
+      new files.
+
+[reconfigure gitlab]: restart_gitlab.md#omnibus-gitlab-reconfigure "How to reconfigure Omnibus GitLab"
+[restart gitlab]: restart_gitlab.md#installations-from-source "How to restart GitLab"
+[eep]: https://about.gitlab.com/gitlab-ee/ "GitLab Enterprise Edition Premium"
+[ee-3867]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/3867

doc/raketasks/README.md

@@ -14,3 +14,4 @@ comments: false
 - [Webhooks](web_hooks.md)
 - [Import](import.md) of git repositories in bulk
 - [Rebuild authorized_keys file](http://docs.gitlab.com/ce/raketasks/maintenance.html#rebuild-authorized_keys-file) task for administrators
+- [Migrate Uploads](../administration/raketasks/uploads/migrate.md)

doc/workflow/lfs/lfs_administration.md

@@ -5,6 +5,7 @@ Documentation on how to use Git LFS are under [Managing large binary files with
 ## Requirements
 * Git LFS is supported in GitLab starting with version 8.2.
+* Support for object storage, such as AWS S3, was introduced in 10.0.
 * Users need to install [Git LFS client](https://git-lfs.github.com) version 1.0.1 and up.
 ## Configuration
@@ -12,16 +13,18 @@ Documentation on how to use Git LFS are under [Managing large binary files with
 Git LFS objects can be large in size. By default, they are stored on the server
 GitLab is installed on.
-There are two configuration options to help GitLab server administrators:
+There are various configuration options to help GitLab server administrators:
 * Enabling/disabling Git LFS support
 * Changing the location of LFS object storage
+* Setting up AWS S3 compatible object storage
 ### Omnibus packages
 In `/etc/gitlab/gitlab.rb`:
 ```ruby
+# Change to true to enable lfs
 gitlab_rails['lfs_enabled'] = false
 # Optionally, change the storage path location. Defaults to
@@ -35,11 +38,114 @@ gitlab_rails['lfs_storage_path'] = "/mnt/storage/lfs-objects"
 In `config/gitlab.yml`:
 ```yaml
+# Change to true to enable lfs
   lfs:
     enabled: false
     storage_path: /mnt/storage/lfs-objects
 ```
+## Setting up S3 compatible object storage
+
+> **Note:** [Introduced][ee-2760] in [GitLab Premium][eep] 10.0.
+> Available in [GitLab CE][ce] 10.7
+
+It is possible to store LFS objects on remote object storage instead of on a local disk.
+
+This allows you to offload storage to an external AWS S3 compatible service, freeing up disk space locally. You can also host your own S3 compatible storage decoupled from GitLab, with with a service such as [Minio](https://www.minio.io/).
+
+Object storage currently transfers files first to GitLab, and then on the object storage in a second stage. This can be done either by using a rake task to transfer existing objects, or in a background job after each file is received.
+
+### Object Storage Settings
+
+For source installations the following settings are nested under `lfs:` and then `object_store:`. On omnibus installs they are prefixed by `lfs_object_store_`.
+
+| Setting | Description | Default |
+|---------|-------------|---------|
+| `enabled` | Enable/disable object storage | `false` |
+| `remote_directory` | The bucket name where LFS objects will be stored| |
+| `background_upload` | Set to false to disable automatic upload. Option may be removed once upload is direct to S3 | `true` |
+| `proxy_download` | Set to true to enable proxying all files served. Option allows to reduce egress traffic as this allows clients to download directly from remote storage instead of proxying all data | `false` |
+| `connection` | Various connection options described below | |
+
+#### S3 compatible connection settings
+
+The connection settings match those provided by [Fog](https://github.com/fog), and are as follows:
+
+| Setting | Description | Default |
+|---------|-------------|---------|
+| `provider` | Always `AWS` for compatible hosts | AWS |
+| `aws_access_key_id` | AWS credentials, or compatible | |
+| `aws_secret_access_key` | AWS credentials, or compatible | |
+| `region` | AWS region | us-east-1 |
+| `host` | S3 compatible host for when not using AWS, e.g. `localhost` or `storage.example.com` | s3.amazonaws.com |
+| `endpoint` | Can be used when configuring an S3 compatible service such as [Minio](https://www.minio.io), by entering a URL such as `http://127.0.0.1:9000` | (optional) |
+| `path_style` | Set to true to use `host/bucket_name/object` style paths instead of `bucket_name.host/object`. Leave as false for AWS S3 | false |
+
+
+### From source
+
+1. Edit `/home/git/gitlab/config/gitlab.yml` and add or amend the following
+   lines:
+
+	```yaml
+	lfs:
+	enabled: true
+	object_store:
+	  enabled: false
+	  remote_directory: lfs-objects # Bucket name
+	  connection:
+	    provider: AWS
+	    aws_access_key_id: 1ABCD2EFGHI34JKLM567N
+	    aws_secret_access_key: abcdefhijklmnopQRSTUVwxyz0123456789ABCDE
+	    region: eu-central-1
+	    # Use the following options to configure an AWS compatible host such as Minio
+	    host: 'localhost'
+	    endpoint: 'http://127.0.0.1:9000'
+	    path_style: true
+	```
+
+1. Save the file and [restart GitLab][] for the changes to take effect.
+1. Migrate any existing local LFS objects to the object storage:
+
+	```bash
+	sudo -u git -H bundle exec rake gitlab:lfs:migrate RAILS_ENV=production
+	```
+
+	This will migrate existing LFS objects to object storage. New LFS objects
+	will be forwarded to object storage unless
+	`gitlab_rails['lfs_object_store_background_upload']` is set to false.
+
+### In Omnibus
+
+1. Edit `/etc/gitlab/gitlab.rb` and add the following lines by replacing with
+   the values you want:
+
+	```ruby
+	gitlab_rails['lfs_object_store_enabled'] = true
+	gitlab_rails['lfs_object_store_remote_directory'] = "lfs-objects"
+	gitlab_rails['lfs_object_store_connection'] = {
+	  'provider' => 'AWS',
+	  'region' => 'eu-central-1',
+	  'aws_access_key_id' => '1ABCD2EFGHI34JKLM567N',
+	  'aws_secret_access_key' => 'abcdefhijklmnopQRSTUVwxyz0123456789ABCDE',
+	  # The below options configure an S3 compatible host instead of AWS
+	  'host' => 'localhost',
+	  'endpoint' => 'http://127.0.0.1:9000',
+	  'path_style' => true
+	}
+	```
+
+1. Save the file and [reconfigure GitLab]s for the changes to take effect.
+1. Migrate any existing local LFS objects to the object storage:
+
+      ```bash
+      gitlab-rake gitlab:lfs:migrate
+      ```
+
+      This will migrate existing LFS objects to object storage. New LFS objects
+      will be forwarded to object storage unless
+      `gitlab_rails['lfs_object_store_background_upload']` is set to false.
+
 ## Storage statistics
 You can see the total storage used for LFS objects on groups and projects
@@ -48,10 +154,13 @@ and [projects APIs](../../api/projects.md).
 ## Known limitations
-* Currently, storing GitLab Git LFS objects on a non-local storage (like S3 buckets)
-  is not supported
 * Support for removing unreferenced LFS objects was added in 8.14 onwards.
 * LFS authentications via SSH was added with GitLab 8.12
 * Only compatible with the GitLFS client versions 1.1.0 and up, or 1.0.2.
 * The storage statistics currently count each LFS object multiple times for
   every project linking to it
+
+[reconfigure gitlab]: ../../administration/restart_gitlab.md#omnibus-gitlab-reconfigure "How to reconfigure Omnibus GitLab"
+[restart gitlab]: ../../administration/restart_gitlab.md#installations-from-source "How to restart GitLab"
+[eep]: https://about.gitlab.com/products/ "GitLab Premium"
+[ee-2760]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/2760

lib/api/helpers.rb

@@ -410,7 +410,7 @@ module API
       )
     end
-    def present_file!(path, filename, content_type = 'application/octet-stream')
+    def present_disk_file!(path, filename, content_type = 'application/octet-stream')
       filename ||= File.basename(path)
       header['Content-Disposition'] = "attachment; filename=#{filename}"
       header['Content-Transfer-Encoding'] = 'binary'
@@ -426,13 +426,17 @@ module API
       end
     end
-    def present_artifacts!(artifacts_file)
-      return not_found! unless artifacts_file.exists?
-      if artifacts_file.file_storage?
-        present_file!(artifacts_file.path, artifacts_file.filename)
+    def present_carrierwave_file!(file, supports_direct_download: true)
+      return not_found! unless file.exists?
+      if file.file_storage?
+        present_disk_file!(file.path, file.filename)
+      elsif supports_direct_download && file.class.direct_download_enabled?
+        redirect(file.url)
       else
-        redirect_to(artifacts_file.url)
+        header(*Gitlab::Workhorse.send_url(file.url))
+        status :ok
+        body
       end
     end

lib/api/job_artifacts.rb

@@ -28,7 +28,7 @@ module API
         builds = user_project.latest_successful_builds_for(params[:ref_name])
         latest_build = builds.find_by!(name: params[:job])
-        present_artifacts!(latest_build.artifacts_file)
+        present_carrierwave_file!(latest_build.artifacts_file)
       end
       desc 'Download the artifacts archive from a job' do
@@ -43,7 +43,7 @@ module API
         build = find_build!(params[:job_id])
-        present_artifacts!(build.artifacts_file)
+        present_carrierwave_file!(build.artifacts_file)
       end
       desc 'Download a specific file from artifacts archive' do

lib/api/jobs.rb

@@ -72,7 +72,7 @@ module API
         present build, with: Entities::Job
       end
-      # TODO: We should use `present_file!` and leave this implementation for backward compatibility (when build trace
+      # TODO: We should use `present_disk_file!` and leave this implementation for backward compatibility (when build trace
       #       is saved in the DB instead of file). But before that, we need to consider how to replace the value of
       #       `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse.
       desc 'Get a trace of a specific job of a project'

lib/api/project_export.rb

@@ -25,7 +25,7 @@ module API
         render_api_error!('404 Not found or has expired', 404) unless path
-        present_file!(path, File.basename(path), 'application/gzip')
+        present_disk_file!(path, File.basename(path), 'application/gzip')
       end
       desc 'Start export' do

lib/api/runner.rb

@@ -244,11 +244,12 @@ module API
       params do
         requires :id, type: Integer, desc: %q(Job's ID)
         optional :token, type: String, desc: %q(Job's authentication token)
+        optional :direct_download, default: false, type: Boolean, desc: %q(Perform direct download from remote storage instead of proxying artifacts)
       end
       get '/:id/artifacts' do
         job = authenticate_job!
-        present_artifacts!(job.artifacts_file)
+        present_carrierwave_file!(job.artifacts_file, supports_direct_download: params[:direct_download])
       end
     end
   end

lib/api/v3/builds.rb

@@ -85,7 +85,7 @@ module API
           build = get_build!(params[:build_id])
-          present_artifacts!(build.artifacts_file)
+          present_carrierwave_file!(build.artifacts_file)
         end
         desc 'Download the artifacts file from build' do
@@ -102,10 +102,10 @@ module API
           builds = user_project.latest_successful_builds_for(params[:ref_name])
           latest_build = builds.find_by!(name: params[:job])
-          present_artifacts!(latest_build.artifacts_file)
+          present_carrierwave_file!(latest_build.artifacts_file)
         end
-        # TODO: We should use `present_file!` and leave this implementation for backward compatibility (when build trace
+        # TODO: We should use `present_disk_file!` and leave this implementation for backward compatibility (when build trace
         #       is saved in the DB instead of file). But before that, we need to consider how to replace the value of
         #       `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse.
         desc 'Get a trace of a specific build of a project'

lib/gitlab/ci/trace/http_io.rb

+##
+# This class is compatible with IO class (https://ruby-doc.org/core-2.3.1/IO.html)
+# source: https://gitlab.com/snippets/1685610
+module Gitlab
+  module Ci
+    class Trace
+      class HttpIO
+        BUFFER_SIZE = 128.kilobytes
+
+        InvalidURLError = Class.new(StandardError)
+        FailedToGetChunkError = Class.new(StandardError)
+
+        attr_reader :uri, :size
+        attr_reader :tell
+        attr_reader :chunk, :chunk_range
+
+        alias_method :pos, :tell
+
+        def initialize(url, size)
+          raise InvalidURLError unless ::Gitlab::UrlSanitizer.valid?(url)
+
+          @uri = URI(url)
+          @size = size
+          @tell = 0
+        end
+
+        def close
+          # no-op
+        end
+
+        def binmode
+          # no-op
+        end
+
+        def binmode?
+          true
+        end
+
+        def path
+          nil
+        end
+
+        def url
+          @uri.to_s
+        end
+
+        def seek(pos, where = IO::SEEK_SET)
+          new_pos =
+            case where
+            when IO::SEEK_END
+              size + pos
+            when IO::SEEK_SET
+              pos
+            when IO::SEEK_CUR
+              tell + pos
+            else
+              -1
+            end
+
+          raise 'new position is outside of file' if new_pos < 0 || new_pos > size
+
+          @tell = new_pos
+        end
+
+        def eof?
+          tell == size
+        end
+
+        def each_line
+          until eof?
+            line = readline
+            break if line.nil?
+
+            yield(line)
+          end
+        end
+
+        def read(length = nil)
+          out = ""
+
+          until eof? || (length && out.length >= length)
+            data = get_chunk
+            break if data.empty?
+
+            out << data
+            @tell += data.bytesize
+          end
+
+          out = out[0, length] if length && out.length > length
+
+          out
+        end
+
+        def readline
+          out = ""
+
+          until eof?
+            data = get_chunk
+            new_line = data.index("\n")
+
+            if !new_line.nil?
+              out << data[0..new_line]
+              @tell += new_line + 1
+              break
+            else
+              out << data
+              @tell += data.bytesize
+            end
+          end
+
+          out
+        end
+
+        def write(data)
+          raise NotImplementedError
+        end
+
+        def truncate(offset)
+          raise NotImplementedError
+        end
+
+        def flush
+          raise NotImplementedError
+        end
+
+        def present?
+          true
+        end
+
+        private
+
+        ##
+        # The below methods are not implemented in IO class
+        #
+        def in_range?
+          @chunk_range&.include?(tell)
+        end
+
+        def get_chunk
+          unless in_range?
+            response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https') do |http|
+              http.request(request)
+            end
+
+            raise FailedToGetChunkError unless response.code == '200' || response.code == '206'
+
+            @chunk = response.body.force_encoding(Encoding::BINARY)
+            @chunk_range = response.content_range
+
+            ##
+            # Note: If provider does not return content_range, then we set it as we requested
+            # Provider: minio
+            # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206
+            # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206
+            # Provider: AWS
+            # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206
+            # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206
+            # Provider: GCS
+            # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206
+            # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPOK 200
+            @chunk_range ||= (chunk_start...(chunk_start + @chunk.length))
+          end
+
+          @chunk[chunk_offset..BUFFER_SIZE]
+        end
+
+        def request
+          Net::HTTP::Get.new(uri).tap do |request|
+            request.set_range(chunk_start, BUFFER_SIZE)
+          end
+        end
+
+        def chunk_offset
+          tell % BUFFER_SIZE
+        end
+
+        def chunk_start
+          (tell / BUFFER_SIZE) * BUFFER_SIZE
+        end
+
+        def chunk_end
+          [chunk_start + BUFFER_SIZE, size].min
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/trace/stream.rb

@@ -8,7 +8,7 @@ module Gitlab
         attr_reader :stream
-        delegate :close, :tell, :seek, :size, :path, :truncate, to: :stream, allow_nil: true
+        delegate :close, :tell, :seek, :size, :path, :url, :truncate, to: :stream, allow_nil: true
         delegate :valid?, to: :stream, as: :present?, allow_nil: true

lib/gitlab/verify/lfs_objects.rb

@@ -12,7 +12,7 @@ module Gitlab
       private
       def relation
-        LfsObject.all
+        LfsObject.with_files_stored_locally
       end
       def expected_checksum(lfs_object)

lib/gitlab/verify/uploads.rb

@@ -12,7 +12,7 @@ module Gitlab
       private
       def relation
-        Upload.all
+        Upload.with_files_stored_locally
       end
       def expected_checksum(upload)