Skip to content
Snippets Groups Projects
Commit d1d67546 authored by GitLab Release Tools Bot's avatar GitLab Release Tools Bot
Browse files

Merge branch 'security-12-0-enable-image-proxy' into '12-0-stable'

Use image proxy to mitigate stealing ip addresses

See merge request gitlab/gitlabhq!3192
parents f35fce3f f790b841
No related branches found
No related tags found
No related merge requests found
Showing
with 267 additions and 24 deletions
Loading
Loading
@@ -162,6 +162,10 @@ module ApplicationSettingsHelper
:allow_local_requests_from_hooks_and_services,
:dns_rebinding_protection_enabled,
:archive_builds_in_human_readable,
:asset_proxy_enabled,
:asset_proxy_secret_key,
:asset_proxy_url,
:asset_proxy_whitelist,
:authorized_keys_enabled,
:auto_devops_enabled,
:auto_devops_domain,
Loading
Loading
Loading
Loading
@@ -16,12 +16,19 @@ class ApplicationSetting < ApplicationRecord
# fix a lot of tests using allow_any_instance_of
include ApplicationSettingImplementation
 
attr_encrypted :asset_proxy_secret_key,
mode: :per_attribute_iv,
insecure_mode: true,
key: Settings.attr_encrypted_db_key_base_truncated,
algorithm: 'aes-256-cbc'
serialize :restricted_visibility_levels # rubocop:disable Cop/ActiveRecordSerialize
serialize :import_sources # rubocop:disable Cop/ActiveRecordSerialize
serialize :disabled_oauth_sign_in_sources, Array # rubocop:disable Cop/ActiveRecordSerialize
serialize :domain_whitelist, Array # rubocop:disable Cop/ActiveRecordSerialize
serialize :domain_blacklist, Array # rubocop:disable Cop/ActiveRecordSerialize
serialize :repository_storages # rubocop:disable Cop/ActiveRecordSerialize
serialize :asset_proxy_whitelist, Array # rubocop:disable Cop/ActiveRecordSerialize
 
ignore_column :circuitbreaker_failure_count_threshold
ignore_column :circuitbreaker_failure_reset_time
Loading
Loading
@@ -189,6 +196,17 @@ class ApplicationSetting < ApplicationRecord
allow_nil: true,
numericality: { only_integer: true, greater_than_or_equal_to: 0, less_than: 65536 }
 
validates :asset_proxy_url,
presence: true,
allow_blank: false,
url: true,
if: :asset_proxy_enabled?
validates :asset_proxy_secret_key,
presence: true,
allow_blank: false,
if: :asset_proxy_enabled?
SUPPORTED_KEY_TYPES.each do |type|
validates :"#{type}_key_restriction", presence: true, key_restriction: { type: type }
end
Loading
Loading
Loading
Loading
@@ -21,8 +21,9 @@ module ApplicationSettingImplementation
after_sign_up_text: nil,
akismet_enabled: false,
allow_local_requests_from_hooks_and_services: false,
dns_rebinding_protection_enabled: true,
asset_proxy_enabled: false,
authorized_keys_enabled: true, # TODO default to false if the instance is configured to use AuthorizedKeysCommand
commit_email_hostname: default_commit_email_hostname,
container_registry_token_expire_delay: 5,
default_artifacts_expire_in: '30 days',
default_branch_protection: Settings.gitlab['default_branch_protection'],
Loading
Loading
@@ -31,7 +32,9 @@ module ApplicationSettingImplementation
default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'],
default_projects_limit: Settings.gitlab['default_projects_limit'],
default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'],
diff_max_patch_bytes: Gitlab::Git::Diff::DEFAULT_MAX_PATCH_BYTES,
disabled_oauth_sign_in_sources: [],
dns_rebinding_protection_enabled: true,
domain_whitelist: Settings.gitlab['domain_whitelist'],
dsa_key_restriction: 0,
ecdsa_key_restriction: 0,
Loading
Loading
@@ -50,6 +53,7 @@ module ApplicationSettingImplementation
housekeeping_gc_period: 200,
housekeeping_incremental_repack_period: 10,
import_sources: Settings.gitlab['import_sources'],
local_markdown_version: 0,
max_artifacts_size: Settings.artifacts['max_size'],
max_attachment_size: Settings.gitlab['max_attachment_size'],
mirror_available: true,
Loading
Loading
@@ -61,6 +65,7 @@ module ApplicationSettingImplementation
plantuml_url: nil,
polling_interval_multiplier: 1,
project_export_enabled: true,
protected_ci_variables: false,
recaptcha_enabled: false,
repository_checks_enabled: true,
repository_storages: ['default'],
Loading
Loading
@@ -91,11 +96,7 @@ module ApplicationSettingImplementation
user_default_external: false,
user_default_internal_regex: nil,
user_show_add_ssh_key_message: true,
usage_stats_set_by_user_id: nil,
diff_max_patch_bytes: Gitlab::Git::Diff::DEFAULT_MAX_PATCH_BYTES,
commit_email_hostname: default_commit_email_hostname,
protected_ci_variables: false,
local_markdown_version: 0
usage_stats_set_by_user_id: nil
}
end
 
Loading
Loading
@@ -138,17 +139,20 @@ module ApplicationSettingImplementation
end
 
def domain_whitelist_raw=(values)
self.domain_whitelist = []
self.domain_whitelist = values.split(DOMAIN_LIST_SEPARATOR)
self.domain_whitelist.reject! { |d| d.empty? }
self.domain_whitelist
self.domain_whitelist = domain_string_to_array(values)
end
 
def domain_blacklist_raw=(values)
self.domain_blacklist = []
self.domain_blacklist = values.split(DOMAIN_LIST_SEPARATOR)
self.domain_blacklist.reject! { |d| d.empty? }
self.domain_blacklist
self.domain_blacklist = domain_string_to_array(values)
end
def asset_proxy_whitelist=(values)
values = domain_string_to_array(values) if values.is_a?(String)
# make sure we always whitelist the running host
values << Gitlab.config.gitlab.host unless values.include?(Gitlab.config.gitlab.host)
self[:asset_proxy_whitelist] = values
end
 
def domain_blacklist_file=(file)
Loading
Loading
@@ -296,4 +300,10 @@ module ApplicationSettingImplementation
def expire_performance_bar_allowed_user_ids_cache
Gitlab::PerformanceBar.expire_allowed_user_ids_cache
end
def domain_string_to_array(values)
domain_list = values.split(DOMAIN_LIST_SEPARATOR).map(&:strip)
domain_list.reject! { |d| d.empty? }
domain_list
end
end
Loading
Loading
@@ -6,6 +6,8 @@ module ApplicationSettings
 
attr_reader :params, :application_setting
 
MARKDOWN_CACHE_INVALIDATING_PARAMS = %w(asset_proxy_enabled asset_proxy_url asset_proxy_secret_key asset_proxy_whitelist).freeze
def execute
validate_classification_label(application_setting, :external_authorization_service_default_label)
 
Loading
Loading
@@ -23,7 +25,13 @@ module ApplicationSettings
params[:usage_stats_set_by_user_id] = current_user.id
end
 
@application_setting.update(@params)
@application_setting.assign_attributes(params)
if invalidate_markdown_cache?
@application_setting[:local_markdown_version] = @application_setting.local_markdown_version + 1
end
@application_setting.save
end
 
private
Loading
Loading
@@ -32,6 +40,11 @@ module ApplicationSettings
params.key?(:usage_ping_enabled) || params.key?(:version_check_enabled)
end
 
def invalidate_markdown_cache?
!params.key?(:local_markdown_version) &&
(@application_setting.changes.keys & MARKDOWN_CACHE_INVALIDATING_PARAMS).any?
end
def update_terms(terms)
return unless terms.present?
 
Loading
Loading
---
title: Added image proxy to mitigate potential stealing of IP addresses
merge_request:
author:
type: security
#
# Asset proxy settings
#
ActiveSupport.on_load(:active_record) do
Banzai::Filter::AssetProxyFilter.initialize_settings
end
if Shard.connected? && !Gitlab::Database.read_only?
# The `table_exists?` check is needed because during our migration rollback testing,
# `Shard.connected?` could be cached and return true even though the table doesn't exist
if Shard.connected? && Shard.table_exists? && !Gitlab::Database.read_only?
Shard.populate!
end
# frozen_string_literal: true
class AddAssetProxySettings < ActiveRecord::Migration[5.0]
include Gitlab::Database::MigrationHelpers
# Set this constant to true if this migration requires downtime.
DOWNTIME = false
def change
add_column :application_settings, :asset_proxy_enabled, :boolean, default: false, null: false
add_column :application_settings, :asset_proxy_url, :string
add_column :application_settings, :asset_proxy_whitelist, :text
add_column :application_settings, :encrypted_asset_proxy_secret_key, :text
add_column :application_settings, :encrypted_asset_proxy_secret_key_iv, :string
end
end
Loading
Loading
@@ -228,6 +228,11 @@ ActiveRecord::Schema.define(version: 20190816151221) do
t.integer "custom_project_templates_group_id"
t.boolean "elasticsearch_limit_indexing", default: false, null: false
t.string "geo_node_allowed_ips", default: "0.0.0.0/0, ::/0"
t.boolean "asset_proxy_enabled", default: false, null: false
t.string "asset_proxy_url"
t.text "asset_proxy_whitelist"
t.text "encrypted_asset_proxy_secret_key"
t.string "encrypted_asset_proxy_secret_key_iv"
t.index ["custom_project_templates_group_id"], name: "index_application_settings_on_custom_project_templates_group_id", using: :btree
t.index ["file_template_project_id"], name: "index_application_settings_on_file_template_project_id", using: :btree
t.index ["usage_stats_set_by_user_id"], name: "index_application_settings_on_usage_stats_set_by_user_id", using: :btree
Loading
Loading
Loading
Loading
@@ -63,7 +63,10 @@ Example response:
"performance_bar_allowed_group_id": 42,
"instance_statistics_visibility_private": false,
"user_show_add_ssh_key_message": true,
"local_markdown_version": 0
"local_markdown_version": 0,
"asset_proxy_enabled": true,
"asset_proxy_url": "https://assets.example.com",
"asset_proxy_whitelist": ["example.com", "*.example.com", "your-instance.com"]
}
```
 
Loading
Loading
@@ -121,7 +124,10 @@ Example response:
"performance_bar_allowed_group_id": 42,
"instance_statistics_visibility_private": false,
"user_show_add_ssh_key_message": true,
"local_markdown_version": 0
"local_markdown_version": 0,
"asset_proxy_enabled": true,
"asset_proxy_url": "https://assets.example.com",
"asset_proxy_whitelist": ["example.com", "*.example.com", "your-instance.com"],
}
```
 
Loading
Loading
@@ -139,6 +145,10 @@ are listed in the descriptions of the relevant settings.
| `akismet_api_key` | string | required by: `akismet_enabled` | API key for akismet spam protection. |
| `akismet_enabled` | boolean | no | (**If enabled, requires:** `akismet_api_key`) Enable or disable akismet spam protection. |
| `allow_local_requests_from_hooks_and_services` | boolean | no | Allow requests to the local network from hooks and services. |
| `asset_proxy_enabled` | boolean | no | (**If enabled, requires:** `asset_proxy_url`) Enable proxying of assets. GitLab restart is required to apply changes. |
| `asset_proxy_secret_key` | string | no | Shared secret with the asset proxy server. GitLab restart is required to apply changes. |
| `asset_proxy_url` | string | no | URL of the asset proxy server. GitLab restart is required to apply changes. |
| `asset_proxy_whitelist` | string or array of strings | no | Assets that match these domain(s) will NOT be proxied. Wildcards allowed. Your GitLab installation URL is automatically whitelisted. GitLab restart is required to apply changes. |
| `authorized_keys_enabled` | boolean | no | By default, we write to the `authorized_keys` file to support Git over SSH without additional configuration. GitLab can be optimized to authenticate SSH keys via the database file. Only disable this if you have configured your OpenSSH server to use the AuthorizedKeysCommand. |
| `auto_devops_domain` | string | no | Specify a domain to use by default for every project's Auto Review Apps and Auto Deploy stages. |
| `auto_devops_enabled` | boolean | no | Enable Auto DevOps for projects by default. It will automatically build, test, and deploy applications based on a predefined CI/CD configuration. |
Loading
Loading
@@ -161,7 +171,7 @@ are listed in the descriptions of the relevant settings.
| `email_author_in_body` | boolean | no | Some email servers do not support overriding the email sender name. Enable this option to include the name of the author of the issue, merge request or comment in the email body instead. |
| `enabled_git_access_protocol` | string | no | Enabled protocols for Git access. Allowed values are: `ssh`, `http`, and `nil` to allow both protocols. |
| `enforce_terms` | boolean | no | (**If enabled, requires:** `terms`) Enforce application ToS to all users. |
| `first_day_of_week` | integer | no | Start day of the week for calendar views and date pickers. Valid values are `0` (default) for Sunday, `1` for Monday, and `6` for Saturday. |
| `first_day_of_week` | integer | no | Start day of the week for calendar views and date pickers. Valid values are `0` (default) for Sunday, `1` for Monday, and `6` for Saturday. |
| `gitaly_timeout_default` | integer | no | Default Gitaly timeout, in seconds. This timeout is not enforced for git fetch/push operations or Sidekiq jobs. Set to `0` to disable timeouts. |
| `gitaly_timeout_fast` | integer | no | Gitaly fast operation timeout, in seconds. Some Gitaly operations are expected to be fast. If they exceed this threshold, there may be a problem with a storage shard and 'failing fast' can help maintain the stability of the GitLab instance. Set to `0` to disable timeouts. |
| `gitaly_timeout_medium` | integer | no | Medium Gitaly timeout, in seconds. This should be a value between the Fast and the Default timeout. Set to `0` to disable timeouts. |
Loading
Loading
Loading
Loading
@@ -17,3 +17,4 @@ type: index
- [Enforce Two-factor authentication](two_factor_authentication.md)
- [Send email confirmation on sign-up](user_email_confirmation.md)
- [Security of running jobs](https://docs.gitlab.com/runner/security/)
- [Proxying images](asset_proxy.md)
A possible security concern when managing a public facing GitLab instance is
the ability to steal a users IP address by referencing images in issues, comments, etc.
For example, adding `![Example image](http://example.com/example.png)` to
an issue description will cause the image to be loaded from the external
server in order to be displayed. However this also allows the external server
to log the IP address of the user.
One way to mitigate this is by proxying any external images to a server you
control. GitLab handles this by allowing you to run the "Camo" server
[cactus/go-camo](https://github.com/cactus/go-camo#how-it-works).
The image request is sent to the Camo server, which then makes the request for
the original image. This way an attacker only ever seems the IP address
of your Camo server.
Once you have your Camo server up and running, you can configure GitLab to
proxy image requests to it. The following settings are supported:
| Attribute | Description |
| --------- | ----------- |
| `asset_proxy_enabled` | (**If enabled, requires:** `asset_proxy_url`) Enable proxying of assets. |
| `asset_proxy_secret_key` | Shared secret with the asset proxy server. |
| `asset_proxy_url` | URL of the asset proxy server. |
| `asset_proxy_whitelist` | Assets that match these domain(s) will NOT be proxied. Wildcards allowed. Your GitLab installation URL is automatically whitelisted. |
These can be set via the [Application setting API](../api/settings.md)
Note that a GitLab restart is required to apply any changes.
Loading
Loading
@@ -1141,6 +1141,9 @@ module API
attributes.delete(:performance_bar_allowed_group_path)
attributes.delete(:performance_bar_enabled)
 
# let's not expose the secret key in a response
attributes.delete(:asset_proxy_secret_key)
attributes
end
 
Loading
Loading
Loading
Loading
@@ -36,6 +36,10 @@ module API
given akismet_enabled: ->(val) { val } do
requires :akismet_api_key, type: String, desc: 'Generate API key at http://www.akismet.com'
end
optional :asset_proxy_enabled, type: Boolean, desc: 'Enable proxying of assets'
optional :asset_proxy_url, type: String, desc: 'URL of the asset proxy server'
optional :asset_proxy_secret_key, type: String, desc: 'Shared secret with the asset proxy server'
optional :asset_proxy_whitelist, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Assets that match these domain(s) will NOT be proxied. Wildcards allowed. Your GitLab installation URL is automatically whitelisted.'
optional :clientside_sentry_enabled, type: Boolean, desc: 'Sentry can also be used for reporting and logging clientside exceptions. https://sentry.io/for/javascript/'
given clientside_sentry_enabled: ->(val) { val } do
requires :clientside_sentry_dsn, type: String, desc: 'Clientside Sentry Data Source Name'
Loading
Loading
@@ -129,7 +133,7 @@ module API
optional :terminal_max_session_time, type: Integer, desc: 'Maximum time for web terminal websocket connection (in seconds). Set to 0 for unlimited time.'
optional :usage_ping_enabled, type: Boolean, desc: 'Every week GitLab will report license usage back to GitLab, Inc.'
optional :instance_statistics_visibility_private, type: Boolean, desc: 'When set to `true` Instance statistics will only be available to admins'
optional :local_markdown_version, type: Integer, desc: "Local markdown version, increase this value when any cached markdown should be invalidated"
optional :local_markdown_version, type: Integer, desc: 'Local markdown version, increase this value when any cached markdown should be invalidated'
 
ApplicationSetting::SUPPORTED_KEY_TYPES.each do |type|
optional :"#{type}_key_restriction",
Loading
Loading
# frozen_string_literal: true
module API
module Validations
module Types
class CommaSeparatedToArray
def self.coerce
lambda do |value|
case value
when String
value.split(',').map(&:strip)
when Array
value.map { |v| v.to_s.split(',').map(&:strip) }.flatten
else
[]
end
end
end
end
end
end
end
# frozen_string_literal: true
module Banzai
module Filter
# Proxy's images/assets to another server. Reduces mixed content warnings
# as well as hiding the customer's IP address when requesting images.
# Copies the original img `src` to `data-canonical-src` then replaces the
# `src` with a new url to the proxy server.
class AssetProxyFilter < HTML::Pipeline::CamoFilter
def initialize(text, context = nil, result = nil)
super
end
def validate
needs(:asset_proxy, :asset_proxy_secret_key) if asset_proxy_enabled?
end
def asset_host_whitelisted?(host)
context[:asset_proxy_domain_regexp] ? context[:asset_proxy_domain_regexp].match?(host) : false
end
def self.transform_context(context)
context[:disable_asset_proxy] = !Gitlab.config.asset_proxy.enabled
unless context[:disable_asset_proxy]
context[:asset_proxy_enabled] = !context[:disable_asset_proxy]
context[:asset_proxy] = Gitlab.config.asset_proxy.url
context[:asset_proxy_secret_key] = Gitlab.config.asset_proxy.secret_key
context[:asset_proxy_domain_regexp] = Gitlab.config.asset_proxy.domain_regexp
end
context
end
# called during an initializer. Caching the values in Gitlab.config significantly increased
# performance, rather than querying Gitlab::CurrentSettings.current_application_settings
# over and over. However, this does mean that the Rails servers need to get restarted
# whenever the application settings are changed
def self.initialize_settings
application_settings = Gitlab::CurrentSettings.current_application_settings
Gitlab.config['asset_proxy'] ||= Settingslogic.new({})
if application_settings.respond_to?(:asset_proxy_enabled)
Gitlab.config.asset_proxy['enabled'] = application_settings.asset_proxy_enabled
Gitlab.config.asset_proxy['url'] = application_settings.asset_proxy_url
Gitlab.config.asset_proxy['secret_key'] = application_settings.asset_proxy_secret_key
Gitlab.config.asset_proxy['whitelist'] = application_settings.asset_proxy_whitelist || [Gitlab.config.gitlab.host]
Gitlab.config.asset_proxy['domain_regexp'] = compile_whitelist(Gitlab.config.asset_proxy.whitelist)
else
Gitlab.config.asset_proxy['enabled'] = ::ApplicationSetting.defaults[:asset_proxy_enabled]
end
end
def self.compile_whitelist(domain_list)
return if domain_list.empty?
escaped = domain_list.map { |domain| Regexp.escape(domain).gsub('\*', '.*?') }
Regexp.new("^(#{escaped.join('|')})$", Regexp::IGNORECASE)
end
end
end
end
Loading
Loading
@@ -14,10 +14,10 @@ module Banzai
# such as on `mailto:` links. Since we've been using it, do an
# initial parse for validity and then use Addressable
# for IDN support, etc
uri = uri_strict(node['href'].to_s)
uri = uri_strict(node_src(node))
if uri
node.set_attribute('href', uri.to_s)
addressable_uri = addressable_uri(node['href'])
node.set_attribute(node_src_attribute(node), uri.to_s)
addressable_uri = addressable_uri(node_src(node))
else
addressable_uri = nil
end
Loading
Loading
@@ -35,6 +35,16 @@ module Banzai
 
private
 
# if this is a link to a proxied image, then `src` is already the correct
# proxied url, so work with the `data-canonical-src`
def node_src_attribute(node)
node['data-canonical-src'] ? 'data-canonical-src' : 'href'
end
def node_src(node)
node[node_src_attribute(node)]
end
def uri_strict(href)
URI.parse(href)
rescue URI::Error
Loading
Loading
@@ -72,7 +82,7 @@ module Banzai
return unless uri
return unless context[:emailable_links]
 
unencoded_uri_str = Addressable::URI.unencode(node['href'])
unencoded_uri_str = Addressable::URI.unencode(node_src(node))
 
if unencoded_uri_str == node.content && idn?(uri)
node.content = uri.normalize
Loading
Loading
Loading
Loading
@@ -18,6 +18,9 @@ module Banzai
rel: 'noopener noreferrer'
)
 
# make sure the original non-proxied src carries over to the link
link['data-canonical-src'] = img['data-canonical-src'] if img['data-canonical-src']
link.children = img.clone
 
img.replace(link)
Loading
Loading
Loading
Loading
@@ -23,6 +23,14 @@ module Banzai
"'.#{ext}' = substring(@src, string-length(@src) - #{ext.size})"
end
 
if context[:asset_proxy_enabled].present?
src_query.concat(
UploaderHelper::VIDEO_EXT.map do |ext|
"'.#{ext}' = substring(@data-canonical-src, string-length(@data-canonical-src) - #{ext.size})"
end
)
end
"descendant-or-self::img[not(ancestor::a) and (#{src_query.join(' or ')})]"
end
end
Loading
Loading
@@ -48,6 +56,13 @@ module Banzai
target: '_blank',
rel: 'noopener noreferrer',
title: "Download '#{element['title'] || element['alt']}'")
# make sure the original non-proxied src carries over
if element['data-canonical-src']
video['data-canonical-src'] = element['data-canonical-src']
link['data-canonical-src'] = element['data-canonical-src']
end
download_paragraph = doc.document.create_element('p')
download_paragraph.children = link
 
Loading
Loading
Loading
Loading
@@ -6,11 +6,16 @@ module Banzai
def self.filters
FilterArray[
Filter::SanitizationFilter,
Filter::AssetProxyFilter,
Filter::ExternalLinkFilter,
Filter::PlantumlFilter,
Filter::AsciiDocPostProcessingFilter
]
end
def self.transform_context(context)
Filter::AssetProxyFilter.transform_context(context)
end
end
end
end
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment