Merge branch 'jej-pages-to-ce' into 'master'

Adding GitLab Pages to CE

Closes #14605, gitlab-com/infrastructure#1058, gitlab-ee#1333, and #323

See merge request !8463

GITLAB_PAGES_VERSION

+0.2.4

Gemfile

@@ -47,6 +47,9 @@ gem 'rqrcode-rails3', '~> 0.1.7'
 gem 'attr_encrypted', '~> 3.0.0'
 gem 'u2f', '~> 0.2.1'
+# GitLab Pages
+gem 'validates_hostname', '~> 1.0.6'
+
 # Browser detection
 gem 'browser', '~> 2.2'

Gemfile.lock

@@ -785,6 +785,9 @@ GEM
       get_process_mem (~> 0)
       unicorn (>= 4, < 6)
     uniform_notifier (1.10.0)
+    validates_hostname (1.0.6)
+      activerecord (>= 3.0)
+      activesupport (>= 3.0)
     version_sorter (2.1.0)
     virtus (1.0.5)
       axiom-types (~> 0.1)
@@ -998,6 +1001,7 @@ DEPENDENCIES
   unf (~> 0.1.4)
   unicorn (~> 5.1.0)
   unicorn-worker-killer (~> 0.4.4)
+  validates_hostname (~> 1.0.6)
   version_sorter (~> 2.1.0)
   virtus (~> 1.0.1)
   vmstat (~> 2.3.0)

app/controllers/admin/application_settings_controller.rb

@@ -109,6 +109,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
       :plantuml_url,
       :max_artifacts_size,
       :max_attachment_size,
+      :max_pages_size,
       :metrics_enabled,
       :metrics_host,
       :metrics_method_call_threshold,

app/controllers/projects/pages_controller.rb

+class Projects::PagesController < Projects::ApplicationController
+  layout 'project_settings'
+
+  before_action :authorize_read_pages!, only: [:show]
+  before_action :authorize_update_pages!, except: [:show]
+
+  def show
+    @domains = @project.pages_domains.order(:domain)
+  end
+
+  def destroy
+    project.remove_pages
+    project.pages_domains.destroy_all
+
+    respond_to do |format|
+      format.html  do
+        redirect_to(namespace_project_pages_path(@project.namespace, @project),
+                    notice: 'Pages were removed')
+      end
+    end
+  end
+end

app/controllers/projects/pages_domains_controller.rb

+class Projects::PagesDomainsController < Projects::ApplicationController
+  layout 'project_settings'
+
+  before_action :authorize_update_pages!, except: [:show]
+  before_action :domain, only: [:show, :destroy]
+
+  def show
+  end
+
+  def new
+    @domain = @project.pages_domains.new
+  end
+
+  def create
+    @domain = @project.pages_domains.create(pages_domain_params)
+
+    if @domain.valid?
+      redirect_to namespace_project_pages_path(@project.namespace, @project)
+    else
+      render 'new'
+    end
+  end
+
+  def destroy
+    @domain.destroy
+
+    respond_to do |format|
+      format.html do
+        redirect_to(namespace_project_pages_path(@project.namespace, @project),
+                    notice: 'Domain was removed')
+      end
+      format.js
+    end
+  end
+
+  private
+
+  def pages_domain_params
+    params.require(:pages_domain).permit(
+      :certificate,
+      :key,
+      :domain
+    )
+  end
+
+  def domain
+    @domain ||= @project.pages_domains.find_by(domain: params[:id].to_s)
+  end
+end

app/models/ci/build.rb

@@ -256,7 +256,7 @@ module Ci
     end
     def project_id
-      pipeline.project_id
+      gl_project_id
     end
     def project_name
@@ -451,6 +451,7 @@ module Ci
       build_data = Gitlab::DataBuilder::Build.build(self)
       project.execute_hooks(build_data.dup, :build_hooks)
       project.execute_services(build_data.dup, :build_hooks)
+      PagesService.new(build_data).execute
       project.running_or_pending_build_count(force: true)
     end

app/models/namespace.rb

@@ -130,6 +130,7 @@ class Namespace < ActiveRecord::Base
     end
     Gitlab::UploadsTransfer.new.rename_namespace(path_was, path)
+    Gitlab::PagesTransfer.new.rename_namespace(path_was, path)
     remove_exports!

app/models/pages_domain.rb

+class PagesDomain < ActiveRecord::Base
+  belongs_to :project
+
+  validates :domain, hostname: true
+  validates_uniqueness_of :domain, case_sensitive: false
+  validates :certificate, certificate: true, allow_nil: true, allow_blank: true
+  validates :key, certificate_key: true, allow_nil: true, allow_blank: true
+
+  validate :validate_pages_domain
+  validate :validate_matching_key, if: ->(domain) { domain.certificate.present? || domain.key.present? }
+  validate :validate_intermediates, if: ->(domain) { domain.certificate.present? }
+
+  attr_encrypted :key,
+    mode: :per_attribute_iv_and_salt,
+    insecure_mode: true,
+    key: Gitlab::Application.secrets.db_key_base,
+    algorithm: 'aes-256-cbc'
+
+  after_create :update
+  after_save :update
+  after_destroy :update
+
+  def to_param
+    domain
+  end
+
+  def url
+    return unless domain
+
+    if certificate
+      "https://#{domain}"
+    else
+      "http://#{domain}"
+    end
+  end
+
+  def has_matching_key?
+    return false unless x509
+    return false unless pkey
+
+    # We compare the public key stored in certificate with public key from certificate key
+    x509.check_private_key(pkey)
+  end
+
+  def has_intermediates?
+    return false unless x509
+
+    # self-signed certificates doesn't have the certificate chain
+    return true if x509.verify(x509.public_key)
+
+    store = OpenSSL::X509::Store.new
+    store.set_default_paths
+
+    # This forces to load all intermediate certificates stored in `certificate`
+    Tempfile.open('certificate_chain') do |f|
+      f.write(certificate)
+      f.flush
+      store.add_file(f.path)
+    end
+
+    store.verify(x509)
+  rescue OpenSSL::X509::StoreError
+    false
+  end
+
+  def expired?
+    return false unless x509
+    current = Time.new
+    current < x509.not_before || x509.not_after < current
+  end
+
+  def subject
+    return unless x509
+    x509.subject.to_s
+  end
+
+  def certificate_text
+    @certificate_text ||= x509.try(:to_text)
+  end
+
+  private
+
+  def update
+    ::Projects::UpdatePagesConfigurationService.new(project).execute
+  end
+
+  def validate_matching_key
+    unless has_matching_key?
+      self.errors.add(:key, "doesn't match the certificate")
+    end
+  end
+
+  def validate_intermediates
+    unless has_intermediates?
+      self.errors.add(:certificate, 'misses intermediates')
+    end
+  end
+
+  def validate_pages_domain
+    return unless domain
+    if domain.downcase.ends_with?(".#{Settings.pages.host}".downcase)
+      self.errors.add(:domain, "*.#{Settings.pages.host} is restricted")
+    end
+  end
+
+  def x509
+    return unless certificate
+    @x509 ||= OpenSSL::X509::Certificate.new(certificate)
+  rescue OpenSSL::X509::CertificateError
+    nil
+  end
+
+  def pkey
+    return unless key
+    @pkey ||= OpenSSL::PKey::RSA.new(key)
+  rescue OpenSSL::PKey::PKeyError, OpenSSL::Cipher::CipherError
+    nil
+  end
+end

app/models/project.rb

@@ -53,6 +53,8 @@ class Project < ActiveRecord::Base
     update_column(:last_activity_at, self.created_at)
   end
+  after_destroy :remove_pages
+
   # update visibility_level of forks
   after_update :update_forks_visibility_level
   def update_forks_visibility_level
@@ -148,6 +150,7 @@ class Project < ActiveRecord::Base
   has_many :lfs_objects, through: :lfs_objects_projects
   has_many :project_group_links, dependent: :destroy
   has_many :invited_groups, through: :project_group_links, source: :group
+  has_many :pages_domains, dependent: :destroy
   has_many :todos, dependent: :destroy
   has_many :notification_settings, dependent: :destroy, as: :source
@@ -955,6 +958,7 @@ class Project < ActiveRecord::Base
     Gitlab::AppLogger.info "Project was renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}"
     Gitlab::UploadsTransfer.new.rename_project(path_was, path, namespace.path)
+    Gitlab::PagesTransfer.new.rename_project(path_was, path, namespace.path)
   end
   # Expires various caches before a project is renamed.
@@ -1156,6 +1160,45 @@ class Project < ActiveRecord::Base
     ensure_runners_token!
   end
+  def pages_deployed?
+    Dir.exist?(public_pages_path)
+  end
+
+  def pages_url
+    # The hostname always needs to be in downcased
+    # All web servers convert hostname to lowercase
+    host = "#{namespace.path}.#{Settings.pages.host}".downcase
+
+    # The host in URL always needs to be downcased
+    url = Gitlab.config.pages.url.sub(/^https?:\/\//) do |prefix|
+      "#{prefix}#{namespace.path}."
+    end.downcase
+
+    # If the project path is the same as host, we serve it as group page
+    return url if host == path
+
+    "#{url}/#{path}"
+  end
+
+  def pages_path
+    File.join(Settings.pages.path, path_with_namespace)
+  end
+
+  def public_pages_path
+    File.join(pages_path, 'public')
+  end
+
+  def remove_pages
+    # 1. We rename pages to temporary directory
+    # 2. We wait 5 minutes, due to NFS caching
+    # 3. We asynchronously remove pages with force
+    temp_path = "#{path}.#{SecureRandom.hex}.deleted"
+
+    if Gitlab::PagesTransfer.new.rename_project(path, temp_path, namespace.path)
+      PagesWorker.perform_in(5.minutes, :remove, namespace.path, temp_path)
+    end
+  end
+
   def wiki
     @wiki ||= ProjectWiki.new(self, self.owner)
   end

app/policies/project_policy.rb

@@ -110,6 +110,9 @@ class ProjectPolicy < BasePolicy
     can! :admin_pipeline
     can! :admin_environment
     can! :admin_deployment
+    can! :admin_pages
+    can! :read_pages
+    can! :update_pages
   end
   def public_access!
@@ -136,6 +139,7 @@ class ProjectPolicy < BasePolicy
     can! :remove_fork_project
     can! :destroy_merge_request
     can! :destroy_issue
+    can! :remove_pages
   end
   def team_member_owner_access!

app/services/pages_service.rb

+class PagesService
+  attr_reader :data
+
+  def initialize(data)
+    @data = data
+  end
+
+  def execute
+    return unless Settings.pages.enabled
+    return unless data[:build_name] == 'pages'
+    return unless data[:build_status] == 'success'
+
+    PagesWorker.perform_async(:deploy, data[:build_id])
+  end
+end

app/services/projects/transfer_service.rb

@@ -64,6 +64,9 @@ module Projects
         # Move uploads
         Gitlab::UploadsTransfer.new.move_project(project.path, old_namespace.path, new_namespace.path)
+        # Move pages
+        Gitlab::PagesTransfer.new.move_project(project.path, old_namespace.path, new_namespace.path)
+
         project.old_path_with_namespace = old_path
         SystemHooksService.new.execute_hooks_for(project, :transfer)

app/services/projects/update_pages_configuration_service.rb

+module Projects
+  class UpdatePagesConfigurationService < BaseService
+    attr_reader :project
+
+    def initialize(project)
+      @project = project
+    end
+
+    def execute
+      update_file(pages_config_file, pages_config.to_json)
+      reload_daemon
+      success
+    rescue => e
+      error(e.message)
+    end
+
+    private
+
+    def pages_config
+      {
+        domains: pages_domains_config
+      }
+    end
+
+    def pages_domains_config
+      project.pages_domains.map do |domain|
+        {
+          domain: domain.domain,
+          certificate: domain.certificate,
+          key: domain.key,
+        }
+      end
+    end
+
+    def reload_daemon
+      # GitLab Pages daemon constantly watches for modification time of `pages.path`
+      # It reloads configuration when `pages.path` is modified
+      update_file(pages_update_file, SecureRandom.hex(64))
+    end
+
+    def pages_path
+      @pages_path ||= project.pages_path
+    end
+
+    def pages_config_file
+      File.join(pages_path, 'config.json')
+    end
+
+    def pages_update_file
+      File.join(::Settings.pages.path, '.update')
+    end
+
+    def update_file(file, data)
+      unless data
+        FileUtils.remove(file, force: true)
+        return
+      end
+
+      temp_file = "#{file}.#{SecureRandom.hex(16)}"
+      File.open(temp_file, 'w') do |f|
+        f.write(data)
+      end
+      FileUtils.move(temp_file, file, force: true)
+    ensure
+      # In case if the updating fails
+      FileUtils.remove(temp_file, force: true)
+    end
+  end
+end

app/services/projects/update_pages_service.rb

+module Projects
+  class UpdatePagesService < BaseService
+    BLOCK_SIZE = 32.kilobytes
+    MAX_SIZE = 1.terabyte
+    SITE_PATH = 'public/'
+
+    attr_reader :build
+
+    def initialize(project, build)
+      @project, @build = project, build
+    end
+
+    def execute
+      # Create status notifying the deployment of pages
+      @status = create_status
+      @status.enqueue!
+      @status.run!
+
+      raise 'missing pages artifacts' unless build.artifacts_file?
+      raise 'pages are outdated' unless latest?
+
+      # Create temporary directory in which we will extract the artifacts
+      FileUtils.mkdir_p(tmp_path)
+      Dir.mktmpdir(nil, tmp_path) do |archive_path|
+        extract_archive!(archive_path)
+
+        # Check if we did extract public directory
+        archive_public_path = File.join(archive_path, 'public')
+        raise 'pages miss the public folder' unless Dir.exist?(archive_public_path)
+        raise 'pages are outdated' unless latest?
+
+        deploy_page!(archive_public_path)
+        success
+      end
+    rescue => e
+      error(e.message)
+    end
+
+    private
+
+    def success
+      @status.success
+      super
+    end
+
+    def error(message, http_status = nil)
+      @status.allow_failure = !latest?
+      @status.description = message
+      @status.drop
+      super
+    end
+
+    def create_status
+      GenericCommitStatus.new(
+        project: project,
+        pipeline: build.pipeline,
+        user: build.user,
+        ref: build.ref,
+        stage: 'deploy',
+        name: 'pages:deploy'
+      )
+    end
+
+    def extract_archive!(temp_path)
+      if artifacts.ends_with?('.tar.gz') || artifacts.ends_with?('.tgz')
+        extract_tar_archive!(temp_path)
+      elsif artifacts.ends_with?('.zip')
+        extract_zip_archive!(temp_path)
+      else
+        raise 'unsupported artifacts format'
+      end
+    end
+
+    def extract_tar_archive!(temp_path)
+      results = Open3.pipeline(%W(gunzip -c #{artifacts}),
+                               %W(dd bs=#{BLOCK_SIZE} count=#{blocks}),
+                               %W(tar -x -C #{temp_path} #{SITE_PATH}),
+                               err: '/dev/null')
+      raise 'pages failed to extract' unless results.compact.all?(&:success?)
+    end
+
+    def extract_zip_archive!(temp_path)
+      raise 'missing artifacts metadata' unless build.artifacts_metadata?
+
+      # Calculate page size after extract
+      public_entry = build.artifacts_metadata_entry(SITE_PATH, recursive: true)
+
+      if public_entry.total_size > max_size
+        raise "artifacts for pages are too large: #{public_entry.total_size}"
+      end
+
+      # Requires UnZip at least 6.00 Info-ZIP.
+      # -n  never overwrite existing files
+      # We add * to end of SITE_PATH, because we want to extract SITE_PATH and all subdirectories
+      site_path = File.join(SITE_PATH, '*')
+      unless system(*%W(unzip -n #{artifacts} #{site_path} -d #{temp_path}))
+        raise 'pages failed to extract'
+      end
+    end
+
+    def deploy_page!(archive_public_path)
+      # Do atomic move of pages
+      # Move and removal may not be atomic, but they are significantly faster then extracting and removal
+      # 1. We move deployed public to previous public path (file removal is slow)
+      # 2. We move temporary public to be deployed public
+      # 3. We remove previous public path
+      FileUtils.mkdir_p(pages_path)
+      begin
+        FileUtils.move(public_path, previous_public_path)
+      rescue
+      end
+      FileUtils.move(archive_public_path, public_path)
+    ensure
+      FileUtils.rm_r(previous_public_path, force: true)
+    end
+
+    def latest?
+      # check if sha for the ref is still the most recent one
+      # this helps in case when multiple deployments happens
+      sha == latest_sha
+    end
+
+    def blocks
+      # Calculate dd parameters: we limit the size of pages
+      1 + max_size / BLOCK_SIZE
+    end
+
+    def max_size
+      current_application_settings.max_pages_size.megabytes || MAX_SIZE
+    end
+
+    def tmp_path
+      @tmp_path ||= File.join(::Settings.pages.path, 'tmp')
+    end
+
+    def pages_path
+      @pages_path ||= project.pages_path
+    end
+
+    def public_path
+      @public_path ||= File.join(pages_path, 'public')
+    end
+
+    def previous_public_path
+      @previous_public_path ||= File.join(pages_path, "public.#{SecureRandom.hex}")
+    end
+
+    def ref
+      build.ref
+    end
+
+    def artifacts
+      build.artifacts_file.path
+    end
+
+    def latest_sha
+      project.commit(build.ref).try(:sha).to_s
+    end
+
+    def sha
+      build.sha
+    end
+  end
+end

app/validators/certificate_key_validator.rb

+# UrlValidator
+#
+# Custom validator for private keys.
+#
+#   class Project < ActiveRecord::Base
+#     validates :certificate_key, certificate_key: true
+#   end
+#
+class CertificateKeyValidator < ActiveModel::EachValidator
+  def validate_each(record, attribute, value)
+    unless valid_private_key_pem?(value)
+      record.errors.add(attribute, "must be a valid PEM private key")
+    end
+  end
+
+  private
+
+  def valid_private_key_pem?(value)
+    return false unless value
+    pkey = OpenSSL::PKey::RSA.new(value)
+    pkey.private?
+  rescue OpenSSL::PKey::PKeyError
+    false
+  end
+end

app/validators/certificate_validator.rb

+# UrlValidator
+#
+# Custom validator for private keys.
+#
+#   class Project < ActiveRecord::Base
+#     validates :certificate_key, certificate: true
+#   end
+#
+class CertificateValidator < ActiveModel::EachValidator
+  def validate_each(record, attribute, value)
+    unless valid_certificate_pem?(value)
+      record.errors.add(attribute, "must be a valid PEM certificate")
+    end
+  end
+
+  private
+
+  def valid_certificate_pem?(value)
+    return false unless value
+    OpenSSL::X509::Certificate.new(value).present?
+  rescue OpenSSL::X509::CertificateError
+    false
+  end
+end

app/views/admin/application_settings/_form.html.haml

@@ -186,6 +186,14 @@
         = f.text_area :help_page_text, class: 'form-control', rows: 4
         .help-block Markdown enabled
+  %fieldset
+    %legend Pages
+    .form-group
+      = f.label :max_pages_size, 'Maximum size of pages (MB)', class: 'control-label col-sm-2'
+      .col-sm-10
+        = f.number_field :max_pages_size, class: 'form-control'
+        .help-block Zero for unlimited
+
   %fieldset
     %legend Continuous Integration
     .form-group

app/views/layouts/nav/_project_settings.html.haml

@@ -34,3 +34,7 @@
       = link_to namespace_project_pipelines_settings_path(@project.namespace, @project), title: 'CI/CD Pipelines' do
         %span
           CI/CD Pipelines
+  = nav_link(controller: :pages) do
+    = link_to namespace_project_pages_path(@project.namespace, @project), title: 'Pages', data: {placement: 'right'} do
+      %span
+        Pages

app/views/projects/edit.html.haml

@@ -133,6 +133,7 @@
               %hr
               = link_to 'Remove avatar', namespace_project_avatar_path(@project.namespace, @project), data: { confirm: "Project avatar will be removed. Are you sure?"}, method: :delete, class: "btn btn-remove btn-sm remove-avatar"
         = f.submit 'Save changes', class: "btn btn-save"
+
   .row.prepend-top-default
   %hr
   .row.prepend-top-default