CHANGELOG.md

@@ -3,6 +3,26 @@
 The latest version of this file can be found at the master branch of the
 omnibus-gitlab repository.
+## 13.4.7 (2020-12-07)
+
+### Security (3 changes)
+
+- Update GnuPG to version 2.2.23.
+- Update libxml2 to version 2.9.10.
+- Update GraphicsMagick to 1.3.35 and patch PNG vulnerability.
+
+
+## 13.4.6 (2020-11-03)
+
+### Fixed (1 change)
+
+- Geo: Perform point-in-time recovery before promotion of secondary node. !4636
+
+
+## 13.4.5 (2020-11-02)
+
+- No changes.
+
 ## 13.4.4 (2020-10-15)
 ### Fixed (1 change)

GITALY_SERVER_VERSION

-13.4.4
+13.4.7
\ No newline at end of file

VERSION

-13.4.4-ee
+13.4.7-ee
\ No newline at end of file

config/patches/graphicsmagick/png-vulnerability-fix.patch

+Patch adapted from https://sourceforge.net/p/graphicsmagick/code/ci/50395430a37188d0d197e71bd85ed6dd0f649ee3/.
+
+diff --git a/coders/png.c b/coders/png.c
+index 60ce8dc..5878e60 100644
+--- a/coders/png.c
++++ b/coders/png.c
+@@ -5304,7 +5304,7 @@ static Image *ReadMNGImage(const ImageInfo *image_info,
+               if (logging)
+                 (void) LogMagickEvent(CoderEvent,GetMagickModule(),
+                                       "MAGN chunk (%lu bytes): "
+-                                      "First_magnified_object_id=%u, Last_magnified_object_id=%u, "
++                                      "First_magnified_object_id=%u, Las t_magnified_object_id=%u, "
+                                       "MB=%u, ML=%u, MR=%u, MT=%u, MX=%u, MY=%u, "
+                                       "X_method=%u, Y_method=%u",
+                                       length,
+@@ -5679,6 +5679,8 @@ static Image *ReadMNGImage(const ImageInfo *image_info,
+           /*
+             If magnifying and a supported method is requested then
+             magnify the image.
++
++            http://www.libpng.org/pub/mng/spec/mng-1.0-20010209-pdg.html#mng-MAGN
+           */
+           if (((mng_info->magn_methx > 0) && (mng_info->magn_methx <= 5)) &&
+               ((mng_info->magn_methy > 0) && (mng_info->magn_methy <= 5)))
+@@ -5689,7 +5691,28 @@ static Image *ReadMNGImage(const ImageInfo *image_info,
+ 
+               if (logging)
+                 (void) LogMagickEvent(CoderEvent,GetMagickModule(),
+-                                      "  Processing MNG MAGN chunk");
++                                      "  Processing MNG MAGN chunk: MB=%u, ML=%u,"
++                                      " MR=%u, MT=%u, MX=%u, MY=%u,"
++                                      " X_method=%u, Y_method=%u",
++                                      mng_info->magn_mb,mng_info->magn_ml,
++                                      mng_info->magn_mr,mng_info->magn_mt,
++                                      mng_info->magn_mx,mng_info->magn_my,
++                                      mng_info->magn_methx,
++                                      mng_info->magn_methy);
++
++              /*
++                If the image width is 1, then X magnification is done
++                by simple pixel replication.
++              */
++              if (image->columns == 1)
++                  mng_info->magn_methx = 1;
++
++              /*
++                If the image height is 1, then Y magnification is done
++                by simple pixel replication.
++              */
++              if (image->rows == 1)
++                  mng_info->magn_methy = 1;
+ 
+               if (mng_info->magn_methx == 1)
+                 {
+@@ -5734,12 +5757,10 @@ static Image *ReadMNGImage(const ImageInfo *image_info,
+                   Image
+                     *large_image;
+ 
+-                  int
+-                    yy;
+-
+                   long
+                     m,
+-                    y;
++                    y,
++                    yy;
+ 
+                   register long
+                     x;

config/patches/libxml2/50f06b3efb638efb0abd95dc62dca05ae67882c2.patch

+From 50f06b3efb638efb0abd95dc62dca05ae67882c2 Mon Sep 17 00:00:00 2001
+From: Nick Wellnhofer <wellnhofer@aevum.de>
+Date: Fri, 7 Aug 2020 21:54:27 +0200
+Subject: [PATCH] Fix out-of-bounds read with 'xmllint --htmlout'
+
+Make sure that truncated UTF-8 sequences don't cause an out-of-bounds
+array access.
+
+Thanks to @SuhwanSong and the Agency for Defense Development (ADD) for
+the report.
+
+Fixes #178.
+---
+ xmllint.c | 6 ++++++
+ 1 file changed, 6 insertions(+)
+
+diff --git a/xmllint.c b/xmllint.c
+index f6a8e4636..c647486f3 100644
+--- a/xmllint.c
++++ b/xmllint.c
+@@ -528,6 +528,12 @@ static void
+ xmlHTMLEncodeSend(void) {
+     char *result;
+
++    /*
++     * xmlEncodeEntitiesReentrant assumes valid UTF-8, but the buffer might
++     * end with a truncated UTF-8 sequence. This is a hack to at least avoid
++     * an out-of-bounds read.
++     */
++    memset(&buffer[sizeof(buffer)-4], 0, 4);
+     result = (char *) xmlEncodeEntitiesReentrant(NULL, BAD_CAST buffer);
+     if (result) {
+ 	xmlGenericError(xmlGenericErrorContext, "%s", result);
+--
+GitLab
\ No newline at end of file

config/software/gnupg.rb

@@ -15,7 +15,7 @@
 #
 name 'gnupg'
-default_version '2.2.22'
+default_version '2.2.23'
 dependency 'libassuan'
 dependency 'npth'
@@ -30,7 +30,7 @@ license_file 'COPYING.LGPL3'
 skip_transitive_dependency_licensing true
 source url: "https://www.gnupg.org/ftp/gcrypt/gnupg/gnupg-#{version}.tar.bz2",
-       sha256: '7c1370565e1910b9d8c4e0fb57b9de34aa062ec7bb91abad5803d791f38d855b'
+       sha256: '10b55e49d78b3e49f1edb58d7541ecbdad92ddaeeb885b6f486ed23d1cd1da5c'
 relative_path "gnupg-#{version}"

config/software/graphicsmagick.rb

@@ -15,7 +15,7 @@
 #
 name 'graphicsmagick'
-default_version '1.3.34'
+default_version '1.3.35'
 license 'MIT'
 license_file 'Copyright.txt'
@@ -28,13 +28,15 @@ dependency 'libtiff'
 dependency 'zlib'
 source url: "https://ftp.icm.edu.pl/pub/unix/graphics/GraphicsMagick/1.3/GraphicsMagick-#{version}.tar.gz",
-       sha256: '4717f7a32d964c515d83706fd52d34e089c2ffa35f8fbf43c923ce19343cf2f4'
+       sha256: 'd96d5ce2ef7e0e90166551e38742713728bfd33d6f18495a9ddda938700fc055'
 relative_path "GraphicsMagick-#{version}"
 build do
   env = with_standard_compiler_flags(with_embedded_path)
+  patch source: "png-vulnerability-fix.patch"
+
   configure_command = [
     './configure',
     "--prefix=#{install_dir}/embedded",

config/software/libxml2.rb

@@ -15,7 +15,7 @@
 #
 name 'libxml2'
-default_version '2.9.9'
+default_version '2.9.10'
 license 'MIT'
 license_file 'COPYING'
@@ -27,8 +27,8 @@ dependency 'libiconv'
 dependency 'liblzma'
 dependency 'config_guess'
-version '2.9.9' do
-  source md5: 'c04a5a0a042eaa157e8e8c9eabe76bd6'
+version '2.9.10' do
+  source sha256: 'aafee193ffb8fe0c82d4afef6ef91972cbaf5feea100edc2f262750611b4be1f'
 end
 source url: "ftp://xmlsoft.org/libxml2/libxml2-#{version}.tar.gz"
@@ -38,6 +38,8 @@ relative_path "libxml2-#{version}"
 build do
   env = with_standard_compiler_flags(with_embedded_path)
+  patch source: '50f06b3efb638efb0abd95dc62dca05ae67882c2.patch', env: env
+
   configure_command = [
     "--with-zlib=#{install_dir}/embedded",
     "--with-iconv=#{install_dir}/embedded",

files/gitlab-ctl-commands-ee/lib/geo/promote_db.rb

+require 'rainbow/ext/string'
+
+module Geo
+  # PromoteDb promotes standby database as usual "pg-ctl promote" but
+  # if point-in-time LSN file is found, the database will be recovered to that state first
+  class PromoteDb
+    PITR_FILE_NAME = 'geo-pitr-file'.freeze
+
+    attr_accessor :base_path, :data_path
+
+    def initialize(ctl)
+      @base_path = ctl.base_path
+      @data_path = ctl.data_path
+    end
+
+    def execute
+      return true if recovery_to_point_in_time
+
+      puts
+      puts 'Promoting the PostgreSQL read-only replica to primary...'.color(:yellow)
+      puts
+
+      run_command('/opt/gitlab/embedded/bin/gitlab-pg-ctl promote', live: true).error!
+
+      success_message
+    end
+
+    private
+
+    def postgresql_version
+      @postgresql_version ||= GitlabCtl::PostgreSQL.postgresql_version(data_path)
+    end
+
+    def recovery_to_point_in_time
+      lsn = lsn_from_pitr_file
+
+      return if lsn.nil?
+
+      puts
+      puts "Recovery to point #{lsn} and promoting...".color(:yellow)
+      puts
+
+      write_recovery_settings(lsn)
+
+      run_command('gitlab-ctl restart postgresql', live: true).error!
+
+      success_message
+
+      true
+    end
+
+    def lsn_from_pitr_file
+      geo_pitr_file = "#{data_path}/postgresql/data/#{PITR_FILE_NAME}"
+
+      return nil unless File.exist?(geo_pitr_file)
+
+      lsn = File.read(geo_pitr_file)
+
+      lsn.empty? ? nil : lsn
+    end
+
+    def built_recovery_setting_for_pitr(lsn)
+      <<-EOF
+        recovery_target_lsn = '#{lsn}'
+        recovery_target_action = 'promote'
+      EOF
+    end
+
+    def write_recovery_settings(lsn)
+      settings = built_recovery_setting_for_pitr(lsn)
+
+      if postgresql_version >= 12
+        puts "PostgreSQL 12 or newer. Writing settings to postgresql.conf...".color(:green)
+
+        write_geo_config_file(settings)
+      else
+        puts "Writing recovery.conf...".color(:green)
+
+        write_recovery_conf(settings)
+      end
+    end
+
+    def write_geo_config_file(settings)
+      geo_conf_file = "#{data_path}/postgresql/data/gitlab-geo.conf"
+
+      File.open(geo_conf_file, "w", 0640) do |file|
+        file.write(settings)
+      end
+    end
+
+    def write_recovery_conf(settings)
+      recovery_conf = "#{data_path}/postgresql/data/recovery.conf"
+
+      File.open(recovery_conf, 'a', 0640) do |file|
+        file.write(settings)
+      end
+    end
+
+    def run_command(cmd, live: false)
+      GitlabCtl::Util.run_command(cmd, live: live)
+    end
+
+    def success_message
+      puts
+      puts 'The database is successfully promoted!'.color(:green)
+    end
+  end
+end

files/gitlab-ctl-commands-ee/lib/geo/promote_to_primary_node.rb

@@ -3,8 +3,11 @@ require 'rainbow/ext/string'
 module Geo
   class PromoteToPrimaryNode
-    def initialize(base_path, options)
-      @base_path = base_path
+    attr_accessor :ctl, :base_path
+
+    def initialize(ctl, options)
+      @ctl = ctl
+      @base_path = @ctl.base_path
       @options = options
     end
@@ -61,11 +64,7 @@ module Geo
     end
     def promote_postgresql_to_primary
-      puts
-      puts 'Promoting the PostgreSQL to primary...'.color(:yellow)
-      puts
-
-      run_command('/opt/gitlab/embedded/bin/gitlab-pg-ctl promote', live: true).error!
+      Geo::PromoteDb.new(ctl).execute
     end
     def reconfigure

files/gitlab-ctl-commands-ee/lib/geo/replication_toggle_command.rb

 require_relative "./replication_process"
+require_relative "./promote_db"
 require 'optparse'
 require 'English'
@@ -19,6 +20,8 @@ module Geo
     def execute!
       @replication_process.send(@action.to_sym)
+
+      process_pitr_file
     rescue Geo::PsqlError => e
       puts "Postgres encountered an error: #{e.message}"
       exit 1
@@ -33,6 +36,30 @@ module Geo
     private
+    attr_reader :action, :ctl
+
+    def process_pitr_file
+      geo_pitr_file_path = "#{ctl.data_path}/postgresql/data/#{Geo::PromoteDb::PITR_FILE_NAME}"
+
+      if action == 'pause'
+        puts "* Create Geo point-in-time recovery file".color(:green)
+
+        File.write(geo_pitr_file_path, current_lsn)
+      elsif action == 'resume'
+        puts "* Remove Geo point-in-time recovery file".color(:green)
+
+        File.delete(geo_pitr_file_path) if File.exist?(geo_pitr_file_path)
+      end
+    end
+
+    def current_lsn
+      run_query('SELECT pg_last_wal_replay_lsn()')
+    end
+
+    def run_query(query)
+      GitlabCtl::Util.get_command_output("gitlab-psql -d postgres -c '#{query}' -q -t").strip
+    end
+
     def parse_options!
       opts_parser = OptionParser.new do |opts|
         opts.banner = "Usage: gitlab-ctl replication-process-#{@action} [options]"

files/gitlab-ctl-commands-ee/promote_db.rb

+require "#{base_path}/embedded/service/omnibus-ctl-ee/lib/geo/promote_db"
+
+#
+# Copyright:: Copyright (c) 2020 GitLab Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+add_command_under_category('promote-db', 'gitlab-geo', 'Promote secondary PostgreSQL database', 2) do |cmd_name, *args|
+  Geo::PromoteDb.new(self).execute
+end

files/gitlab-ctl-commands-ee/promote_to_primary_node.rb

@@ -18,26 +18,26 @@ require "#{base_path}/embedded/service/omnibus-ctl-ee/lib/geo/promote_to_primary
 #
 add_command_under_category('promote-to-primary-node', 'gitlab-geo', 'Promote to primary node', 2) do |cmd_name, *args|
-  Geo::PromoteToPrimaryNode.new(base_path, get_ctl_options).execute
-end
-def get_ctl_options
-  options = {}
-  OptionParser.new do |opts|
-    opts.banner = "Usage: gitlab-ctl promote-to-primary-node [options]"
-    opts.on('-p', '--[no-]confirm-primary-is-down', 'Do not ask for confirmation that primary is down') do |p|
-      options[:confirm_primary_is_down] = p
-    end
-    opts.on('-m', '--skip-preflight-checks', 'Perform promotion without running any preflight checks') do |m|
-      options[:skip_preflight_checks] = m
-    end
-    opts.on('-f', '--force', 'Proceed even if preflight checks fail') do |f|
-      options[:force] = f
-    end
-  end.parse!(ARGV.dup)
-  options
+  def get_ctl_options
+    options = {}
+    OptionParser.new do |opts|
+      opts.banner = "Usage: gitlab-ctl promote-to-primary-node [options]"
+      opts.on('-p', '--[no-]confirm-primary-is-down', 'Do not ask for confirmation that primary is down') do |p|
+        options[:confirm_primary_is_down] = p
+      end
+      opts.on('-m', '--skip-preflight-checks', 'Perform promotion without running any preflight checks') do |m|
+        options[:skip_preflight_checks] = m
+      end
+      opts.on('-f', '--force', 'Proceed even if preflight checks fail') do |f|
+        options[:force] = f
+      end
+    end.parse!(ARGV.dup)
+    options
+  end
+  Geo::PromoteToPrimaryNode.new(self, get_ctl_options).execute
 end

spec/gitlab-ctl-commands-ee/lib/geo/promote_db_spec.rb

+require 'spec_helper'
+require 'geo/promote_db'
+require 'gitlab_ctl/util'
+
+RSpec.describe Geo::PromoteDb, '#execute' do
+  let(:instance) { double(base_path: '/opt/gitlab/embedded', data_path: '/var/opt/gitlab/postgresql/data') }
+
+  subject(:command) { described_class.new(instance) }
+
+  before do
+    allow($stdout).to receive(:puts)
+    allow($stdout).to receive(:print)
+
+    allow(command).to receive(:run_command).with(any_args)
+
+    allow(command).to receive(:run_command).and_return(double('error!' => nil))
+  end
+
+  context 'when PITR file does not exist' do
+    it 'does not run PITR recovery' do
+      expect(command).not_to receive(:write_recovery_settings)
+
+      command.execute
+    end
+  end
+
+  context 'when PITR file exists' do
+    let(:lsn) { '16/B374D848' }
+
+    before do
+      allow(command).to receive(:lsn_from_pitr_file).and_return(lsn)
+    end
+
+    it 'runs PITR recovery' do
+      expect(command).to receive(:write_recovery_settings).with(lsn)
+
+      expect { command.execute }.to output(
+        /Recovery to point #{lsn} and promoting.../).to_stdout
+    end
+
+    context 'PG version 11' do
+      it 'runs PITR recovery' do
+        allow(command).to receive(:postgresql_version).and_return(11)
+        expect(command).to receive(:write_recovery_conf)
+
+        expect { command.execute }.to output(
+          /Writing recovery.conf/).to_stdout
+      end
+    end
+
+    context 'PG version 12' do
+      it 'runs PITR recovery' do
+        allow(command).to receive(:postgresql_version).and_return(12)
+        expect(command).to receive(:write_geo_config_file)
+
+        expect { command.execute }.to output(
+          /PostgreSQL 12 or newer. Writing settings to postgresql.conf/).to_stdout
+      end
+    end
+  end
+end

spec/gitlab-ctl-commands-ee/lib/geo/promote_to_primary_node_spec.rb

 require 'spec_helper'
 require 'fileutils'
 require 'geo/promote_to_primary_node'
+require 'geo/promote_db'
 require 'geo/promotion_preflight_checks'
 require 'gitlab_ctl/util'
 RSpec.describe Geo::PromoteToPrimaryNode, '#execute' do
   let(:options) { { skip_preflight_checks: true } }
-  subject(:command) { described_class.new(nil, options) }
+  let(:instance) { double(base_path: '/opt/gitlab/embedded', data_path: '/var/opt/gitlab/postgresql/data') }
+
+  subject(:command) { described_class.new(instance, options) }
   let(:temp_directory) { Dir.mktmpdir }
   let(:gitlab_config_path) { File.join(temp_directory, 'gitlab.rb') }
@@ -23,6 +26,23 @@ RSpec.describe Geo::PromoteToPrimaryNode, '#execute' do
     FileUtils.rm_rf(temp_directory)
   end
+  describe '#promote_postgresql_to_primary' do
+    before do
+      allow(STDIN).to receive(:gets).and_return('y')
+
+      allow(command).to receive(:toggle_geo_roles).and_return(true)
+      allow(command).to receive(:reconfigure).and_return(true)
+      allow(command).to receive(:promote_to_primary).and_return(true)
+      allow(command).to receive(:success_message).and_return(true)
+    end
+
+    it 'promotes the database' do
+      expect_any_instance_of(Geo::PromoteDb).to receive(:execute)
+
+      command.execute
+    end
+  end
+
   describe '#run_preflight_checks' do
     before do
       allow(STDIN).to receive(:gets).and_return('y')
@@ -57,7 +77,7 @@ RSpec.describe Geo::PromoteToPrimaryNode, '#execute' do
       it 'passes given options to preflight checks command' do
         expect(Geo::PromotionPreflightChecks).to receive(:new).with(
-          nil, options).and_call_original
+          '/opt/gitlab/embedded', options).and_call_original
         command.execute
       end

spec/gitlab-ctl-commands-ee/lib/geo/replication_toggle_command_spec.rb

@@ -4,19 +4,26 @@ $LOAD_PATH << './files/gitlab-ctl-commands-ee/lib'
 $LOAD_PATH << './files/gitlab-ctl-commands/lib'
 require 'geo/replication_toggle_command'
+require 'geo/promote_to_primary_node'
+require 'gitlab_ctl/util'
 RSpec.describe Geo::ReplicationToggleCommand do
   let(:status) { double('Command status', error?: false) }
   let(:arguments) { [] }
-  let(:ctl_instance) { double('gitlab-ctl instance', base_path: '') }
+  let(:ctl_instance) { double('gitlab-ctl instance', base_path: '', data_path: 'data_path') }
+
+  before do
+    allow_any_instance_of(Geo::ReplicationToggleCommand).to receive(:current_lsn).and_return('16/B374D848')
+  end
   describe 'pause' do
     subject { described_class.new(ctl_instance, 'pause', arguments) }
     it 'calls pause' do
       expect_any_instance_of(Geo::ReplicationProcess).to receive(:pause)
-      subject.execute!
+      expect(File).to receive(:write).with('data_path/postgresql/data/geo-pitr-file', '16/B374D848')
+      expect { subject.execute! }.to output(/Create Geo point-in-time recovery file/).to_stdout
     end
     it 'rescues and exits if postgres has an error' do
@@ -33,8 +40,9 @@ RSpec.describe Geo::ReplicationToggleCommand do
       it 'uses the specified database' do
         expect(Geo::ReplicationProcess).to receive(:new).with(any_args, { db_name: 'database_i_want' }).and_call_original
         expect_any_instance_of(Geo::ReplicationProcess).to receive(:pause)
-        subject.execute!
+        expect(File).to receive(:write).with('data_path/postgresql/data/geo-pitr-file', '16/B374D848')
+        expect { subject.execute! }.to output(/Create Geo point-in-time recovery file/).to_stdout
       end
     end
   end
@@ -45,7 +53,7 @@ RSpec.describe Geo::ReplicationToggleCommand do
     it 'calls resume' do
       expect_any_instance_of(Geo::ReplicationProcess).to receive(:resume)
-      subject.execute!
+      expect { subject.execute! }.to output(/Remove Geo point-in-time recovery file/).to_stdout
     end
     it 'rescues and exits if postgres has an error' do
@@ -71,7 +79,7 @@ RSpec.describe Geo::ReplicationToggleCommand do
         expect(Geo::ReplicationProcess).to receive(:new).with(any_args, { db_name: 'database_i_want' }).and_call_original
         expect_any_instance_of(Geo::ReplicationProcess).to receive(:resume)
-        subject.execute!
+        expect { subject.execute! }.to output(/Remove Geo point-in-time recovery file/).to_stdout
       end
     end
   end

spec/gitlab-ctl-commands-ee/promote_db_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require 'geo/promote_db'
+
+RSpec.describe 'gitlab-ctl promote-db' do
+  let(:klass) { Geo::PromoteDb }
+  let(:command_name) { 'promote-db' }
+  let(:command_script) { 'promote_db' }
+
+  include_context 'ctl'
+
+  it_behaves_like 'gitlab geo promotion commands', 'promote-db'
+end