Add latest changes from gitlab-org/gitlab@master

GITLAB_SHELL_VERSION

-10.0.0
+10.1.0

GITLAB_WORKHORSE_VERSION

-8.12.0
+8.14.0

app/assets/stylesheets/framework/job_log.scss

@@ -12,7 +12,7 @@
 }
 .log-line {
-  padding: 1px $gl-padding 1px $job-log-line-padding;
+  padding: 1px $gl-padding-8 1px $job-log-line-padding;
   min-height: $gl-line-height-20;
 }
@@ -20,7 +20,7 @@
   color: $gl-text-color-inverted;
   padding: 0 $gl-padding-8;
   min-width: $job-line-number-width;
-  margin-left: -$job-line-number-width;
+  margin-left: -$job-line-number-margin;
   padding-right: 1em;
   &:hover,

app/assets/stylesheets/framework/variables.scss

@@ -606,9 +606,10 @@ $blame-blue: #254e77;
 */
 $builds-trace-bg: #111;
 $job-log-highlight-height: 18px;
-$job-log-line-padding: 62px;
-$job-line-number-width: 40px;
-$job-arrow-margin: 50px;
+$job-log-line-padding: 55px;
+$job-line-number-width: 50px;
+$job-line-number-margin: 43px;
+$job-arrow-margin: 55px;
 /*
 * Commit Page

app/models/ci/build.rb

@@ -128,6 +128,12 @@ module Ci
     scope :with_stale_live_trace, -> { with_live_trace.finished_before(12.hours.ago) }
     scope :finished_before, -> (date) { finished.where('finished_at < ?', date) }
+    scope :with_secure_reports_from_options, -> (job_type) { where('options like :job_type', job_type: "%:artifacts:%:reports:%:#{job_type}:%") }
+
+    scope :with_secure_reports_from_config_options, -> (job_types) do
+      joins(:metadata).where("ci_builds_metadata.config_options -> 'artifacts' -> 'reports' ?| array[:job_types]", job_types: job_types)
+    end
+
     scope :matches_tag_ids, -> (tag_ids) do
       matcher = ::ActsAsTaggableOn::Tagging
         .where(taggable_type: CommitStatus.name)

changelogs/unreleased/rd-fix-show-trial-errors-helper.yml

+---
+title: Fix error where helper was incorrectly returning `true`
+merge_request: 18231
+author:
+type: fixed

changelogs/unreleased/workhorse-8-14.yml

+---
+title: Update Workhorse to v8.14.0
+merge_request: 18391
+author:
+type: other

doc/administration/gitaly/praefect.md

@@ -25,40 +25,43 @@ three Gitaly nodes, `Praefect-Git-1`,  `Praefect-Git-2`, and `Praefect-Git-3`.
 There should be no knowledge other than with Praefect about the existence of
 the `Praefect-Git-X` nodes.
-### Enable the daemon
-Praefect is expected to run on their own host, this means that no other service
-other than the support services run on this machine.
-Praefect is disabled by default, to enable praefect uncomment the following line
-and set it to `true`: `# praefect['enable'] = false'`
-```ruby
-praefect['enable'] = true
-```
-
-By default praefect will listen on port `:2305`. It's recommended to enable
-prometheus to expose metrics. Uncomment the line so it looks like:
-
-```ruby
-praefect['prometheus_listen_addr'] = "localhost:9652"
-```
-Preafect needs at least one storage to store the Git data on. This node should
-run Gitaly and should not be listed as storage for GitLab itself, that is, the
-only way it receives traffic is through Praefect and it's not listed in the
-`git_data_dirs` on any `gitlab.rb` in your GitLab cluster.
-To set the nodes as depicted in the diagram above, the configuration should look
-like:
+### Setup
+In this setup guide, the Gitaly node will be added first, then Praefect, and
+lastly we update the GitLab configuration.
+#### Gitaly
+In their own machine, configure the Gitaly server as described in the
+[gitaly documentation](index.md#3-gitaly-server-configuration).
+#### Praefect
+Next, Praefect has to be enabled on its own node. Disable all other services,
+and add each Gitaly node that will be connected to Praefect. In the example below,
+the Gitaly nodes are named `praefect-git-X`. Note that one node is designated as
+primary, by setting the primary to `true`:
 ```ruby
+# /etc/gitlab/gitlab.rb
+
+# Avoid running unnecessary services on the Gitaly server
+postgresql['enable'] = false
+redis['enable'] = false
+nginx['enable'] = false
+prometheus['enable'] = false
+unicorn['enable'] = false
+sidekiq['enable'] = false
+gitlab_workhorse['enable'] = false
+gitaly['enable'] = false
+
+praefect['enable'] = true
 praefect['storage_nodes'] = [
   {
     'storage' => 'praefect-git-1',
     'address' => 'tcp://praefect-git-1.internal',
     'primary' => true
-}
+  },
   {
     'storage' => 'praefect-git-2',
     'address' => 'tcp://praefect-git-2.internal'
@@ -70,10 +73,9 @@ praefect['storage_nodes'] = [
 ]
 ```
-Save the file, and run `gitlab-ctl reconfigure`. To test if Praefect is running,
-you could run `gitlab-ctl status` which should list praefect as being up.
-### Enable Preafect as storage backend in GitLab
+Save the file and [reconfigure Praefect](../restart_gitlab.md#omnibus-gitlab-reconfigure).
+#### GitLab
 When Praefect is running, it should be exposed as a storage to GitLab. This
 is done through setting the `git_data_dirs`. Assuming the default storage

doc/ci/caching/index.md

@@ -12,8 +12,9 @@ content of a previous job. It can be particularly useful when you are
 developing software that depends on other libraries which are fetched via the
 internet during build time.
-If caching is enabled, it's shared between pipelines and jobs by default,
-starting from GitLab 9.0.
+If caching is enabled, it's shared between pipelines and jobs at the project
+level by default, starting from GitLab 9.0. Caches are not shared across
+projects.
 Make sure you read the [`cache` reference](../yaml/README.md#cache) to learn
 how it is defined in `.gitlab-ci.yml`.

doc/development/contributing/merge_request_workflow.md

@@ -220,6 +220,8 @@ requirements.
 1. Working and clean code that is commented where needed.
 1. [Unit, integration, and system tests](../testing_guide/index.md) that all pass
    on the CI server.
+1. Regressions and bugs are covered with tests that reduce the risk of the issue happening
+   again.
 1. Performance/scalability implications have been considered, addressed, and tested.
 1. [Documented](../documentation/index.md) in the `/doc` directory.
 1. [Changelog entry added](../changelog.md), if necessary.

doc/development/internal_api.md

@@ -47,6 +47,7 @@ POST /internal/allowed
 | `protocol` | string | yes     | SSH when called from GitLab-shell, HTTP or SSH when called from Gitaly |
 | `action`   | string | yes     | Git command being run (`git-upload-pack`, `git-receive-pack`, `git-upload-archive`) |
 | `changes`  | string | yes     | `<oldrev> <newrev> <refname>` when called from Gitaly, The magic string `_any` when called from GitLab Shell |
+| `check_ip` | string | no     | Ip adress from which call to GitLab Shell was made |
 Example request:

doc/user/group/index.md

@@ -351,7 +351,7 @@ Add one or more whitelisted IP subnets using CIDR notation in comma separated fo
 coming from a different IP address won't be able to access the restricted
 content.
-Restriction currently applies to UI and API access, Git actions via SSH are not restricted.
+Restriction currently applies to UI, API access and Git actions via SSH.
 To avoid accidental lock-out, admins and group owners are are able to access
 the group regardless of the IP restriction.

lib/api/internal/base.rb

@@ -26,20 +26,11 @@ module API
         def ee_post_receive_response_hook(response)
           # Hook for EE to add messages
         end
-      end
-      namespace 'internal' do
-        # Check if git command is allowed for project
-        #
-        # Params:
-        #   key_id - ssh key id for Git over SSH
-        #   user_id - user id for Git over HTTP or over SSH in keyless SSH CERT mode
-        #   username - user name for Git over SSH in keyless SSH cert mode
-        #   protocol - Git access protocol being used, e.g. HTTP or SSH
-        #   project - project full_path (not path on disk)
-        #   action - git action (git-upload-pack or git-receive-pack)
-        #   changes - changes as "oldrev newrev ref", see Gitlab::ChangesList
-        post "/allowed" do
+        def check_allowed(params)
+          # This is a separate method so that EE can alter its behaviour more
+          # easily.
+
           # Stores some Git-specific env thread-safely
           env = parse_env
           Gitlab::Git::HookEnv.set(gl_repository, env) if project
@@ -53,11 +44,11 @@ module API
                            @project ||= access_checker.project
                            result
                          rescue Gitlab::GitAccess::UnauthorizedError => e
-                           break response_with_status(code: 401, success: false, message: e.message)
+                           return response_with_status(code: 401, success: false, message: e.message)
                          rescue Gitlab::GitAccess::TimeoutError => e
-                           break response_with_status(code: 503, success: false, message: e.message)
+                           return response_with_status(code: 503, success: false, message: e.message)
                          rescue Gitlab::GitAccess::NotFoundError => e
-                           break response_with_status(code: 404, success: false, message: e.message)
+                           return response_with_status(code: 404, success: false, message: e.message)
                          end
           log_user_activity(actor.user)
@@ -91,6 +82,26 @@ module API
             response_with_status(code: 500, success: false, message: UNKNOWN_CHECK_RESULT_ERROR)
           end
         end
+      end
+
+      namespace 'internal' do
+        # Check if git command is allowed for project
+        #
+        # Params:
+        #   key_id - ssh key id for Git over SSH
+        #   user_id - user id for Git over HTTP or over SSH in keyless SSH CERT mode
+        #   username - user name for Git over SSH in keyless SSH cert mode
+        #   protocol - Git access protocol being used, e.g. HTTP or SSH
+        #   project - project full_path (not path on disk)
+        #   action - git action (git-upload-pack or git-receive-pack)
+        #   changes - changes as "oldrev newrev ref", see Gitlab::ChangesList
+        #   check_ip - optional, only in EE version, may limit access to
+        #     group resources based on its IP restrictions
+        post "/allowed" do
+          # It was moved to a separate method so that EE can alter its behaviour more
+          # easily.
+          check_allowed(params)
+        end
         # rubocop: disable CodeReuse/ActiveRecord
         post "/lfs_authenticate" do

lib/gitlab/sidekiq_daemon/memory_killer.rb

@@ -39,8 +39,8 @@ module Gitlab
         while enabled?
           begin
-            restart_sidekiq unless rss_within_range?
             sleep(CHECK_INTERVAL_SECONDS)
+            restart_sidekiq unless rss_within_range?
           rescue => e
             log_exception(e, __method__)
           rescue Exception => e # rubocop:disable Lint/RescueException

lib/gitlab/sidekiq_logging/structured_logger.rb

@@ -70,10 +70,11 @@ module Gitlab
       end
       def add_time_keys!(time, payload)
-        payload['duration'] = time[:duration].round(3)
-        payload['system_s'] = time[:stime].round(3)
-        payload['user_s'] = time[:utime].round(3)
-        payload['child_s'] = time[:ctime].round(3) if time[:ctime] > 0
+        payload['duration'] = time[:duration].round(6)
+
+        # ignore `cpu_s` if the platform does not support Process::CLOCK_THREAD_CPUTIME_ID (time[:cputime] == 0)
+        # supported OS version can be found at: https://www.rubydoc.info/stdlib/core/2.1.6/Process:clock_gettime
+        payload['cpu_s'] = time[:cputime].round(6) if time[:cputime] > 0
         payload['completed_at'] = Time.now.utc
       end
@@ -98,30 +99,24 @@ module Gitlab
       end
       def elapsed_by_absolute_time(start)
-        (Time.now.utc - start).to_f.round(3)
+        (Time.now.utc - start).to_f.round(6)
       end
       def elapsed(t0)
         t1 = get_time
         {
           duration: t1[:now] - t0[:now],
-          stime: t1[:times][:stime] - t0[:times][:stime],
-          utime: t1[:times][:utime] - t0[:times][:utime],
-          ctime: ctime(t1[:times]) - ctime(t0[:times])
+          cputime: t1[:thread_cputime] - t0[:thread_cputime]
         }
       end
       def get_time
         {
           now: current_time,
-          times: Process.times
+          thread_cputime: defined?(Process::CLOCK_THREAD_CPUTIME_ID) ? Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID) : 0
         }
       end
-      def ctime(times)
-        times[:cstime] + times[:cutime]
-      end
-
       def current_time
         Gitlab::Metrics::System.monotonic_time
       end
@@ -129,7 +124,7 @@ module Gitlab
       def format_time(timestamp)
         return timestamp if timestamp.is_a?(String)
-        Time.at(timestamp).utc.iso8601(3)
+        Time.at(timestamp).utc.iso8601(6)
       end
       def limited_job_args(args)

spec/factories/ci/builds.rb

@@ -330,6 +330,38 @@ FactoryBot.define do
       options { {} }
     end
+    trait :dast do
+      options do
+        {
+            artifacts: { reports: { dast: 'gl-dast-report.json' } }
+        }
+      end
+    end
+
+    trait :sast do
+      options do
+        {
+            artifacts: { reports: { sast: 'gl-sast-report.json' } }
+        }
+      end
+    end
+
+    trait :dependency_scanning do
+      options do
+        {
+            artifacts: { reports: { dependency_scanning: 'gl-dependency-scanning-report.json' } }
+        }
+      end
+    end
+
+    trait :container_scanning do
+      options do
+        {
+            artifacts: { reports: { container_scanning: 'gl-container-scanning-report.json' } }
+        }
+      end
+    end
+
     trait :non_playable do
       status { 'created' }
       self.when { 'manual' }

spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb

@@ -40,6 +40,7 @@ describe Gitlab::SidekiqDaemon::MemoryKiller do
             message: "Exception from start_working: My Exception")
         expect(memory_killer).to receive(:rss_within_range?).twice.and_raise(StandardError, 'My Exception')
+        expect(memory_killer).to receive(:sleep).twice.with(Gitlab::SidekiqDaemon::MemoryKiller::CHECK_INTERVAL_SECONDS)
         expect { subject }.not_to raise_exception
       end
@@ -53,6 +54,7 @@ describe Gitlab::SidekiqDaemon::MemoryKiller do
         expect(memory_killer).to receive(:rss_within_range?).once.and_raise(Exception, 'My Exception')
+        expect(memory_killer).to receive(:sleep).with(Gitlab::SidekiqDaemon::MemoryKiller::CHECK_INTERVAL_SECONDS)
         expect(Sidekiq.logger).to receive(:warn).once
           .with(
             class: described_class.to_s,

spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb

@@ -23,13 +23,15 @@ describe Gitlab::SidekiqLogging::StructuredLogger do
     end
     let(:logger) { double }
+    let(:clock_thread_cputime_start) { 0.222222299 }
+    let(:clock_thread_cputime_end) { 1.333333799 }
     let(:start_payload) do
       job.merge(
         'message' => 'TestWorker JID-da883554ee4fe414012f5f42: start',
         'job_status' => 'start',
         'pid' => Process.pid,
-        'created_at' => created_at.iso8601(3),
-        'enqueued_at' => created_at.iso8601(3),
+        'created_at' => created_at.iso8601(6),
+        'enqueued_at' => created_at.iso8601(6),
         'scheduling_latency_s' => scheduling_latency_s
       )
     end
@@ -38,9 +40,8 @@ describe Gitlab::SidekiqLogging::StructuredLogger do
         'message' => 'TestWorker JID-da883554ee4fe414012f5f42: done: 0.0 sec',
         'job_status' => 'done',
         'duration' => 0.0,
-        "completed_at" => timestamp.iso8601(3),
-        "system_s" => 0.0,
-        "user_s" => 0.0
+        "completed_at" => timestamp.iso8601(6),
+        "cpu_s" => 1.111112
       )
     end
     let(:exception_payload) do
@@ -57,12 +58,7 @@ describe Gitlab::SidekiqLogging::StructuredLogger do
       allow(subject).to receive(:current_time).and_return(timestamp.to_f)
-      allow(Process).to receive(:times).and_return(
-        stime:  0.0,
-        utime:  0.0,
-        cutime: 0.0,
-        cstime: 0.0
-      )
+      allow(Process).to receive(:clock_gettime).with(Process::CLOCK_THREAD_CPUTIME_ID).and_return(clock_thread_cputime_start, clock_thread_cputime_end)
     end
     subject { described_class.new }
@@ -187,31 +183,22 @@ describe Gitlab::SidekiqLogging::StructuredLogger do
         end
       end
     end
-    def ctime(times)
-      times[:cstime] + times[:cutime]
-    end
-    context 'with ctime value greater than 0' do
-      let(:times_start) { { stime: 0.04999, utime: 0.0483, cstime: 0.0188, cutime: 0.0188 } }
-      let(:times_end)   { { stime: 0.0699, utime: 0.0699, cstime: 0.0399, cutime: 0.0399 } }
-      before do
-        end_payload['system_s'] = 0.02
-        end_payload['user_s'] = 0.022
-        end_payload['child_s'] = 0.042
-        allow(Process).to receive(:times).and_return(times_start, times_end)
-      end
-      it 'logs with ctime data and other cpu data' do
-        Timecop.freeze(timestamp) do
-          expect(logger).to receive(:info).with(start_payload.except('args')).ordered
-          expect(logger).to receive(:info).with(end_payload.except('args')).ordered
-
-          subject.call(job, 'test_queue') { }
-        end
-      end
+  end
+  describe '#add_time_keys!' do
+    let(:time) { { duration: 0.1231234, cputime: 1.2342345 } }
+    let(:payload) { { 'class' => 'my-class', 'message' => 'my-message', 'job_status' => 'my-job-status' } }
+    let(:current_utc_time) { '2019-09-23 10:00:58 UTC' }
+    let(:payload_with_time_keys) { { 'class' => 'my-class', 'message' => 'my-message', 'job_status' => 'my-job-status', 'duration' => 0.123123, 'cpu_s' => 1.234235, 'completed_at' => current_utc_time } }
+    subject { described_class.new }
+    it 'update payload correctly' do
+      expect(Time).to receive_message_chain(:now, :utc).and_return(current_utc_time)
+      subject.send(:add_time_keys!, time, payload)
+      expect(payload).to eq(payload_with_time_keys)
     end
   end
 end