Add latest changes from gitlab-org/gitlab@master

app/policies/project_policy.rb

@@ -314,6 +314,7 @@ class ProjectPolicy < BasePolicy
     enable :admin_operations
     enable :read_deploy_token
     enable :create_deploy_token
+    enable :read_pod_logs
   end
   rule { (mirror_available & can?(:admin_project)) | admin }.enable :admin_remote_mirror

app/serializers/environment_entity.rb

@@ -47,6 +47,22 @@ class EnvironmentEntity < Grape::Entity
     environment.available? && can?(current_user, :stop_environment, environment)
   end
+  expose :logs_path, if: -> (*) { can_read_pod_logs? } do |environment|
+    project_logs_path(environment.project, environment_name: environment.name)
+  end
+
+  expose :logs_api_path, if: -> (*) { can_read_pod_logs? } do |environment|
+    if environment.elastic_stack_available?
+      elasticsearch_project_logs_path(environment.project, environment_name: environment.name, format: :json)
+    else
+      k8s_project_logs_path(environment.project, environment_name: environment.name, format: :json)
+    end
+  end
+
+  expose :enable_advanced_logs_querying, if: -> (*) { can_read_pod_logs? } do |environment|
+    environment.elastic_stack_available?
+  end
+
   private
   alias_method :environment, :object
@@ -63,6 +79,10 @@ class EnvironmentEntity < Grape::Entity
     can?(current_user, :update_environment, environment)
   end
+  def can_read_pod_logs?
+    can?(current_user, :read_pod_logs, environment.project)
+  end
+
   def cluster_platform_kubernetes?
     deployment_platform && deployment_platform.is_a?(Clusters::Platforms::Kubernetes)
   end

app/services/pod_logs/base_service.rb

+# frozen_string_literal: true
+
+module PodLogs
+  class BaseService < ::BaseService
+    include ReactiveCaching
+    include Stepable
+
+    attr_reader :cluster, :namespace, :params
+
+    CACHE_KEY_GET_POD_LOG = 'get_pod_log'
+    K8S_NAME_MAX_LENGTH = 253
+
+    SUCCESS_RETURN_KEYS = %i(status logs pod_name container_name pods).freeze
+
+    def id
+      cluster.id
+    end
+
+    def initialize(cluster, namespace, params: {})
+      @cluster = cluster
+      @namespace = namespace
+      @params = filter_params(params.dup.stringify_keys).to_hash
+    end
+
+    def execute
+      with_reactive_cache(
+        CACHE_KEY_GET_POD_LOG,
+        namespace,
+        params
+      ) do |result|
+        result
+      end
+    end
+
+    def calculate_reactive_cache(request, _namespace, _params)
+      case request
+      when CACHE_KEY_GET_POD_LOG
+        execute_steps
+      else
+        exception = StandardError.new('Unknown reactive cache request')
+        Gitlab::ErrorTracking.track_and_raise_for_dev_exception(exception, request: request)
+        error(_('Unknown cache key'))
+      end
+    end
+
+    private
+
+    def valid_params
+      %w(pod_name container_name)
+    end
+
+    def check_arguments(result)
+      return error(_('Cluster does not exist')) if cluster.nil?
+      return error(_('Namespace is empty')) if namespace.blank?
+
+      success(result)
+    end
+
+    def check_param_lengths(_result)
+      pod_name = params['pod_name'].presence
+      container_name = params['container_name'].presence
+
+      if pod_name&.length.to_i > K8S_NAME_MAX_LENGTH
+        return error(_('pod_name cannot be larger than %{max_length}'\
+          ' chars' % { max_length: K8S_NAME_MAX_LENGTH }))
+      elsif container_name&.length.to_i > K8S_NAME_MAX_LENGTH
+        return error(_('container_name cannot be larger than'\
+          ' %{max_length} chars' % { max_length: K8S_NAME_MAX_LENGTH }))
+      end
+
+      success(pod_name: pod_name, container_name: container_name)
+    end
+
+    def get_raw_pods(result)
+      result[:raw_pods] = cluster.kubeclient.get_pods(namespace: namespace)
+
+      success(result)
+    end
+
+    def get_pod_names(result)
+      result[:pods] = result[:raw_pods].map(&:metadata).map(&:name)
+
+      success(result)
+    end
+
+    def check_pod_name(result)
+      # If pod_name is not received as parameter, get the pod logs of the first
+      # pod of this namespace.
+      result[:pod_name] ||= result[:pods].first
+
+      unless result[:pod_name]
+        return error(_('No pods available'))
+      end
+
+      unless result[:pods].include?(result[:pod_name])
+        return error(_('Pod does not exist'))
+      end
+
+      success(result)
+    end
+
+    def check_container_name(result)
+      pod_details = result[:raw_pods].first { |p| p.metadata.name == result[:pod_name] }
+      containers = pod_details.spec.containers.map(&:name)
+
+      # select first container if not specified
+      result[:container_name] ||= containers.first
+
+      unless result[:container_name]
+        return error(_('No containers available'))
+      end
+
+      unless containers.include?(result[:container_name])
+        return error(_('Container does not exist'))
+      end
+
+      success(result)
+    end
+
+    def pod_logs(result)
+      raise NotImplementedError
+    end
+
+    def filter_return_keys(result)
+      result.slice(*SUCCESS_RETURN_KEYS)
+    end
+
+    def filter_params(params)
+      params.slice(*valid_params)
+    end
+  end
+end

app/services/pod_logs/elasticsearch_service.rb

+# frozen_string_literal: true
+
+module PodLogs
+  class ElasticsearchService < BaseService
+    steps :check_arguments,
+          :check_param_lengths,
+          :get_raw_pods,
+          :get_pod_names,
+          :check_pod_name,
+          :check_container_name,
+          :check_times,
+          :check_search,
+          :pod_logs,
+          :filter_return_keys
+
+    self.reactive_cache_worker_finder = ->(id, _cache_key, namespace, params) { new(::Clusters::Cluster.find(id), namespace, params: params) }
+
+    private
+
+    def valid_params
+      %w(pod_name container_name search start end)
+    end
+
+    def check_times(result)
+      result[:start] = params['start'] if params.key?('start') && Time.iso8601(params['start'])
+      result[:end] = params['end'] if params.key?('end') && Time.iso8601(params['end'])
+
+      success(result)
+    rescue ArgumentError
+      error(_('Invalid start or end time format'))
+    end
+
+    def check_search(result)
+      result[:search] = params['search'] if params.key?('search')
+
+      success(result)
+    end
+
+    def pod_logs(result)
+      client = cluster&.application_elastic_stack&.elasticsearch_client
+      return error(_('Unable to connect to Elasticsearch')) unless client
+
+      result[:logs] = ::Gitlab::Elasticsearch::Logs.new(client).pod_logs(
+        namespace,
+        result[:pod_name],
+        result[:container_name],
+        result[:search],
+        result[:start],
+        result[:end]
+      )
+
+      success(result)
+    rescue Elasticsearch::Transport::Transport::ServerError => e
+      ::Gitlab::ErrorTracking.track_exception(e)
+
+      error(_('Elasticsearch returned status code: %{status_code}') % {
+        # ServerError is the parent class of exceptions named after HTTP status codes, eg: "Elasticsearch::Transport::Transport::Errors::NotFound"
+        # there is no method on the exception other than the class name to determine the type of error encountered.
+        status_code: e.class.name.split('::').last
+      })
+    end
+  end
+end

app/services/pod_logs/kubernetes_service.rb

+# frozen_string_literal: true
+
+module PodLogs
+  class KubernetesService < BaseService
+    LOGS_LIMIT = 500.freeze
+    REPLACEMENT_CHAR = "\u{FFFD}"
+
+    EncodingHelperError = Class.new(StandardError)
+
+    steps :check_arguments,
+          :check_param_lengths,
+          :get_raw_pods,
+          :get_pod_names,
+          :check_pod_name,
+          :check_container_name,
+          :pod_logs,
+          :encode_logs_to_utf8,
+          :split_logs,
+          :filter_return_keys
+
+    self.reactive_cache_worker_finder = ->(id, _cache_key, namespace, params) { new(::Clusters::Cluster.find(id), namespace, params: params) }
+
+    private
+
+    def pod_logs(result)
+      result[:logs] = cluster.kubeclient.get_pod_log(
+        result[:pod_name],
+        namespace,
+        container: result[:container_name],
+        tail_lines: LOGS_LIMIT,
+        timestamps: true
+      ).body
+
+      success(result)
+    rescue Kubeclient::ResourceNotFoundError
+      error(_('Pod not found'))
+    rescue Kubeclient::HttpError => e
+      ::Gitlab::ErrorTracking.track_exception(e)
+
+      error(_('Kubernetes API returned status code: %{error_code}') % {
+        error_code: e.error_code
+      })
+    end
+
+    # Check https://gitlab.com/gitlab-org/gitlab/issues/34965#note_292261879
+    # for more details on why this is necessary.
+    def encode_logs_to_utf8(result)
+      return success(result) if result[:logs].nil?
+      return success(result) if result[:logs].encoding == Encoding::UTF_8
+
+      result[:logs] = encode_utf8(result[:logs])
+
+      success(result)
+    rescue EncodingHelperError
+      error(_('Unable to convert Kubernetes logs encoding to UTF-8'))
+    end
+
+    def split_logs(result)
+      result[:logs] = result[:logs].strip.lines(chomp: true).map do |line|
+        # message contains a RFC3339Nano timestamp, then a space, then the log line.
+        # resolution of the nanoseconds can vary, so we split on the first space
+        values = line.split(' ', 2)
+        {
+          timestamp: values[0],
+          message: values[1]
+        }
+      end
+
+      success(result)
+    end
+
+    def encode_utf8(logs)
+      utf8_logs = Gitlab::EncodingHelper.encode_utf8(logs.dup, replace: REPLACEMENT_CHAR)
+
+      # Gitlab::EncodingHelper.encode_utf8 can return '' or nil if an exception
+      # is raised while encoding. We prefer to return an error rather than wrongly
+      # display blank logs.
+      no_utf8_logs = logs.present? && utf8_logs.blank?
+      unexpected_encoding = utf8_logs&.encoding != Encoding::UTF_8
+
+      if no_utf8_logs || unexpected_encoding
+        raise EncodingHelperError, 'Could not convert Kubernetes logs to UTF-8'
+      end
+
+      utf8_logs
+    end
+  end
+end

app/views/layouts/nav/sidebar/_project.html.haml

@@ -263,7 +263,11 @@
                   %span
                     = _('Serverless')
-            = render_if_exists 'layouts/nav/sidebar/pod_logs_link' # EE-specific
+            - if project_nav_tab?(:environments) && can?(current_user, :read_pod_logs, @project)
+              = nav_link(controller: :logs, action: [:index]) do
+                = link_to project_logs_path(@project), title: _('Logs') do
+                  %span
+                    = _('Logs')
             - if project_nav_tab? :clusters
               - show_cluster_hint = show_gke_cluster_integration_callout?(@project)

app/views/projects/logs/empty_logs.html.haml

+- page_title _('Logs')
+
+.row.empty-state
+  .col-sm-12
+    .svg-content
+      = image_tag 'illustrations/operations_log_pods_empty.svg'
+  .col-12
+    .text-content
+      %h4.text-center
+        = s_('Environments|No deployed environments')
+      %p.state-description.text-center
+        = s_('Logs|To see the logs, deploy your code to an environment.')
+      .text-center
+        = link_to s_('Environments|Learn about environments'), help_page_path('ci/environments'), class: 'btn btn-success'

app/views/projects/logs/index.html.haml

+#environment-logs{ data: environment_logs_data(@project, @environment) }

changelogs/unreleased/ak-move-logs-to-core.yml

+---
+title: Move pod logs to core
+merge_request: 25455
+author:
+type: changed

changelogs/unreleased/mo-fix-capybara-screenshots-rails.yml

+---
+title: Fix capybara screenshots path name for rails configuration
+merge_request: 27002
+author:
+type: fixed

config/gitlab.yml.example

@@ -1227,7 +1227,7 @@ test:
     client_path: tmp/tests/gitaly
     token: secret
   workhorse:
-    secret_file: tmp/tests/gitlab_workhorse_secret
+    secret_file: tmp/gitlab_workhorse_test_secret
   backup:
     path: tmp/tests/backups
   pseudonymizer:

config/routes/project.rb

@@ -175,6 +175,13 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
           end
         end
+        resources :logs, only: [:index] do
+          collection do
+            get :k8s
+            get :elasticsearch
+          end
+        end
+
         resources :starrers, only: [:index]
         resources :forks, only: [:index, :new, :create]
         resources :group_links, only: [:index, :create, :update, :destroy], constraints: { id: /\d+/ }

doc/development/i18n/proofreader.md

@@ -56,6 +56,7 @@ are very appreciative of the work done by translators and proofreaders!
   - Adi Ferdian - [GitLab](https://gitlab.com/adiferd), [Crowdin](https://crowdin.com/profile/adiferd)
   - Ahmad Naufal Mukhtar - [GitLab](https://gitlab.com/anaufalm), [Crowdin](https://crowdin.com/profile/anaufalm)
 - Italian
+  - Massimiliano Cuttini - [GitLab](https://gitlab.com/maxcuttins), [Crowdin](https://crowdin.com/profile/maxcuttins)
   - Paolo Falomo - [GitLab](https://gitlab.com/paolofalomo), [Crowdin](https://crowdin.com/profile/paolo.falomo)
 - Japanese
   - Hiroyuki Sato - [GitLab](https://gitlab.com/hiroponz), [Crowdin](https://crowdin.com/profile/hiroponz)

doc/user/admin_area/settings/continuous_integration.md

@@ -191,3 +191,16 @@ To set required pipeline configuration:
 1. Click **Save changes**.
 ![Required pipeline](img/admin_required_pipeline.png)
+
+## Package Registry configuration **(PREMIUM ONLY)**
+
+GitLab administrators can disable the forwarding of NPM requests to [npmjs.com](https://www.npmjs.com/).
+
+To disable it:
+
+1. Go to **Admin Area > Settings > CI/CD**.
+1. Expand the **Package Registry** section.
+1. Uncheck **Enable forwarding of NPM package requests to npmjs.org**.
+1. Click **Save changes**.
+
+![NPM package requests forwarding](img/admin_package_registry_npm_package_requests_forward.png)

doc/user/incident_management/index.md

@@ -88,12 +88,13 @@ dropdown box above the upper right corner of the panel:
 The options are:
-- [View logs](#view-logs-ultimate) **(ULTIMATE)**
+- [View logs](#view-logs)
 - [Download CSV](#download-csv)
-##### View logs **(ULTIMATE)**
+##### View logs
 > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/201846) in GitLab Ultimate 12.8.
+> [Moved](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25455) to [GitLab Core](https://about.gitlab.com/pricing/) 12.9.
 This can be useful if you are triaging an application incident and need to
 [explore logs](../project/integrations/prometheus.md#view-logs-ultimate)

doc/user/packages/npm_registry/index.md

@@ -269,6 +269,14 @@ Or if you're using Yarn:
 yarn add @my-project-scope/my-package
 ```
+### Forwarding requests to npmjs.org
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/55344) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.9.
+
+By default, when an NPM package is not found in the GitLab NPM Registry, the request will be forwarded to [npmjs.com](https://www.npmjs.com/).
+
+Administrators can disable this behavior in the [Continuous Integration settings](../../admin_area/settings/continuous_integration.md).
+
 ## Removing a package
 In the packages view of your project page, you can delete packages by clicking

doc/user/project/clusters/index.md

@@ -27,7 +27,7 @@ Using the GitLab project Kubernetes integration, you can:
 - Use [Web terminals](#web-terminals).
 - Use [Deploy Boards](#deploy-boards-premium). **(PREMIUM)**
 - Use [Canary Deployments](#canary-deployments-premium). **(PREMIUM)**
-- View [Logs](#logs-ultimate). **(ULTIMATE)**
+- View [Logs](#logs).
 - Run serverless workloads on [Kubernetes with Knative](serverless/index.md).
 ### Deploy Boards **(PREMIUM)**
@@ -48,7 +48,7 @@ the need to leave GitLab.
 [Read more about Canary Deployments](../canary_deployments.md)
-### Logs **(ULTIMATE)**
+### Logs
 GitLab makes it easy to view the logs of running pods in connected Kubernetes clusters. By displaying the logs directly in GitLab, developers can avoid having to manage console tools or jump to a different interface.

doc/user/project/clusters/kubernetes_pod_logs.md

-# Kubernetes Logs **(ULTIMATE)**
+# Kubernetes Logs
 > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/4752) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.0.
+> [Moved](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25455) to [GitLab Core](https://about.gitlab.com/pricing/) 12.9.
 GitLab makes it easy to view the logs of running pods in [connected Kubernetes clusters](index.md).
 By displaying the logs directly in GitLab, developers can avoid having to manage console tools or jump to a different interface.

lib/gitlab/background_migration/backfill_snippet_repositories.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Class that will fill the project_repositories table for projects that
+    # are on hashed storage and an entry is missing in this table.
+    class BackfillSnippetRepositories
+      MAX_RETRIES = 2
+
+      def perform(start_id, stop_id)
+        Snippet.includes(:author, snippet_repository: :shard).where(id: start_id..stop_id).find_each do |snippet|
+          # We need to expire the exists? value for the cached method in case it was cached
+          snippet.repository.expire_exists_cache
+
+          next if repository_present?(snippet)
+
+          retry_index = 0
+
+          begin
+            create_repository_and_files(snippet)
+
+            logger.info(message: 'Snippet Migration: repository created and migrated', snippet: snippet.id)
+          rescue => e
+            retry_index += 1
+
+            retry if retry_index < MAX_RETRIES
+
+            logger.error(message: "Snippet Migration: error migrating snippet. Reason: #{e.message}", snippet: snippet.id)
+
+            destroy_snippet_repository(snippet)
+            delete_repository(snippet)
+          end
+        end
+      end
+
+      private
+
+      def repository_present?(snippet)
+        snippet.snippet_repository && !snippet.empty_repo?
+      end
+
+      def create_repository_and_files(snippet)
+        snippet.create_repository
+        create_commit(snippet)
+      end
+
+      def destroy_snippet_repository(snippet)
+        # Removing the db record
+        snippet.snippet_repository&.destroy
+      rescue => e
+        logger.error(message: "Snippet Migration: error destroying snippet repository. Reason: #{e.message}", snippet: snippet.id)
+      end
+
+      def delete_repository(snippet)
+        # Removing the repository in disk
+        snippet.repository.remove if snippet.repository_exists?
+      rescue => e
+        logger.error(message: "Snippet Migration: error deleting repository. Reason: #{e.message}", snippet: snippet.id)
+      end
+
+      def logger
+        @logger ||= Gitlab::BackgroundMigration::Logger.build
+      end
+
+      def snippet_action(snippet)
+        # We don't need the previous_path param
+        # Because we're not updating any existing file
+        [{ file_path: filename(snippet),
+           content: snippet.content }]
+      end
+
+      def filename(snippet)
+        snippet.file_name.presence || empty_file_name
+      end
+
+      def empty_file_name
+        @empty_file_name ||= "#{SnippetRepository::DEFAULT_EMPTY_FILE_NAME}1.txt"
+      end
+
+      def commit_attrs
+        @commit_attrs ||= { branch_name: 'master', message: 'Initial commit' }
+      end
+
+      def create_commit(snippet)
+        snippet.snippet_repository.multi_files_action(snippet.author, snippet_action(snippet), commit_attrs)
+      end
+    end
+  end
+end