Add latest changes from gitlab-org/gitlab@master

changelogs/unreleased/rename_services_template_to_instance.yml

+---
+title: 'Service model: Rename template attribute to instance'
+merge_request: 23595
+author:
+type: other

config/sidekiq_queues.yml

@@ -194,7 +194,7 @@
   - 1
 - - project_update_repository_storage
   - 1
-- - propagate_service_template
+- - propagate_instance_level_service
   - 1
 - - reactive_caching
   - 1

db/migrate/20200123040535_add_multi_column_index_on_lfs_objects_projects.rb

+# frozen_string_literal: true
+
+class AddMultiColumnIndexOnLfsObjectsProjects < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :lfs_objects_projects, [:project_id, :lfs_object_id]
+  end
+
+  def down
+    remove_concurrent_index :lfs_objects_projects, [:project_id, :lfs_object_id]
+  end
+end

db/migrate/20200123045415_remove_project_id_index_on_lfs_objects_projects.rb

+# frozen_string_literal: true
+
+class RemoveProjectIdIndexOnLfsObjectsProjects < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    remove_concurrent_index :lfs_objects_projects, :project_id
+  end
+
+  def down
+    add_concurrent_index :lfs_objects_projects, :project_id
+  end
+end

db/migrate/20200123092602_rename_services_template_to_instance.rb

+# frozen_string_literal: true
+
+class RenameServicesTemplateToInstance < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    rename_column_concurrently :services, :template, :instance
+  end
+
+  def down
+    undo_rename_column_concurrently :services, :template, :instance
+  end
+end

db/migrate/20200129234037_replace_conan_metadata_index.rb

+# frozen_string_literal: true
+
+class ReplaceConanMetadataIndex < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  OLD_INDEX = 'index_packages_conan_metadata_on_package_id'
+  NEW_INDEX = 'index_packages_conan_metadata_on_package_id_username_channel'
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :packages_conan_metadata,
+                         [:package_id, :package_username, :package_channel],
+                         unique: true, name: NEW_INDEX
+
+    remove_concurrent_index_by_name :packages_conan_metadata, OLD_INDEX
+  end
+
+  def down
+    add_concurrent_index :packages_conan_metadata, :package_id, name: OLD_INDEX
+
+    remove_concurrent_index_by_name :packages_conan_metadata, NEW_INDEX
+  end
+end

db/post_migrate/20191021101942_remove_empty_github_service_templates.rb

@@ -23,6 +23,10 @@ class RemoveEmptyGithubServiceTemplates < ActiveRecord::Migration[5.2]
   private
   def relationship
+    # The column `template` was renamed to `instance`. Column information needs
+    # to be resetted to avoid cache problems after migrating down.
+    RemoveEmptyGithubServiceTemplates::Service.reset_column_information
+
     RemoveEmptyGithubServiceTemplates::Service.where(template: true, type: 'GithubService')
   end
 end

db/post_migrate/20200123101859_cleanup_rename_services_template_to_instance.rb

+# frozen_string_literal: true
+
+class CleanupRenameServicesTemplateToInstance < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    cleanup_concurrent_column_rename :services, :template, :instance
+  end
+
+  def down
+    undo_cleanup_concurrent_column_rename :services, :template, :instance
+  end
+end

db/post_migrate/20200127111840_fix_projects_without_project_feature.rb

+# frozen_string_literal: true
+
+class FixProjectsWithoutProjectFeature < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  BATCH_SIZE = 50_000
+  MIGRATION = 'FixProjectsWithoutProjectFeature'
+
+  disable_ddl_transaction!
+
+  class Project < ActiveRecord::Base
+    include EachBatch
+  end
+
+  def up
+    queue_background_migration_jobs_by_range_at_intervals(Project, MIGRATION, 2.minutes, batch_size: BATCH_SIZE)
+  end
+
+  def down
+    # no-op
+  end
+end

db/post_migrate/20200206111847_migrate_propagate_service_template_sidekiq_queue.rb

+# frozen_string_literal: true
+
+class MigratePropagateServiceTemplateSidekiqQueue < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    sidekiq_queue_migrate 'propagate_service_template', to: 'propagate_instance_level_service'
+  end
+
+  def down
+    sidekiq_queue_migrate 'propagate_instance_level_service', to: 'propagate_service_template'
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema.define(version: 2020_02_06_091544) do
+ActiveRecord::Schema.define(version: 2020_02_06_111847) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "pg_trgm"
@@ -2347,7 +2347,7 @@ ActiveRecord::Schema.define(version: 2020_02_06_091544) do
     t.datetime "updated_at"
     t.integer "repository_type", limit: 2
     t.index ["lfs_object_id"], name: "index_lfs_objects_projects_on_lfs_object_id"
-    t.index ["project_id"], name: "index_lfs_objects_projects_on_project_id"
+    t.index ["project_id", "lfs_object_id"], name: "index_lfs_objects_projects_on_project_id_and_lfs_object_id"
   end
   create_table "licenses", id: :serial, force: :cascade do |t|
@@ -2945,7 +2945,7 @@ ActiveRecord::Schema.define(version: 2020_02_06_091544) do
     t.datetime_with_timezone "updated_at", null: false
     t.string "package_username", limit: 255, null: false
     t.string "package_channel", limit: 255, null: false
-    t.index ["package_id"], name: "index_packages_conan_metadata_on_package_id", unique: true
+    t.index ["package_id", "package_username", "package_channel"], name: "index_packages_conan_metadata_on_package_id_username_channel", unique: true
   end
   create_table "packages_dependencies", force: :cascade do |t|
@@ -3827,7 +3827,6 @@ ActiveRecord::Schema.define(version: 2020_02_06_091544) do
     t.datetime "updated_at"
     t.boolean "active", default: false, null: false
     t.text "properties"
-    t.boolean "template", default: false
     t.boolean "push_events", default: true
     t.boolean "issues_events", default: true
     t.boolean "merge_requests_events", default: true
@@ -3844,9 +3843,10 @@ ActiveRecord::Schema.define(version: 2020_02_06_091544) do
     t.boolean "deployment_events", default: false, null: false
     t.string "description", limit: 500
     t.boolean "comment_on_event_enabled", default: true, null: false
+    t.boolean "instance", default: false
+    t.index ["instance"], name: "index_services_on_instance"
     t.index ["project_id"], name: "index_services_on_project_id"
     t.index ["project_id"], name: "tmp_index_on_project_id_partial_with_prometheus_services", where: "((type)::text = 'PrometheusService'::text)"
-    t.index ["template"], name: "index_services_on_template"
     t.index ["type"], name: "index_services_on_type"
   end

doc/api/projects.md

@@ -2000,11 +2000,6 @@ Allows modification of the forked relationship between existing projects. Availa
 ### Create a forked from/to relation between existing projects
-CAUTION: **Warning:**
-This will destroy the LFS objects stored in the fork.
-So to retain the LFS objects, make sure you've pulled them **before** creating the fork relation,
-and push them again **after** creating the fork relation.
-
 ```
 POST /projects/:id/fork/:forked_from_id
 ```

doc/user/application_security/dependency_scanning/index.md

@@ -151,6 +151,8 @@ using environment variables.
 | `PIP_REQUIREMENTS_FILE`                 | Pip requirements file to be scanned. |
 | `MAVEN_CLI_OPTS`                        | List of command line arguments that will be passed to `maven` by the analyzer. The default is `"-DskipTests --batch-mode"`. See an example for [using private repos](#using-private-maven-repos). |
 | `BUNDLER_AUDIT_UPDATE_DISABLED`         | Disable automatic updates for the `bundler-audit` analyzer (default: `"false"`). Useful if you're running Dependency Scanning in an offline, air-gapped environment.|
+| `BUNDLER_AUDIT_ADVISORY_DB_URL`         | URL of the advisory database used by bundler-audit (default: `https://github.com/rubysec/ruby-advisory-db`). |
+| `BUNDLER_AUDIT_ADVISORY_DB_REF_NAME`    | Git ref for the advisory database specified by `BUNDLER_AUDIT_ADVISORY_DB_URL` (default: `master`). |
 ### Using private Maven repos

lib/api/services.rb

@@ -132,7 +132,7 @@ module API
       helpers do
         # rubocop: disable CodeReuse/ActiveRecord
         def slash_command_service(project, service_slug, params)
-          project.services.active.where(template: false).find do |service|
+          project.services.active.where(instance: false).find do |service|
             service.try(:token) == params[:token] && service.to_param == service_slug.underscore
           end
         end

lib/gitlab/background_migration/fix_projects_without_project_feature.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # This migration creates missing project_features records
+    # for the projects within the given range of ids
+    class FixProjectsWithoutProjectFeature
+      def perform(from_id, to_id)
+        if number_of_created_records = create_missing!(from_id, to_id) > 0
+          log(number_of_created_records, from_id, to_id)
+        end
+      end
+
+      private
+
+      def create_missing!(from_id, to_id)
+        result = ActiveRecord::Base.connection.select_one(sql(from_id, to_id))
+        return 0 unless result
+
+        result['number_of_created_records']
+      end
+
+      def sql(from_id, to_id)
+        <<~SQL
+          WITH created_records AS (
+            INSERT INTO project_features (
+              project_id,
+              merge_requests_access_level,
+              issues_access_level,
+              wiki_access_level,
+              snippets_access_level,
+              builds_access_level,
+              repository_access_level,
+              forking_access_level,
+              pages_access_level,
+              created_at,
+              updated_at
+            )
+              SELECT projects.id,
+                20, 20, 20, 20, 20, 20, 20,
+                #{pages_access_level},
+                TIMEZONE('UTC', NOW()), TIMEZONE('UTC', NOW())
+              FROM projects
+              WHERE projects.id BETWEEN #{Integer(from_id)} AND #{Integer(to_id)}
+              AND NOT EXISTS (
+                SELECT 1 FROM project_features
+                WHERE project_features.project_id = projects.id
+              )
+            ON CONFLICT (project_id) DO NOTHING
+            RETURNING *
+          )
+          SELECT COUNT(*) as number_of_created_records
+          FROM created_records
+        SQL
+      end
+
+      def pages_access_level
+        if ::Gitlab::Pages.access_control_is_forced?
+          "10"
+        else
+          "GREATEST(projects.visibility_level, 10)"
+        end
+      end
+
+      def log(count, from_id, to_id)
+        logger = Gitlab::BackgroundMigration::Logger.build
+
+        logger.info(message: "FixProjectsWithoutProjectFeature: created missing project_features for #{count} projects in id=#{from_id}...#{to_id}")
+      end
+    end
+  end
+end

lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml

@@ -37,11 +37,8 @@ sast:
         fi
       fi
     - |
-      printenv | grep -E '^(DOCKER_|CI|GITLAB_|FF_|HOME|PWD|OLDPWD|PATH|SHLVL|HOSTNAME)' | cut -d'=' -f1 | \
-        (while IFS='\\n' read -r VAR; do unset -v "$VAR"; done; /bin/printenv > .env)
-    - |
-      docker run \
-        --env-file .env \
+      ENVS=`printenv | grep -vE '^(DOCKER_|CI|GITLAB_|FF_|HOME|PWD|OLDPWD|PATH|SHLVL|HOSTNAME)' | sed -n '/^[^\t]/s/=.*//p' | sed '/^$/d' | sed 's/^/-e /g' | tr '\n' ' '`
+      docker run "$ENVS" \
         --volume "$PWD:/code" \
         --volume /var/run/docker.sock:/var/run/docker.sock \
         "registry.gitlab.com/gitlab-org/security-products/sast:$SAST_VERSION" /app/bin/run /code

lib/gitlab/graphql/connections/keyset/conditions/base_condition.rb

@@ -6,6 +6,12 @@ module Gitlab
       module Keyset
         module Conditions
           class BaseCondition
+            # @param [Arel::Table] arel_table for the relation being ordered
+            # @param [Array<OrderInfo>] order_list of extracted orderings
+            # @param [Array] values from the decoded cursor
+            # @param [Array<String>] operators determining sort comparison
+            # @param [Symbol] before_or_after indicates whether we want
+            #        items :before the cursor or :after the cursor
             def initialize(arel_table, order_list, values, operators, before_or_after)
               @arel_table, @order_list, @values, @operators, @before_or_after = arel_table, order_list, values, operators, before_or_after
@@ -20,18 +26,25 @@ module Gitlab
             attr_reader :arel_table, :order_list, :values, :operators, :before_or_after
-            def table_condition(attribute, value, operator)
+            def table_condition(order_info, value, operator)
+              if order_info.named_function
+                target = order_info.named_function
+                value  = value&.downcase if target&.name&.downcase == 'lower'
+              else
+                target = arel_table[order_info.attribute_name]
+              end
+
               case operator
               when '>'
-                arel_table[attribute].gt(value)
+                target.gt(value)
               when '<'
-                arel_table[attribute].lt(value)
+                target.lt(value)
               when '='
-                arel_table[attribute].eq(value)
+                target.eq(value)
               when 'is_null'
-                arel_table[attribute].eq(nil)
+                target.eq(nil)
               when 'is_not_null'
-                arel_table[attribute].not_eq(nil)
+                target.not_eq(nil)
               end
             end
           end

lib/gitlab/graphql/connections/keyset/order_info.rb

@@ -5,10 +5,10 @@ module Gitlab
     module Connections
       module Keyset
         class OrderInfo
-          attr_reader :attribute_name, :sort_direction
+          attr_reader :attribute_name, :sort_direction, :named_function
           def initialize(order_value)
-            @attribute_name, @sort_direction =
+            @attribute_name, @sort_direction, @named_function =
               if order_value.is_a?(String)
                 extract_nulls_last_order(order_value)
               else
@@ -69,11 +69,24 @@ module Gitlab
           def extract_nulls_last_order(order_value)
             tokens = order_value.downcase.split
-            [tokens.first, (tokens[1] == 'asc' ? :asc : :desc)]
+            [tokens.first, (tokens[1] == 'asc' ? :asc : :desc), nil]
           end
           def extract_attribute_values(order_value)
-            [order_value.expr.name, order_value.direction]
+            named = nil
+            name  = if ordering_by_lower?(order_value)
+                      named = order_value.expr
+                      named.expressions[0].name.to_s
+                    else
+                      order_value.expr.name
+                    end
+
+            [name, order_value.direction, named]
+          end
+
+          # determine if ordering using LOWER, eg. "ORDER BY LOWER(boards.name)"
+          def ordering_by_lower?(order_value)
+            order_value.expr.is_a?(Arel::Nodes::NamedFunction) && order_value.expr&.name&.downcase == 'lower'
           end
         end
       end

lib/gitlab/graphql/connections/keyset/query_builder.rb

@@ -40,17 +40,16 @@ module Gitlab
           #       "issues"."id" > 500
           #
           def conditions
-            attr_names  = order_list.map { |field| field.attribute_name }
-            attr_values = attr_names.map { |name| decoded_cursor[name] }
-            if attr_names.count == 1 && attr_values.first.nil?
+            attr_values = order_list.map { |field| decoded_cursor[field.attribute_name] }
+            if order_list.count == 1 && attr_values.first.nil?
               raise Gitlab::Graphql::Errors::ArgumentError.new('Before/after cursor invalid: `nil` was provided as only sortable value')
             end
-            if attr_names.count == 1 || attr_values.first.present?
-              Keyset::Conditions::NotNullCondition.new(arel_table, attr_names, attr_values, operators, before_or_after).build
+            if order_list.count == 1 || attr_values.first.present?
+              Keyset::Conditions::NotNullCondition.new(arel_table, order_list, attr_values, operators, before_or_after).build
             else
-              Keyset::Conditions::NullCondition.new(arel_table, attr_names, attr_values, operators, before_or_after).build
+              Keyset::Conditions::NullCondition.new(arel_table, order_list, attr_values, operators, before_or_after).build
             end
           end