diff --git a/app/assets/javascripts/build.js b/app/assets/javascripts/build.js index ae1a23132a73bbcd5b1a29c4202dbcc5e21e2f54..e85a7bc1772897c85d11b30aac3ca164f375d4b1 100644 --- a/app/assets/javascripts/build.js +++ b/app/assets/javascripts/build.js @@ -166,7 +166,7 @@ window.Build = (function () { Build.prototype.getBuildTrace = function () { return $.ajax({ url: `${this.pageUrl}/trace.json`, - data: this.state, + data: { state: this.state }, }) .done((log) => { gl.utils.setCiStatusFavicon(`${this.pageUrl}/status.json`); diff --git a/app/assets/javascripts/pipelines/components/pipeline_url.vue b/app/assets/javascripts/pipelines/components/pipeline_url.vue index f0b44dfa6d8c04c3035160e3d6c9e59d6761d048..76b97af39f1e22c8ee3a15baf99bbae9ba417cc7 100644 --- a/app/assets/javascripts/pipelines/components/pipeline_url.vue +++ b/app/assets/javascripts/pipelines/components/pipeline_url.vue @@ -28,8 +28,7 @@ popoverOptions() { return { html: true, - delay: { hide: 600 }, - trigger: 'hover', + trigger: 'focus', placement: 'top', title: '<div class="autodevops-title">This pipeline makes use of a predefined CI/CD configuration enabled by <b>Auto DevOps.</b></div>', content: `<a class="autodevops-link" href="${this.autoDevopsHelpPath}" target="_blank" rel="noopener noreferrer nofollow">Learn more about Auto DevOps</a>`, @@ -75,6 +74,7 @@ </span> <a v-if="pipeline.flags.auto_devops" + tabindex="0" class="js-pipeline-url-autodevops label label-info autodevops-badge" v-popover="popoverOptions" role="button"> diff --git a/app/helpers/issuables_helper.rb b/app/helpers/issuables_helper.rb index 1fcb74930d2c96df91af0bf752323da1a0b0a30f..a681acc4bf5b619098312f17b07f5f607bf2ee6b 100644 --- a/app/helpers/issuables_helper.rb +++ b/app/helpers/issuables_helper.rb @@ -249,16 +249,25 @@ def issuables_count_for_state(issuable_type, state) Gitlab::IssuablesCountForState.new(finder)[state] end - def close_issuable_url(issuable) - issuable_url(issuable, close_reopen_params(issuable, :close)) + def close_issuable_path(issuable) + issuable_path(issuable, close_reopen_params(issuable, :close)) end - def reopen_issuable_url(issuable) - issuable_url(issuable, close_reopen_params(issuable, :reopen)) + def reopen_issuable_path(issuable) + issuable_path(issuable, close_reopen_params(issuable, :reopen)) end - def close_reopen_issuable_url(issuable, should_inverse = false) - issuable.closed? ^ should_inverse ? reopen_issuable_url(issuable) : close_issuable_url(issuable) + def close_reopen_issuable_path(issuable, should_inverse = false) + issuable.closed? ^ should_inverse ? reopen_issuable_path(issuable) : close_issuable_path(issuable) + end + + def issuable_path(issuable, *options) + case issuable + when Issue + issue_path(issuable, *options) + when MergeRequest + merge_request_path(issuable, *options) + end end def issuable_url(issuable, *options) diff --git a/app/models/project.rb b/app/models/project.rb index 2d4cec34a118a57bfaa008649dd8b4ec29f565ee..ed0561aef73d5067d96bb97899ca2f59b43e343e 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -75,6 +75,7 @@ class Project < ActiveRecord::Base attr_accessor :old_path_with_namespace attr_accessor :template_name attr_writer :pipeline_status + attr_accessor :skip_disk_validation alias_attribute :title, :name @@ -231,7 +232,7 @@ class Project < ActiveRecord::Base validates :import_url, importable_url: true, if: [:external_import?, :import_url_changed?] validates :star_count, numericality: { greater_than_or_equal_to: 0 } validate :check_limit, on: :create - validate :can_create_repository?, on: [:create, :update], if: ->(project) { !project.persisted? || project.renamed? } + validate :check_repository_path_availability, on: [:create, :update], if: ->(project) { !project.persisted? || project.renamed? } validate :avatar_type, if: ->(project) { project.avatar.present? && project.avatar_changed? } validates :avatar, file_size: { maximum: 200.kilobytes.to_i } @@ -1015,7 +1016,8 @@ def expire_caches_before_rename(old_path) end # Check if repository already exists on disk - def can_create_repository? + def check_repository_path_availability + return true if skip_disk_validation return false unless repository_storage_path expires_full_path_cache # we need to clear cache to validate renames correctly diff --git a/app/models/repository.rb b/app/models/repository.rb index 43f2f2eda8a92aaeb4dd24963e069b84925dc989..264eefd693728883441578ea63a667e808ff1a9a 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -40,7 +40,10 @@ class Repository CACHED_METHODS = %i(size commit_count rendered_readme contribution_guide changelog license_blob license_key gitignore koding_yml gitlab_ci_yml branch_names tag_names branch_count - tag_count avatar exists? empty? root_ref).freeze + tag_count avatar exists? empty? root_ref has_visible_content?).freeze + + # Methods that use cache_method but only memoize the value + MEMOIZED_CACHED_METHODS = %i(license).freeze # Certain method caches should be refreshed when certain types of files are # changed. This Hash maps file types (as returned by Gitlab::FileDetector) to @@ -97,12 +100,6 @@ def path_to_repo ) end - # we need to have this method here because it is not cached in ::Git and - # the method is called multiple times for every request - def has_visible_content? - branch_count > 0 - end - def inspect "#<#{self.class.name}:#{@disk_path}>" end @@ -281,7 +278,7 @@ def expire_tags_cache end def expire_branches_cache - expire_method_caches(%i(branch_names branch_count)) + expire_method_caches(%i(branch_names branch_count has_visible_content?)) @local_branches = nil @branch_exists_memo = nil end @@ -352,7 +349,7 @@ def expire_root_ref_cache def expire_emptiness_caches return unless empty? - expire_method_caches(%i(empty?)) + expire_method_caches(%i(empty? has_visible_content?)) end def lookup_cache @@ -529,9 +526,10 @@ def commit_count_for_ref(ref) delegate :tag_names, to: :raw_repository cache_method :tag_names, fallback: [] - delegate :branch_count, :tag_count, to: :raw_repository + delegate :branch_count, :tag_count, :has_visible_content?, to: :raw_repository cache_method :branch_count, fallback: 0 cache_method :tag_count, fallback: 0 + cache_method :has_visible_content?, fallback: false def avatar if tree = file_on_head(:avatar) diff --git a/app/uploaders/object_store_uploader.rb b/app/uploaders/object_store_uploader.rb index 79edc84ef4f3a57fe14a5bc1e81bce8f8388bdea..95f1b593e7e29a39270e3d321243608abe834aa2 100644 --- a/app/uploaders/object_store_uploader.rb +++ b/app/uploaders/object_store_uploader.rb @@ -159,7 +159,7 @@ def verify_license!(new_file) end def exists? - file.try(:exists?) + file.present? end def cache_dir diff --git a/app/views/shared/issuable/_close_reopen_button.html.haml b/app/views/shared/issuable/_close_reopen_button.html.haml index f16bc8dd43077ed5f384d521b3b0781cf64cc3de..9ef015047c9801177043fad28b891109acb0a22d 100644 --- a/app/views/shared/issuable/_close_reopen_button.html.haml +++ b/app/views/shared/issuable/_close_reopen_button.html.haml @@ -3,9 +3,9 @@ - button_method = issuable_close_reopen_button_method(issuable) - if can_update && is_current_user - = link_to "Close #{display_issuable_type}", close_issuable_url(issuable), method: button_method, + = link_to "Close #{display_issuable_type}", close_issuable_path(issuable), method: button_method, class: "hidden-xs hidden-sm btn btn-grouped btn-close js-btn-issue-action #{issuable_button_visibility(issuable, true)}", title: "Close #{display_issuable_type}" - = link_to "Reopen #{display_issuable_type}", reopen_issuable_url(issuable), method: button_method, + = link_to "Reopen #{display_issuable_type}", reopen_issuable_path(issuable), method: button_method, class: "hidden-xs hidden-sm btn btn-grouped btn-reopen js-btn-issue-action #{issuable_button_visibility(issuable, false)}", title: "Reopen #{display_issuable_type}" - elsif can_update && !is_current_user = render 'shared/issuable/close_reopen_report_toggle', issuable: issuable diff --git a/app/views/shared/issuable/_close_reopen_report_toggle.html.haml b/app/views/shared/issuable/_close_reopen_report_toggle.html.haml index a38cd319e3caf5d38d3101bbd9d8ee3d9369be9b..39a5171c1d6c41500719b693fff63e90b53aac19 100644 --- a/app/views/shared/issuable/_close_reopen_report_toggle.html.haml +++ b/app/views/shared/issuable/_close_reopen_report_toggle.html.haml @@ -7,7 +7,7 @@ - button_method = issuable_close_reopen_button_method(issuable) .pull-left.btn-group.prepend-left-10.issuable-close-dropdown.droplab-dropdown.js-issuable-close-dropdown - = link_to "#{display_button_action} #{display_issuable_type}", close_reopen_issuable_url(issuable), + = link_to "#{display_button_action} #{display_issuable_type}", close_reopen_issuable_path(issuable), method: button_method, class: "#{button_class} btn-#{button_action}", title: "#{display_button_action} #{display_issuable_type}" = button_tag type: 'button', class: "#{toggle_class} btn-#{button_action}-color", @@ -16,7 +16,7 @@ %ul#issuable-close-menu.js-issuable-close-menu.dropdown-menu{ class: button_responsive_class, data: { dropdown: true } } %li.close-item{ class: "#{issuable_button_visibility(issuable, true) || 'droplab-item-selected'}", - data: { text: "Close #{display_issuable_type}", url: close_issuable_url(issuable), + data: { text: "Close #{display_issuable_type}", url: close_issuable_path(issuable), button_class: "#{button_class} btn-close", toggle_class: "#{toggle_class} btn-close-color", method: button_method } } %button.btn.btn-transparent = icon('check', class: 'icon') @@ -26,7 +26,7 @@ = display_issuable_type %li.reopen-item{ class: "#{issuable_button_visibility(issuable, false) || 'droplab-item-selected'}", - data: { text: "Reopen #{display_issuable_type}", url: reopen_issuable_url(issuable), + data: { text: "Reopen #{display_issuable_type}", url: reopen_issuable_path(issuable), button_class: "#{button_class} btn-reopen", toggle_class: "#{toggle_class} btn-reopen-color", method: button_method } } %button.btn.btn-transparent = icon('check', class: 'icon') diff --git a/changelogs/unreleased-ee/do-not-perform-disk-check.yml b/changelogs/unreleased-ee/do-not-perform-disk-check.yml new file mode 100644 index 0000000000000000000000000000000000000000..cc139ee2c9e240daf53edfb13ed8709a733ad3bd --- /dev/null +++ b/changelogs/unreleased-ee/do-not-perform-disk-check.yml @@ -0,0 +1,5 @@ +--- +title: File uploaders do not perform hard check, only soft check +merge_request: +author: +type: fixed diff --git a/changelogs/unreleased-ee/sh-delta-check-with-annotated-tags.yml b/changelogs/unreleased-ee/sh-delta-check-with-annotated-tags.yml new file mode 100644 index 0000000000000000000000000000000000000000..0d7c2cdfda7ec11c4b9fcc8215e19710b09ab603 --- /dev/null +++ b/changelogs/unreleased-ee/sh-delta-check-with-annotated-tags.yml @@ -0,0 +1,5 @@ +--- +title: Fix EE delta size check handling with annotated tags +merge_request: +author: +type: fixed diff --git a/changelogs/unreleased-ee/tc-geo-use-database-tasks.yml b/changelogs/unreleased-ee/tc-geo-use-database-tasks.yml new file mode 100644 index 0000000000000000000000000000000000000000..202bb801ae296f403170ce5897e03ee96d8ee929 --- /dev/null +++ b/changelogs/unreleased-ee/tc-geo-use-database-tasks.yml @@ -0,0 +1,5 @@ +--- +title: Rewrite Geo database rake tasks so they operate on the correct database +merge_request: 3052 +author: +type: fixed diff --git a/changelogs/unreleased/38528-build-url.yml b/changelogs/unreleased/38528-build-url.yml new file mode 100644 index 0000000000000000000000000000000000000000..357b9aacea8d42c3f62bceebf4d2b0f8efa9d581 --- /dev/null +++ b/changelogs/unreleased/38528-build-url.yml @@ -0,0 +1,5 @@ +--- +title: Fixes data parameter not being sent in ajax request for jobs log +merge_request: +author: +type: fixed diff --git a/changelogs/unreleased/38582-popover-badge.yml b/changelogs/unreleased/38582-popover-badge.yml new file mode 100644 index 0000000000000000000000000000000000000000..ccec679a13f117920f5f93163d2fea3eacb07e9f --- /dev/null +++ b/changelogs/unreleased/38582-popover-badge.yml @@ -0,0 +1,5 @@ +--- +title: Improves UX of autodevops popover to match gpg one +merge_request: +author: +type: fixed diff --git a/changelogs/unreleased/bvl-fix-close-issuable-link.yml b/changelogs/unreleased/bvl-fix-close-issuable-link.yml new file mode 100644 index 0000000000000000000000000000000000000000..140a9d35cc15319daa9670bcbb8591f7333e9acb --- /dev/null +++ b/changelogs/unreleased/bvl-fix-close-issuable-link.yml @@ -0,0 +1,5 @@ +--- +title: Fix CSRF validation issue when closing/opening merge requests from the UI +merge_request: 14555 +author: +type: fixed diff --git a/changelogs/unreleased/fix-kubectl-180.yml b/changelogs/unreleased/fix-kubectl-180.yml new file mode 100644 index 0000000000000000000000000000000000000000..beb71cecd57c4beaed9892c0f241a610ed592f7e --- /dev/null +++ b/changelogs/unreleased/fix-kubectl-180.yml @@ -0,0 +1,5 @@ +--- +title: 'Kubernetes integration: ensure v1.8.0 compatibility' +merge_request: 14635 +author: +type: fixed diff --git a/changelogs/unreleased/sh-fix-import-repos.yml b/changelogs/unreleased/sh-fix-import-repos.yml new file mode 100644 index 0000000000000000000000000000000000000000..5764b3bdc01a4521f47ae7f9e4567d3ce72c00ef --- /dev/null +++ b/changelogs/unreleased/sh-fix-import-repos.yml @@ -0,0 +1,5 @@ +--- +title: Fix gitlab-rake gitlab:import:repos task failing +merge_request: +author: +type: fixed diff --git a/changelogs/unreleased/sh-fix-issue-38646.yml b/changelogs/unreleased/sh-fix-issue-38646.yml new file mode 100644 index 0000000000000000000000000000000000000000..5c205775662e3654e75befb222476831190ee4f5 --- /dev/null +++ b/changelogs/unreleased/sh-fix-issue-38646.yml @@ -0,0 +1,5 @@ +--- +title: Fix pushes to an empty repository not invalidating has_visible_content? cache +merge_request: +author: +type: fixed diff --git a/changelogs/unreleased/sh-restore-all-refs-backup.yml b/changelogs/unreleased/sh-restore-all-refs-backup.yml new file mode 100644 index 0000000000000000000000000000000000000000..eaac0c71dd0fed62ceece2771dfd66cafbd4ed79 --- /dev/null +++ b/changelogs/unreleased/sh-restore-all-refs-backup.yml @@ -0,0 +1,5 @@ +--- +title: Ensure all refs are restored on a restore from backup +merge_request: +author: +type: fixed diff --git a/doc/topics/autodevops/index.md b/doc/topics/autodevops/index.md index 0b7c11ea7009597bdeb2b9316cf9df6cf6776398..5561784ed0b1d514d9e95fe08a00d027a9b3644c 100644 --- a/doc/topics/autodevops/index.md +++ b/doc/topics/autodevops/index.md @@ -511,7 +511,7 @@ sudo gitlab-rails console Then run: ```ruby -Feature.get(:auto_devops_banner_disabled).disable +Feature.get(:auto_devops_banner_disabled).enable ``` Or through the HTTP API with an admin access token: diff --git a/ee/lib/ee/gitlab/deltas.rb b/ee/lib/ee/gitlab/deltas.rb index 470e8604624ffe26e585d16076ab3cf1f7489d31..c765730dc4566f66d936f9267f70baf45b564ef5 100644 --- a/ee/lib/ee/gitlab/deltas.rb +++ b/ee/lib/ee/gitlab/deltas.rb @@ -7,6 +7,9 @@ def self.delta_size_check(change, repo) begin tree_a = repo.lookup(change[:oldrev]) tree_b = repo.lookup(change[:newrev]) + + return size_of_deltas unless diffable?(tree_a) && diffable?(tree_b) + diff = tree_a.diff(tree_b) diff.each_delta do |d| @@ -25,6 +28,10 @@ def self.delta_size_check(change, repo) size_of_deltas end end + + def self.diffable?(object) + [Rugged::Commit, Rugged::Tree].include?(object.class) + end end end end diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 4e92be8511098176f51795479782f4fee4bfe018..3ad09a1b421356c6ff6f13de084b38c3eac1ba1d 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -78,7 +78,7 @@ def restore project.ensure_storage_path_exists cmd = if File.exist?(path_to_project_bundle) - %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_project_bundle} #{path_to_project_repo}) + %W(#{Gitlab.config.git.bin_path} clone --bare --mirror #{path_to_project_bundle} #{path_to_project_repo}) else %W(#{Gitlab.config.git.bin_path} init --bare #{path_to_project_repo}) end diff --git a/lib/gitlab/bare_repository_importer.rb b/lib/gitlab/bare_repository_importer.rb index 9323bfc7fb23139b827d6964a86800824a2339d3..1d98d187805f316d0287c7ec7bd56f222096f1e8 100644 --- a/lib/gitlab/bare_repository_importer.rb +++ b/lib/gitlab/bare_repository_importer.rb @@ -56,7 +56,8 @@ def create_project name: project_path, path: project_path, repository_storage: storage_name, - namespace_id: group&.id + namespace_id: group&.id, + skip_disk_validation: true } project = Projects::CreateService.new(user, project_params).execute diff --git a/lib/gitlab/geo/database_tasks.rb b/lib/gitlab/geo/database_tasks.rb new file mode 100644 index 0000000000000000000000000000000000000000..fab3bd13834d0461c5428767b7d3a8a71bd65b24 --- /dev/null +++ b/lib/gitlab/geo/database_tasks.rb @@ -0,0 +1,193 @@ +module Gitlab + module Geo + module DatabaseTasks + extend self + + DATABASE_CONFIG = 'config/database.yml'.freeze + GEO_DATABASE_CONFIG = 'config/database_geo.yml'.freeze + GEO_DB_DIR = 'db/geo'.freeze + + def method_missing(method_name, *args, &block) + with_geo_db do + ActiveRecord::Tasks::DatabaseTasks.public_send(method_name, *args, &block) # rubocop:disable GitlabSecurity/PublicSend + end + end + + def respond_to_missing?(method_name, include_private = false) + ActiveRecord::Tasks::DatabaseTasks.respond_to?(method_name) || super + end + + def rollback + step = ENV['STEP'] ? ENV['STEP'].to_i : 1 + + with_geo_db do + ActiveRecord::Migrator.rollback(ActiveRecord::Migrator.migrations_paths, step) + end + end + + def version + with_geo_db do + ActiveRecord::Migrator.current_version + end + end + + def dump_schema_after_migration? + with_geo_db do + !!ActiveRecord::Base.dump_schema_after_migration + end + end + + def pending_migrations + with_geo_db do + ActiveRecord::Migrator.open(ActiveRecord::Migrator.migrations_paths).pending_migrations + end + end + + def abort_if_no_geo_config! + @geo_config_exists ||= File.exist?(Rails.root.join(GEO_DATABASE_CONFIG)) + + unless @geo_config_exists + abort("Failed to open #{GEO_DATABASE_CONFIG}. Consult the documentation on how to set up GitLab Geo.") + end + end + + module Schema + extend self + + def dump + require 'active_record/schema_dumper' + + Gitlab::Geo::DatabaseTasks.with_geo_db do + filename = ENV['SCHEMA'] || File.join(ActiveRecord::Tasks::DatabaseTasks.db_dir, 'schema.rb') + File.open(filename, "w:utf-8") do |file| + ActiveRecord::SchemaDumper.dump(ActiveRecord::Base.connection, file) + end + end + end + end + + module Migrate + extend self + + def up + version = ENV['VERSION'] ? ENV['VERSION'].to_i : nil + raise 'VERSION is required' unless version + + Gitlab::Geo::DatabaseTasks.with_geo_db do + ActiveRecord::Migrator.run(:up, ActiveRecord::Migrator.migrations_paths, version) + end + end + + def down + version = ENV['VERSION'] ? ENV['VERSION'].to_i : nil + raise 'VERSION is required - To go down one migration, run db:rollback' unless version + + Gitlab::Geo::DatabaseTasks.with_geo_db do + ActiveRecord::Migrator.run(:down, ActiveRecord::Migrator.migrations_paths, version) + end + end + + # rubocop: disable Rails/Output + def status + Gitlab::Geo::DatabaseTasks.with_geo_db do + unless ActiveRecord::SchemaMigration.table_exists? + abort 'Schema migrations table does not exist yet.' + end + + db_list = ActiveRecord::SchemaMigration.normalized_versions + file_list = + ActiveRecord::Migrator.migrations_paths.flat_map do |path| + # match "20091231235959_some_name.rb" and "001_some_name.rb" pattern + Dir.foreach(path).grep(/^(\d{3,})_(.+)\.rb$/) do + version = ActiveRecord::SchemaMigration.normalize_migration_number($1) + status = db_list.delete(version) ? 'up' : 'down' + [status, version, $2.humanize] + end + end + + db_list.map! do |version| + ['up', version, '********** NO FILE **********'] + end + # output + puts "\ndatabase: #{ActiveRecord::Base.connection_config[:database]}\n\n" + puts "#{'Status'.center(8)} #{'Migration ID'.ljust(14)} Migration Name" + puts "-" * 50 + (db_list + file_list).sort_by { |_, version, _| version }.each do |status, version, name| + puts "#{status.center(8)} #{version.ljust(14)} #{name}" + end + puts + end + end + # rubocop: enable Rails/Output + end + + module Test + extend self + + def load + Gitlab::Geo::DatabaseTasks.with_geo_db do + begin + should_reconnect = ActiveRecord::Base.connection_pool.active_connection? + ActiveRecord::Schema.verbose = false + ActiveRecord::Tasks::DatabaseTasks.load_schema_for ActiveRecord::Base.configurations['test'], :ruby, ENV['SCHEMA'] + ensure + if should_reconnect + ActiveRecord::Base.establish_connection(ActiveRecord::Base.configurations[ActiveRecord::Tasks::DatabaseTasks.env]) + end + end + end + end + + def purge + Gitlab::Geo::DatabaseTasks.with_geo_db do + ActiveRecord::Tasks::DatabaseTasks.purge ActiveRecord::Base.configurations['test'] + end + end + end + + def geo_settings + { + database_config: YAML.load_file(GEO_DATABASE_CONFIG), + db_dir: GEO_DB_DIR, + migrations_paths: [Rails.root.join(GEO_DB_DIR, 'migrate')], + seed_loader: SeedLoader.new + } + end + + def with_geo_db + abort_if_no_geo_config! + + original_settings = { + database_config: ActiveRecord::Tasks::DatabaseTasks.database_configuration&.dup || YAML.load_file(DATABASE_CONFIG), + db_dir: ActiveRecord::Tasks::DatabaseTasks.db_dir, + migrations_paths: ActiveRecord::Tasks::DatabaseTasks.migrations_paths, + seed_loader: ActiveRecord::Tasks::DatabaseTasks.seed_loader + } + + set_db_env(geo_settings) + + yield + ensure + set_db_env(original_settings) + end + + def set_db_env(settings) + ActiveRecord::Tasks::DatabaseTasks.database_configuration = settings[:database_config] + ActiveRecord::Tasks::DatabaseTasks.db_dir = settings[:db_dir] + ActiveRecord::Tasks::DatabaseTasks.migrations_paths = settings[:migrations_paths] + ActiveRecord::Tasks::DatabaseTasks.seed_loader = settings[:seed_loader] + + ActiveRecord::Base.configurations = ActiveRecord::Tasks::DatabaseTasks.database_configuration || {} + ActiveRecord::Migrator.migrations_paths = ActiveRecord::Tasks::DatabaseTasks.migrations_paths + + ActiveRecord::Base.establish_connection(ActiveRecord::Base.configurations[ActiveRecord::Tasks::DatabaseTasks.env]) + end + + class SeedLoader + def load_seed + load('db/geo/seeds.rb') + end + end + end + end +end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 6c639f286ee2a1963e0d317244f5284561850f9d..984e34a9ef1d658725d0c50eb7f4c0e1ea6f1dd0 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -190,6 +190,28 @@ def branch_count end end + def has_local_branches? + gitaly_migrate(:has_local_branches) do |is_enabled| + if is_enabled + gitaly_ref_client.has_local_branches? + else + has_local_branches_rugged? + end + end + end + + def has_local_branches_rugged? + rugged.branches.each(:local).any? do |ref| + begin + ref.name && ref.target # ensures the branch is valid + + true + rescue Rugged::ReferenceError + false + end + end + end + # Returns the number of valid tags def tag_count gitaly_migrate(:tag_names) do |is_enabled| @@ -909,7 +931,9 @@ def empty_repo? # This method return true if repository contains some content visible in project page. # def has_visible_content? - branch_count > 0 + return @has_visible_content if defined?(@has_visible_content) + + @has_visible_content = has_local_branches? end def gitaly_repository diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 8ef873d58489601d020863a0ba15fa13b6e8ceee..52c1f1ab3d05284fd9d7122ee1aa61cdb6f34093 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -57,6 +57,14 @@ def count_branch_names branch_names.count end + # TODO implement a more efficient RPC for this https://gitlab.com/gitlab-org/gitaly/issues/616 + def has_local_branches? + request = Gitaly::FindAllBranchNamesRequest.new(repository: @gitaly_repo) + response = GitalyClient.call(@storage, :ref_service, :find_all_branch_names, request).first + + response&.names.present? + end + def local_branches(sort_by: nil) request = Gitaly::FindLocalBranchesRequest.new(repository: @gitaly_repo) request.sort_by = sort_by_param(sort_by) if sort_by diff --git a/lib/gitlab/kubernetes.rb b/lib/gitlab/kubernetes.rb index cdbdfa10d0e7084d87636ec1bba3e3f2b292fe14..da43bd0af4b567ea46c15b37836ec73d9a2d1f56 100644 --- a/lib/gitlab/kubernetes.rb +++ b/lib/gitlab/kubernetes.rb @@ -113,7 +113,7 @@ def to_kubeconfig(url:, namespace:, token:, ca_pem: nil) def kubeconfig_embed_ca_pem(config, ca_pem) cluster = config.dig(:clusters, 0, :cluster) - cluster[:'certificate-authority-data'] = Base64.encode64(ca_pem) + cluster[:'certificate-authority-data'] = Base64.strict_encode64(ca_pem) end end end diff --git a/lib/tasks/geo.rake b/lib/tasks/geo.rake index 49e8433486081daf811b02504188840ad668d546..6fb3e667b47c166c695a9743e325720f19c1eef4 100644 --- a/lib/tasks/geo.rake +++ b/lib/tasks/geo.rake @@ -1,80 +1,145 @@ +require 'gitlab/geo' +require 'gitlab/geo/database_tasks' + task spec: ['geo:db:test:prepare'] namespace :geo do namespace :db do |ns| - %i(drop create setup migrate rollback seed version reset).each do |task_name| - task task_name do - Rake::Task["db:#{task_name}"].invoke - end + desc 'Drops the Geo tracking database from config/database_geo.yml for the current RAILS_ENV.' + task :drop do + Gitlab::Geo::DatabaseTasks.drop_current end - namespace :schema do - %i(load dump).each do |task_name| - task task_name do - Rake::Task["db:schema:#{task_name}"].invoke - end - end + desc 'Creates the Geo tracking database from config/database_geo.yml for the current RAILS_ENV.' + task :create do + Gitlab::Geo::DatabaseTasks.create_current end - namespace :migrate do - %i(up down redo).each do |task_name| - task task_name do - Rake::Task["db:migrate:#{task_name}"].invoke - end - end + desc 'Create the Geo tracking database, load the schema, and initialize with the seed data.' + task setup: ['geo:db:schema:load', 'geo:db:seed'] + + desc 'Migrate the Geo tracking database (options: VERSION=x, VERBOSE=false, SCOPE=blog).' + task migrate: [:environment] do + Gitlab::Geo::DatabaseTasks.migrate + + ns['_dump'].invoke end - namespace :test do - task :prepare do - Rake::Task['db:test:prepare'].invoke - end + desc 'Rolls the schema back to the previous version (specify steps w/ STEP=n).' + task rollback: [:environment] do + Gitlab::Geo::DatabaseTasks.rollback + + ns['_dump'].invoke end - # append and prepend proper tasks to all the tasks defined above - ns.tasks.each do |task| - task.enhance ['geo:config:check', 'geo:config:set'] do - Rake::Task['geo:config:restore'].invoke + desc 'Retrieves the current schema version number.' + task version: [:environment] do + puts "Current version: #{Gitlab::Geo::DatabaseTasks.version}" + end - # Reenable the tasks, otherwise the following tasks are run only once - # per invocation of `rake`! - Rake::Task['geo:config:check'].reenable - Rake::Task['geo:config:set'].reenable - Rake::Task['geo:config:restore'].reenable - end + desc 'Drops and recreates the database from db/geo/schema.rb for the current environment and loads the seeds.' + task reset: [:environment] do + ns['drop'].invoke + ns['create'].invoke + ns['setup'].invoke + end + + desc 'Load the seed data from db/geo/seeds.rb' + task seed: [:environment] do + ns['abort_if_pending_migrations'].invoke + + Gitlab::Geo::DatabaseTasks.load_seed end desc 'Display database encryption key' task show_encryption_key: :environment do puts Rails.application.secrets.db_key_base end - end - namespace :config do - task :check do - unless File.exist?(Rails.root.join('config/database_geo.yml')) - abort('You should run these tasks only when GitLab Geo is enabled.') + # IMPORTANT: This task won't dump the schema if ActiveRecord::Base.dump_schema_after_migration is set to false + task :_dump do + if Gitlab::Geo::DatabaseTasks.dump_schema_after_migration? + ns["schema:dump"].invoke + end + # Allow this task to be called as many times as required. An example is the + # migrate:redo task, which calls other two internally that depend on this one. + ns['_dump'].reenable + end + + # desc "Raises an error if there are pending migrations" + task abort_if_pending_migrations: [:environment] do + pending_migrations = Gitlab::Geo::DatabaseTasks.pending_migrations + + if pending_migrations.any? + puts "You have #{pending_migrations.size} pending #{pending_migrations.size > 1 ? 'migrations:' : 'migration:'}" + pending_migrations.each do |pending_migration| + puts ' %4d %s' % [pending_migration.version, pending_migration.name] + end + abort %{Run `rake geo:db:migrate` to update your database then try again.} + end + end + + namespace :schema do + desc 'Load a schema.rb file into the database' + task load: [:environment] do + Gitlab::Geo::DatabaseTasks.load_schema_current(:ruby, ENV['SCHEMA']) + end + + desc 'Create a db/geo/schema.rb file that is portable against any DB supported by AR' + task dump: [:environment] do + Gitlab::Geo::DatabaseTasks::Schema.dump + + ns['schema:dump'].reenable end end - task :set do - # save current configuration - @previous_config = { - config: Rails.application.config.dup, - schema: ENV['SCHEMA'] - } - - # set config variables for geo database - ENV['SCHEMA'] = 'db/geo/schema.rb' - Rails.application.config.paths['db'] = ['db/geo'] - Rails.application.config.paths['db/migrate'] = ['db/geo/migrate'] - Rails.application.config.paths['db/seeds.rb'] = ['db/geo/seeds.rb'] - Rails.application.config.paths['config/database'] = ['config/database_geo.yml'] + namespace :migrate do + desc 'Runs the "up" for a given migration VERSION.' + task up: [:environment] do + Gitlab::Geo::DatabaseTasks::Migrate.up + + ns['_dump'].invoke + end + + desc 'Runs the "down" for a given migration VERSION.' + task down: [:environment] do + Gitlab::Geo::DatabaseTasks::Migrate.down + + ns['_dump'].invoke + end + + desc 'Rollbacks the database one migration and re migrate up (options: STEP=x, VERSION=x).' + task redo: [:environment] do + if ENV['VERSION'] + ns['migrate:down'].invoke + ns['migrate:up'].invoke + else + ns['rollback'].invoke + ns['migrate'].invoke + end + end + + desc 'Display status of migrations' + task status: [:environment] do + Gitlab::Geo::DatabaseTasks::Migrate.status + end end - task :restore do - # restore config variables to previous values - ENV['SCHEMA'] = @previous_config[:schema] - Rails.application.config = @previous_config[:config] + namespace :test do + desc 'Check for pending migrations and load the test schema' + task prepare: [:environment] do + ns['test:load'].invoke + end + + # desc "Recreate the test database from the current schema" + task load: [:environment, 'geo:db:test:purge'] do + Gitlab::Geo::DatabaseTasks::Test.load + end + + # desc "Empty the test database" + task purge: [:environment] do + Gitlab::Geo::DatabaseTasks::Test.purge + end end end diff --git a/qa/qa/page/admin/menu.rb b/qa/qa/page/admin/menu.rb index b01a4e10f93fd183f7a9f42587382055c6f3bf12..f4619042e34e57becc3b22f66be4b0390f6bdf3c 100644 --- a/qa/qa/page/admin/menu.rb +++ b/qa/qa/page/admin/menu.rb @@ -3,15 +3,10 @@ module Page module Admin class Menu < Page::Base def go_to_license - within_middle_menu { click_link 'License' } - end - - private - - def within_middle_menu - page.within('.nav-control') do - yield - end + link = find_link 'License' + # Click space to scroll this link into the view + link.send_keys(:space) + link.click end end end diff --git a/spec/ee/spec/lib/ee/gitlab/deltas_spec.rb b/spec/ee/spec/lib/ee/gitlab/deltas_spec.rb index f024e4f76359a6bee7ad33c1049ba56949f676ce..62abbcda421d927564d58ee9d4a95042d5b7589f 100644 --- a/spec/ee/spec/lib/ee/gitlab/deltas_spec.rb +++ b/spec/ee/spec/lib/ee/gitlab/deltas_spec.rb @@ -12,5 +12,19 @@ expect(described_class.delta_size_check(change, project.repository)).to be > 0 end + + it 'handles annotated tags on an object' do + object_id = 'faaf198af3a36dbf41961466703cc1d47c61d051' + options = { message: 'test tag message\n', + tagger: { name: 'John Smith', email: 'john@gmail.com' } } + result = project.repository.rugged.tags.create('annotated-tag', object_id, options) + + change = { + oldrev: result.annotation.oid, + newrev: TestEnv::BRANCH_SHA['master'] + } + + expect(described_class.delta_size_check(change, project.repository)).to eq(0) + end end end diff --git a/spec/fixtures/config/kubeconfig.yml b/spec/fixtures/config/kubeconfig.yml index c4e8e573c32af6e1a6e47a0d9615c37e577778c1..5152dae0104acbc9c6783e073ea76473f08f415a 100644 --- a/spec/fixtures/config/kubeconfig.yml +++ b/spec/fixtures/config/kubeconfig.yml @@ -4,7 +4,7 @@ clusters: - name: gitlab-deploy cluster: server: https://kube.domain.com - certificate-authority-data: "UEVN\n" + certificate-authority-data: "UEVN" contexts: - name: gitlab-deploy context: diff --git a/spec/javascripts/build_spec.js b/spec/javascripts/build_spec.js index 351496110959bee21be568f306a3291a2d762bc0..d5b0f23e7b7161491a1fd7fa907615546f664365 100644 --- a/spec/javascripts/build_spec.js +++ b/spec/javascripts/build_spec.js @@ -289,4 +289,18 @@ describe('Build', () => { }); }); }); + + describe('getBuildTrace', () => { + it('should request build trace with state parameter', (done) => { + spyOn(jQuery, 'ajax').and.callThrough(); + new Build(); + + setTimeout(() => { + expect(jQuery.ajax).toHaveBeenCalledWith( + { url: `${BUILD_URL}/trace.json`, data: { state: '' } }, + ); + done(); + }, 0); + }); + }); }); diff --git a/spec/lib/gitlab/geo/database_tasks_spec.rb b/spec/lib/gitlab/geo/database_tasks_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..8766ae2a25dd0b0babd18d48be915e430678ed0b --- /dev/null +++ b/spec/lib/gitlab/geo/database_tasks_spec.rb @@ -0,0 +1,118 @@ +require 'spec_helper' + +describe Gitlab::Geo::DatabaseTasks do + let(:schema_file) { Rails.root.join('tmp', 'tests', 'geo_schema.rb').to_s } + subject { described_class } + + before do + stub_env('SCHEMA', schema_file) # schema will be dumped to this file + end + + after do + FileUtils.rm_rf(schema_file) + end + + [:drop_current, :create_current, :migrate, :load_seed, :load_schema_current].each do |method_name| + it "defines the missing method #{method_name}" do + is_expected.to respond_to(method_name) + end + + it "forwards method #{method_name} to ActiveRecord::Tasks::DatabaseTasks" do + expect(ActiveRecord::Tasks::DatabaseTasks).to receive(method_name) + + subject.public_send(method_name) + end + end + + describe '.rollback' do + context 'ENV["STEP"] not set' do + it 'calls ActiveRecord::Migrator.rollback with step 1' do + expect(ActiveRecord::Migrator).to receive(:rollback).with(anything, 1) + + subject.rollback + end + end + end + + describe '.version' do + it 'returns a Number' do + expect(subject.version).to be_an(Integer) + end + end + + describe '.dump_schema_after_migration?' do + it 'returns a true value' do + expect(subject.dump_schema_after_migration?).to be_truthy + end + end + + describe '.pending_migrations' do + it 'returns an array' do + expect(subject.pending_migrations).to be_an(Array) + end + end + + describe described_class::Schema do + describe '.dump' do + it 'calls ActiveRecord::SchemaDumper.dump' do + expect(ActiveRecord::SchemaDumper).to receive(:dump) + + subject.dump + end + end + end + + describe described_class::Migrate do + describe '.up' do + it 'requires ENV["VERSION"] to be set' do + expect { subject.up }.to raise_error(String) + end + + it 'calls ActiveRecord::Migrator.run' do + stub_env('VERSION', '19700101120000') + expect(ActiveRecord::Migrator).to receive(:run).with(:up, any_args) + + subject.up + end + end + + describe '.down' do + it 'requires ENV["VERSION"] to be set' do + expect { subject.down }.to raise_error(String) + end + + it 'calls ActiveRecord::Migrator.run' do + stub_env('VERSION', '19700101120000') + expect(ActiveRecord::Migrator).to receive(:run).with(:down, any_args) + + subject.down + end + end + + describe '.status' do + it 'outputs "database: gitlabhq_geo_test"' do + expect(ActiveRecord::SchemaMigration).to receive(:normalized_versions).and_return([]) + + expect { subject.status }.to output(/database: gitlabhq_geo_test/).to_stdout + end + end + end + + describe described_class::Test do + describe '.load' do + it 'calls ActiveRecord::Tasks::DatabaseTasks.load_schema_for' do + expect(ActiveRecord::Tasks::DatabaseTasks).to receive(:load_schema_for) + + subject.load + end + end + + describe '.purge' do + it 'calls ActiveRecord::Tasks::DatabaseTasks.load_schema_for' do + expect(ActiveRecord::Tasks::DatabaseTasks).to receive(:purge) + + subject.purge + end + end + end +end diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb index 6c014fac2b023a340274fac9b9524b8af37bf30c..236265a9090d9680f299552e484ca5514433a7a2 100644 --- a/spec/lib/gitlab/git/repository_spec.rb +++ b/spec/lib/gitlab/git/repository_spec.rb @@ -389,6 +389,40 @@ def submodule_url(path) end end + describe '#has_local_branches?' do + shared_examples 'check for local branches' do + it { expect(repository.has_local_branches?).to eq(true) } + + context 'mutable' do + let(:repository) { Gitlab::Git::Repository.new('default', TEST_MUTABLE_REPO_PATH, '') } + + after do + ensure_seeds + end + + it 'returns false when there are no branches' do + # Sanity check + expect(repository.has_local_branches?).to eq(true) + + FileUtils.rm_rf(File.join(repository.path, 'packed-refs')) + heads_dir = File.join(repository.path, 'refs/heads') + FileUtils.rm_rf(heads_dir) + FileUtils.mkdir_p(heads_dir) + + expect(repository.has_local_branches?).to eq(false) + end + end + end + + context 'with gitaly' do + it_behaves_like 'check for local branches' + end + + context 'without gitaly', skip_gitaly_mock: true do + it_behaves_like 'check for local branches' + end + end + describe "#delete_branch" do shared_examples "deleting a branch" do let(:repository) { Gitlab::Git::Repository.new('default', TEST_MUTABLE_REPO_PATH, '') } diff --git a/spec/models/project_services/kubernetes_service_spec.rb b/spec/models/project_services/kubernetes_service_spec.rb index df617e3d8add55c2fab58b1ed4811fab823b2ea4..61dee1a3add407b9eb45bc4470440763c4746d9b 100644 --- a/spec/models/project_services/kubernetes_service_spec.rb +++ b/spec/models/project_services/kubernetes_service_spec.rb @@ -208,7 +208,7 @@ config.dig('users', 0, 'user')['token'] = 'token' config.dig('contexts', 0, 'context')['namespace'] = namespace config.dig('clusters', 0, 'cluster')['certificate-authority-data'] = - Base64.encode64('CA PEM DATA') + Base64.strict_encode64('CA PEM DATA') YAML.dump(config) end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index ef6e04c4f206da51b59f0468f0d45f18863f616b..0066d4a610a7199978790a297bfe57499dd3859c 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -3196,6 +3196,17 @@ def enable_lfs end end + describe '#check_repository_path_availability' do + let(:project) { build(:project) } + + it 'skips gitlab-shell exists?' do + project.skip_disk_validation = true + + expect(project.gitlab_shell).not_to receive(:exists?) + expect(project.check_repository_path_availability).to be_truthy + end + end + describe '#latest_successful_pipeline_for_default_branch' do let(:project) { build(:project) } diff --git a/spec/models/repository_spec.rb b/spec/models/repository_spec.rb index ed9f3aaa1b2348e2dab75264bdfef6aad98ac25f..0ffe4cddf14a9eb364e374c74b6f3a7ed38fe150 100644 --- a/spec/models/repository_spec.rb +++ b/spec/models/repository_spec.rb @@ -1140,21 +1140,31 @@ def expect_to_raise_storage_error end describe '#has_visible_content?' do - subject { repository.has_visible_content? } + before do + # If raw_repository.has_visible_content? gets called more than once then + # caching is broken. We don't want that. + expect(repository.raw_repository).to receive(:has_visible_content?) + .once + .and_return(result) + end - describe 'when there are no branches' do - before do - allow(repository.raw_repository).to receive(:branch_count).and_return(0) - end + context 'when true' do + let(:result) { true } - it { is_expected.to eq(false) } + it 'returns true and caches it' do + expect(repository.has_visible_content?).to eq(true) + # Second call hits the cache + expect(repository.has_visible_content?).to eq(true) + end end - describe 'when there are branches' do - it 'returns true' do - expect(repository.raw_repository).to receive(:branch_count).and_return(3) + context 'when false' do + let(:result) { false } - expect(subject).to eq(true) + it 'returns false and caches it' do + expect(repository.has_visible_content?).to eq(false) + # Second call hits the cache + expect(repository.has_visible_content?).to eq(false) end end end @@ -1271,6 +1281,7 @@ def expect_to_raise_storage_error allow(repository).to receive(:empty?).and_return(true) expect(cache).to receive(:expire).with(:empty?) + expect(cache).to receive(:expire).with(:has_visible_content?) repository.expire_emptiness_caches end @@ -1279,6 +1290,7 @@ def expect_to_raise_storage_error allow(repository).to receive(:empty?).and_return(false) expect(cache).not_to receive(:expire).with(:empty?) + expect(cache).not_to receive(:expire).with(:has_visible_content?) repository.expire_emptiness_caches end @@ -1679,7 +1691,7 @@ def merge(repository, user, merge_request, options = {}) describe '#expire_branches_cache' do it 'expires the cache' do expect(repository).to receive(:expire_method_caches) - .with(%i(branch_names branch_count)) + .with(%i(branch_names branch_count has_visible_content?)) .and_call_original repository.expire_branches_cache @@ -1940,6 +1952,15 @@ def merge(repository, user, merge_request, options = {}) repository.expire_all_method_caches end + + it 'all cache_method definitions are in the lists of method caches' do + methods = repository.methods.map do |method| + match = /^_uncached_(.*)/.match(method) + match[1].to_sym if match + end.compact + + expect(methods).to match_array(Repository::CACHED_METHODS + Repository::MEMOIZED_CACHED_METHODS) + end end describe '#file_on_head' do diff --git a/spec/services/projects/create_service_spec.rb b/spec/services/projects/create_service_spec.rb index 5da634e2fb10e451a34b77c2cdc7cb3c588aa612..7af7e170046232dd5809d894784f7cae60dc4cab 100644 --- a/spec/services/projects/create_service_spec.rb +++ b/spec/services/projects/create_service_spec.rb @@ -208,6 +208,15 @@ end end + context 'when skip_disk_validation is used' do + it 'sets the project attribute' do + opts[:skip_disk_validation] = true + project = create_project(user, opts) + + expect(project.skip_disk_validation).to be_truthy + end + end + def create_project(user, opts) Projects::CreateService.new(user, opts).execute end diff --git a/spec/workers/git_garbage_collect_worker_spec.rb b/spec/workers/git_garbage_collect_worker_spec.rb index f7b67b8efc6a0ce1b33e96ef4a72dfaf29dd5330..ab656d619f47fed80af8d011be2dc2f5267c6a9c 100644 --- a/spec/workers/git_garbage_collect_worker_spec.rb +++ b/spec/workers/git_garbage_collect_worker_spec.rb @@ -32,7 +32,7 @@ expect_any_instance_of(Repository).to receive(:after_create_branch).and_call_original expect_any_instance_of(Repository).to receive(:branch_names).and_call_original expect_any_instance_of(Repository).to receive(:has_visible_content?).and_call_original - expect_any_instance_of(Gitlab::Git::Repository).to receive(:branch_count).and_call_original + expect_any_instance_of(Gitlab::Git::Repository).to receive(:has_visible_content?).and_call_original subject.perform(project.id, :gc, lease_key, lease_uuid) end @@ -47,7 +47,6 @@ expect(subject).not_to receive(:command) expect_any_instance_of(Repository).not_to receive(:after_create_branch).and_call_original expect_any_instance_of(Repository).not_to receive(:branch_names).and_call_original - expect_any_instance_of(Repository).not_to receive(:branch_count).and_call_original expect_any_instance_of(Repository).not_to receive(:has_visible_content?).and_call_original subject.perform(project.id, :gc, lease_key, lease_uuid) @@ -78,7 +77,7 @@ expect_any_instance_of(Repository).to receive(:after_create_branch).and_call_original expect_any_instance_of(Repository).to receive(:branch_names).and_call_original expect_any_instance_of(Repository).to receive(:has_visible_content?).and_call_original - expect_any_instance_of(Gitlab::Git::Repository).to receive(:branch_count).and_call_original + expect_any_instance_of(Gitlab::Git::Repository).to receive(:has_visible_content?).and_call_original subject.perform(project.id) end @@ -93,7 +92,6 @@ expect(subject).not_to receive(:command) expect_any_instance_of(Repository).not_to receive(:after_create_branch).and_call_original expect_any_instance_of(Repository).not_to receive(:branch_names).and_call_original - expect_any_instance_of(Repository).not_to receive(:branch_count).and_call_original expect_any_instance_of(Repository).not_to receive(:has_visible_content?).and_call_original subject.perform(project.id)