Use BFG object maps to clean projects

app/assets/javascripts/behaviors/requires_input.js

@@ -50,10 +50,11 @@ function hideOrShowHelpBlock(form) {
 }
 $(() => {
-  const $form = $('form.js-requires-input');
-  if ($form) {
+  $('form.js-requires-input').each((i, el) => {
+    const $form = $(el);
+
     $form.requiresInput();
     hideOrShowHelpBlock($form);
     $('.select2.js-select-namespace').change(() => hideOrShowHelpBlock($form));
-  }
+  });
 });

app/assets/javascripts/lib/utils/file_upload.js

+export default (buttonSelector, fileSelector) => {
+  const btn = document.querySelector(buttonSelector);
+  const fileInput = document.querySelector(fileSelector);
+  const form = btn.closest('form');
+
+  btn.addEventListener('click', () => {
+    fileInput.click();
+  });
+
+  fileInput.addEventListener('change', () => {
+    form.querySelector('.js-filename').textContent = fileInput.value.replace(/^.*[\\\/]/, ''); // eslint-disable-line no-useless-escape
+  });
+};

app/assets/javascripts/pages/projects/edit/index.js

@@ -3,8 +3,8 @@ import initSettingsPanels from '~/settings_panels';
 import setupProjectEdit from '~/project_edit';
 import initConfirmDangerModal from '~/confirm_danger_modal';
 import mountBadgeSettings from '~/pages/shared/mount_badge_settings';
+import fileUpload from '~/lib/utils/file_upload';
 import initProjectLoadingSpinner from '../shared/save_project_loader';
-import projectAvatar from '../shared/project_avatar';
 import initProjectPermissionsSettings from '../shared/permissions';
 document.addEventListener('DOMContentLoaded', () => {
@@ -12,7 +12,7 @@ document.addEventListener('DOMContentLoaded', () => {
   setupProjectEdit();
   // Initialize expandable settings panels
   initSettingsPanels();
-  projectAvatar();
+  fileUpload('.js-choose-project-avatar-button', '.js-project-avatar-input');
   initProjectPermissionsSettings();
   initConfirmDangerModal();
   mountBadgeSettings(PROJECT_BADGE);

app/assets/javascripts/pages/projects/settings/repository/form.js

@@ -7,6 +7,7 @@ import initDeployKeys from '~/deploy_keys';
 import ProtectedBranchCreate from '~/protected_branches/protected_branch_create';
 import ProtectedBranchEditList from '~/protected_branches/protected_branch_edit_list';
 import DueDateSelectors from '~/due_date_select';
+import fileUpload from '~/lib/utils/file_upload';
 export default () => {
   new ProtectedTagCreate();
@@ -16,4 +17,5 @@ export default () => {
   new ProtectedBranchCreate();
   new ProtectedBranchEditList();
   new DueDateSelectors();
+  fileUpload('.js-choose-file', '.js-object-map-input');
 };

app/assets/javascripts/pages/projects/shared/project_avatar.js

-import $ from 'jquery';
-
-export default function projectAvatar() {
-  $('.js-choose-project-avatar-button').bind('click', function onClickAvatar() {
-    const form = $(this).closest('form');
-    return form.find('.js-project-avatar-input').click();
-  });
-
-  $('.js-project-avatar-input').bind('change', function onClickAvatarInput() {
-    const form = $(this).closest('form');
-    const filename = $(this)
-      .val()
-      .replace(/^.*[\\\/]/, ''); // eslint-disable-line no-useless-escape
-    return form.find('.js-avatar-filename').text(filename);
-  });
-}

app/controllers/projects/settings/repository_controller.rb

@@ -5,6 +5,7 @@ module Projects
     class RepositoryController < Projects::ApplicationController
       before_action :authorize_admin_project!
       before_action :remote_mirror, only: [:show]
+      before_action :check_cleanup_feature_flag!, only: :cleanup
       def show
         render_show
@@ -20,8 +21,26 @@ module Projects
         render_show
       end
+      def cleanup
+        cleanup_params = params.require(:project).permit(:bfg_object_map)
+        result = Projects::UpdateService.new(project, current_user, cleanup_params).execute
+
+        if result[:status] == :success
+          RepositoryCleanupWorker.perform_async(project.id, current_user.id)
+          flash[:notice] = _('Repository cleanup has started. You will receive an email once the cleanup operation is complete.')
+        else
+          flash[:alert] = _('Failed to upload object map file')
+        end
+
+        redirect_to project_settings_repository_path(project)
+      end
+
       private
+      def check_cleanup_feature_flag!
+        render_404 unless ::Feature.enabled?(:project_cleanup, project)
+      end
+
       def render_show
         @deploy_keys = DeployKeysPresenter.new(@project, current_user: current_user)
         @deploy_tokens = @project.deploy_tokens.active

app/helpers/projects_helper.rb

@@ -257,6 +257,10 @@ module ProjectsHelper
     "xcode://clone?repo=#{CGI.escape(default_url_to_repo(project))}"
   end
+  def link_to_bfg
+    link_to 'BFG', 'https://rtyley.github.io/bfg-repo-cleaner/', target: '_blank', rel: 'noopener noreferrer'
+  end
+
   def legacy_render_context(params)
     params[:legacy_render] ? { markdown_engine: :redcarpet } : {}
   end

app/mailers/emails/projects.rb

@@ -24,6 +24,21 @@ module Emails
            subject: subject("Project export error"))
     end
+    def repository_cleanup_success_email(project, user)
+      @project = project
+      @user = user
+
+      mail(to: user.notification_email, subject: subject("Project cleanup has completed"))
+    end
+
+    def repository_cleanup_failure_email(project, user, error)
+      @project = project
+      @user = user
+      @error = error
+
+      mail(to: user.notification_email, subject: subject("Project cleanup failure"))
+    end
+
     def repository_push_email(project_id, opts = {})
       @message =
         Gitlab::Email::Message::RepositoryPush.new(self, project_id, opts)

app/models/project.rb

@@ -339,6 +339,7 @@ class Project < ActiveRecord::Base
     presence: true,
     inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } }
   validates :variables, variable_duplicates: { scope: :environment_scope }
+  validates :bfg_object_map, file_size: { maximum: :max_attachment_size }
   # Scopes
   scope :pending_delete, -> { where(pending_delete: true) }
@@ -412,6 +413,9 @@ class Project < ActiveRecord::Base
                                             only_integer: true,
                                             message: 'needs to be beetween 10 minutes and 1 month' }
+  # Used by Projects::CleanupService to hold a map of rewritten object IDs
+  mount_uploader :bfg_object_map, AttachmentUploader
+
   # Returns a project, if it is not about to be removed.
   #
   # id - The ID of the project to retrieve.
@@ -1973,6 +1977,10 @@ class Project < ActiveRecord::Base
     Ability.allowed?(user, :read_project_snippet, self)
   end
+  def max_attachment_size
+    Gitlab::CurrentSettings.max_attachment_size.megabytes.to_i
+  end
+
   private
   def use_hashed_storage

app/services/notification_service.rb

@@ -466,6 +466,14 @@ class NotificationService
     end
   end
+  def repository_cleanup_success(project, user)
+    mailer.send(:repository_cleanup_success_email, project, user).deliver_later
+  end
+
+  def repository_cleanup_failure(project, user, error)
+    mailer.send(:repository_cleanup_failure_email, project, user, error).deliver_later
+  end
+
   protected
   def new_resource_email(target, method)

app/services/projects/cleanup_service.rb

+# frozen_string_literal: true
+
+module Projects
+  # The CleanupService removes data from the project repository following a
+  # BFG rewrite: https://rtyley.github.io/bfg-repo-cleaner/
+  #
+  # Before executing this service, all refs rewritten by BFG should have been
+  # pushed to the repository
+  class CleanupService < BaseService
+    NoUploadError = StandardError.new("Couldn't find uploaded object map")
+
+    include Gitlab::Utils::StrongMemoize
+
+    # Attempt to clean up the project following the push. Warning: this is
+    # destructive!
+    #
+    # path is the path of an upload of a BFG object map file. It contains a line
+    # per rewritten object, with the old and new SHAs space-separated. It can be
+    # used to update or remove content that references the objects that BFG has
+    # altered
+    #
+    # Currently, only the project repository is modified by this service, but we
+    # may wish to modify other data sources in the future.
+    def execute
+      apply_bfg_object_map!
+
+      # Remove older objects that are no longer referenced
+      GitGarbageCollectWorker.new.perform(project.id, :gc)
+
+      # The cache may now be inaccurate, and holding onto it could prevent
+      # bugs assuming the presence of some object from manifesting for some
+      # time. Better to feel the pain immediately.
+      project.repository.expire_all_method_caches
+
+      project.bfg_object_map.remove!
+    end
+
+    private
+
+    def apply_bfg_object_map!
+      raise NoUploadError unless project.bfg_object_map.exists?
+
+      project.bfg_object_map.open do |io|
+        repository_cleaner.apply_bfg_object_map(io)
+      end
+    end
+
+    def repository_cleaner
+      @repository_cleaner ||= Gitlab::Git::RepositoryCleaner.new(repository.raw)
+    end
+  end
+end

app/views/errors/access_denied.html.haml

@@ -9,7 +9,7 @@
     %p
       = message
   %p
-    = s_('403|Please contact your GitLab administrator to get the permission.')
+    = s_('403|Please contact your GitLab administrator to get permission.')
   .action-container.js-go-back{ style: 'display: none' }
     %a{ href: 'javascript:history.back()', class: 'btn btn-success' }
       = s_('Go Back')

app/views/notify/repository_cleanup_failure_email.text.erb

+Repository cleanup failed on <%= @project.web_url %>
+
+<%= @error %>

app/views/notify/repository_cleanup_success_email.text.erb

+Repository cleanup succeeded on <%= @project.web_url %>
+
+Repository size is now <%= "%.1f" % (@project.repository.size || 0) %> MiB

app/views/projects/cleanup/_show.html.haml

+- return unless Feature.enabled?(:project_cleanup, @project)
+
+- expanded = Rails.env.test?
+
+%section.settings.no-animate#cleanup{ class: ('expanded' if expanded) }
+  .settings-header
+    %h4= _('Repository cleanup')
+    %button.btn.js-settings-toggle
+      = expanded ? _('Collapse') : _('Expand')
+    %p
+      = _("Clean up after running %{bfg} on the repository" % { bfg: link_to_bfg }).html_safe
+      = link_to icon('question-circle'),
+          help_page_path('user/project/repository/reducing_the_repo_size_using_git.md'),
+          target: '_blank', rel: 'noopener noreferrer'
+
+  .settings-content
+    - url = cleanup_namespace_project_settings_repository_path(@project.namespace, @project)
+    = form_for @project, url: url, method: :post, authenticity_token: true, html: { class: 'js-requires-input' } do |f|
+      %fieldset.prepend-top-0.append-bottom-10
+        .append-bottom-10
+          %h5.prepend-top-0
+            = _("Upload object map")
+          %button.btn.btn-default.js-choose-file{ type: "button" }
+            = _("Choose a file")
+          %span.prepend-left-default.js-filename
+            = _("No file selected")
+          = f.file_field :bfg_object_map, accept: 'text/plain', class: "hidden js-object-map-input", required: true
+        .form-text.text-muted
+          = _("The maximum file size allowed is %{max_attachment_size}mb") % { max_attachment_size: Gitlab::CurrentSettings.max_attachment_size }
+      = f.submit _('Start cleanup'), class: 'btn btn-success'
+

app/views/projects/edit.html.haml

@@ -53,7 +53,7 @@
                 = _("Project avatar in repository: %{link}").html_safe % { link: @project.avatar_in_git }
             .prepend-top-5.append-bottom-10
               %button.btn.js-choose-project-avatar-button{ type: 'button' }= _("Choose file...")
-              %span.file_name.prepend-left-default.js-avatar-filename= _("No file chosen")
+              %span.file_name.prepend-left-default.js-filename= _("No file chosen")
               = f.file_field :avatar, class: "js-project-avatar-input hidden"
             .form-text.text-muted= _("The maximum file size allowed is 200KB.")
             - if @project.avatar?

app/views/projects/settings/repository/show.html.haml

@@ -13,3 +13,4 @@
 = render "projects/protected_tags/index"
 = render @deploy_keys
 = render "projects/deploy_tokens/index"
+= render "projects/cleanup/show"

app/workers/all_queues.yml

@@ -133,3 +133,4 @@
 - create_note_diff_file
 - delete_diff_files
 - detect_repository_languages
+- repository_cleanup

app/workers/repository_cleanup_worker.rb

+# frozen_string_literal: true
+
+class RepositoryCleanupWorker
+  include ApplicationWorker
+
+  sidekiq_options retry: 3
+
+  sidekiq_retries_exhausted do |msg, err|
+    next if err.is_a?(ActiveRecord::RecordNotFound)
+
+    args = msg['args'] + [msg['error_message']]
+
+    new.perform_failure(*args)
+  end
+
+  def perform(project_id, user_id)
+    project = Project.find(project_id)
+    user = User.find(user_id)
+
+    Projects::CleanupService.new(project, user).execute
+
+    notification_service.repository_cleanup_success(project, user)
+  end
+
+  def perform_failure(project_id, user_id, error)
+    project = Project.find(project_id)
+    user = User.find(user_id)
+
+    # Ensure the file is removed
+    project.bfg_object_map.remove!
+    notification_service.repository_cleanup_failure(project, user, error)
+  end
+
+  private
+
+  def notification_service
+    @notification_service ||= NotificationService.new
+  end
+end

changelogs/unreleased/19376-post-bfg-cleanup.yml

+---
+title: Use BFG object maps to clean projects
+merge_request: 23189
+author:
+type: added