Rebase repo check MR

app/controllers/admin/projects_controller.rb

 class Admin::ProjectsController < Admin::ApplicationController
-  before_action :project, only: [:show, :transfer]
+  before_action :project, only: [:show, :transfer, :repo_check]
   before_action :group, only: [:show, :transfer]
   def index
@@ -8,6 +8,7 @@ class Admin::ProjectsController < Admin::ApplicationController
     @projects = @projects.where("visibility_level IN (?)", params[:visibility_levels]) if params[:visibility_levels].present?
     @projects = @projects.with_push if params[:with_push].present?
     @projects = @projects.abandoned if params[:abandoned].present?
+    @projects = @projects.where(last_repo_check_failed: true) if params[:last_repo_check_failed].present?
     @projects = @projects.non_archived unless params[:with_archived].present?
     @projects = @projects.search(params[:name]) if params[:name].present?
     @projects = @projects.sort(@sort = params[:sort])
@@ -30,6 +31,27 @@ class Admin::ProjectsController < Admin::ApplicationController
     redirect_to admin_namespace_project_path(@project.namespace, @project)
   end
+  def repo_check
+    SingleRepoCheckWorker.perform_async(@project.id)
+
+    redirect_to(
+      admin_namespace_project_path(@project.namespace, @project),
+      notice: 'Repo check was triggered'
+    )
+  end
+
+  def clear_repo_check_states
+    Project.update_all(
+      last_repo_check_failed: false,
+      last_repo_check_at: nil
+    )
+
+    redirect_to(
+      admin_namespaces_projects_path,
+      notice: 'All project repo check states were cleared'
+    )
+  end
+
   protected
   def project

app/mailers/repo_check_mailer.rb

+class RepoCheckMailer < BaseMailer
+  include ActionView::Helpers::TextHelper
+
+  def notify(failed_count)
+    if failed_count == 1
+      @message = "One project failed its last repository check"
+    else
+      @message = "#{failed_count} projects failed their last repository check"
+    end
+
+    mail(
+      to: User.admins.pluck(:email),
+      subject: @message
+    )
+  end
+end

app/views/admin/logs/show.html.haml

 - page_title "Logs"
 - loggers = [Gitlab::GitLogger, Gitlab::AppLogger,
-             Gitlab::ProductionLogger, Gitlab::SidekiqLogger]
+             Gitlab::ProductionLogger, Gitlab::SidekiqLogger,
+             Gitlab::RepoCheckLogger]
 %ul.nav-links.log-tabs
   - loggers.each do |klass|
     %li{ class: (klass == Gitlab::GitLogger ? 'active' : '') }

app/views/admin/projects/index.html.haml

@@ -3,7 +3,7 @@
 .row.prepend-top-default
   %aside.col-md-3
-    .admin-filter
+    .panel.admin-filter
       = form_tag admin_namespaces_projects_path, method: :get, class: '' do
         .form-group
           = label_tag :name, 'Name:'
@@ -38,11 +38,22 @@
                   %span.descr
                     = visibility_level_icon(level)
                     = label
-        %hr
+        %fieldset
+          %strong Problems
+          .checkbox
+            = label_tag :last_repo_check_failed do
+              = check_box_tag :last_repo_check_failed, 1, params[:last_repo_check_failed]
+              %span Last repo check failed
+
         = hidden_field_tag :sort, params[:sort]
         = button_tag "Search", class: "btn submit btn-primary"
         = link_to "Reset", admin_namespaces_projects_path, class: "btn btn-cancel"
+    .panel.panel-default.repo-check-states
+      .panel-heading
+        Repo check states
+      .panel-body
+        = link_to 'Clear all', clear_repo_check_states_admin_namespace_projects_path(0), data: { confirm: 'This will clear repo check states for ALL projects in the database. This cannot be undone. Are you sure?' }, method: :put, class: "btn btn-sm btn-remove"
   %section.col-md-9
     .panel.panel-default
       .panel-heading

app/views/admin/projects/show.html.haml

@@ -5,6 +5,14 @@
     %i.fa.fa-pencil-square-o
     Edit
 %hr
+- if @project.last_repo_check_failed?
+  .row
+    .col-md-12
+      .panel
+        .panel-heading.alert.alert-danger
+          Last repo check failed. See
+          = link_to 'repocheck.log', admin_logs_path
+          for error messages.
 .row
   .col-md-6
     .panel.panel-default
@@ -95,6 +103,30 @@
             .col-sm-offset-2.col-sm-10
               = f.submit 'Transfer', class: 'btn btn-primary'
+    .panel.panel-default.repo-check
+      .panel-heading
+        Repo check
+      .panel-body
+        = form_for @project, url: repo_check_admin_namespace_project_path(@project.namespace, @project), method: :post do |f|
+          .form-group
+            - if @project.last_repo_check_at.nil?
+              This repository has never been checked.
+            - else
+              This repository was last checked
+              = @project.last_repo_check_at.to_s(:medium) + '.'
+              The check
+              - if @project.last_repo_check_failed?
+                = succeed '.' do
+                  %strong.cred failed
+                See
+                = link_to 'repocheck.log', admin_logs_path
+                for error messages.
+              - else
+                passed.
+
+          .form-group
+            = f.submit 'Trigger repo check', class: 'btn btn-primary'
+
   .col-md-6
     - if @group
       .panel.panel-default

app/views/repo_check_mailer/notify.html.haml

+%p
+  #{@message}.
+
+%p
+  = link_to "See the affected projects in the GitLab admin panel", admin_namespaces_projects_url(last_repo_check_failed: 1)

app/views/repo_check_mailer/notify.text.haml

+#{@message}.
+\
+View details: #{admin_namespaces_projects_url(last_repo_check_failed: 1)}

app/workers/admin_email_worker.rb

+class AdminEmailWorker
+  include Sidekiq::Worker
+
+  sidekiq_options retry: false # this job auto-repeats via sidekiq-cron
+
+  def perform
+    repo_check_failed_count = Project.where(last_repo_check_failed: true).count
+    return if repo_check_failed_count.zero?
+
+    RepoCheckMailer.notify(repo_check_failed_count).deliver_now
+  end
+end

app/workers/repo_check_worker.rb

+class RepoCheckWorker
+  include Sidekiq::Worker
+
+  RUN_TIME = 3600
+
+  sidekiq_options retry: false
+
+  def perform
+    start = Time.now
+
+    # This loop will break after a little more than one hour ('a little
+    # more' because `git fsck` may take a few minutes), or if it runs out of
+    # projects to check. By default sidekiq-cron will start a new
+    # RepoCheckWorker each hour so that as long as there are repositories to
+    # check, only one (or two) will be checked at a time.
+    project_ids.each do |project_id|
+      break if Time.now - start >= RUN_TIME
+
+      next if !try_obtain_lease(project_id)
+
+      SingleRepoCheckWorker.new.perform(project_id)
+    end
+  end
+
+  private
+
+  # In an ideal world we would use Project.where(...).find_each.
+  # Unfortunately, calling 'find_each' drops the 'where', so we must build
+  # an array of IDs instead.
+  def project_ids
+    limit = 10_000
+    never_checked_projects = Project.where('last_repo_check_at IS NULL').limit(limit).
+      pluck(:id)
+    old_check_projects = Project.where('last_repo_check_at < ?', 1.week.ago).
+      reorder('last_repo_check_at ASC').limit(limit).pluck(:id)
+    never_checked_projects + old_check_projects
+  end
+
+  def try_obtain_lease(id)
+    lease = Gitlab::ExclusiveLease.new(
+      "project_repo_check:#{id}",
+      timeout: RUN_TIME
+    )
+    lease.try_obtain
+  end
+end

app/workers/single_repo_check_worker.rb

+class SingleRepoCheckWorker
+  include Sidekiq::Worker
+
+  sidekiq_options retry: false
+
+  def perform(project_id)
+    project = Project.find(project_id)
+    update(project, success: check(project))
+  end
+
+  private
+
+  def check(project)
+    [project.repository.path_to_repo, project.wiki.wiki.path].all? do |path|
+      git_fsck(path)
+    end
+  end
+
+  def git_fsck(path)
+    cmd = %W(nice git --git-dir=#{path} fsck)
+    output, status = Gitlab::Popen.popen(cmd)
+    return true if status.zero?
+
+    Gitlab::RepoCheckLogger.error("command failed: #{cmd.join(' ')}\n#{output}")
+    false
+  end
+
+  def update(project, success:)
+    project.update_columns(
+      last_repo_check_failed: !success,
+      last_repo_check_at: Time.now,
+    )
+  end
+end

config/gitlab.yml.example

@@ -155,6 +155,13 @@ production: &base
     # Flag stuck CI builds as failed
     stuck_ci_builds_worker:
       cron: "0 0 * * *"
+    # Periodically run 'git fsck' on all repositories. If started more than
+    # once per hour you will have concurrent 'git fsck' jobs.
+    repo_check_worker:
+      cron: "20 * * * *"
+    # Send admin emails once a day
+    admin_email_worker:
+      cron: "0 0 * * *"
   #

config/initializers/1_settings.rb

@@ -239,7 +239,12 @@ Settings['cron_jobs'] ||= Settingslogic.new({})
 Settings.cron_jobs['stuck_ci_builds_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['stuck_ci_builds_worker']['cron'] ||= '0 0 * * *'
 Settings.cron_jobs['stuck_ci_builds_worker']['job_class'] = 'StuckCiBuildsWorker'
-
+Settings.cron_jobs['repo_check_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['repo_check_worker']['cron'] ||= '20 * * * *'
+Settings.cron_jobs['repo_check_worker']['job_class'] = 'RepoCheckWorker'
+Settings.cron_jobs['admin_email_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['admin_email_worker']['cron'] ||= '0 0 * * *'
+Settings.cron_jobs['admin_email_worker']['job_class'] = 'AdminEmailWorker'
 #
 # GitLab Shell

config/routes.rb

@@ -264,6 +264,11 @@ Rails.application.routes.draw do
         member do
           put :transfer
+          post :repo_check
+        end
+
+        collection do
+          put :clear_repo_check_states
         end
         resources :runner_projects

db/migrate/20160315135439_project_add_repo_check.rb

+class ProjectAddRepoCheck < ActiveRecord::Migration
+  def change
+    add_column :projects, :last_repo_check_failed, :boolean, default: false
+    add_column :projects, :last_repo_check_at, :datetime
+  end
+end

db/schema.rb

@@ -732,6 +732,8 @@ ActiveRecord::Schema.define(version: 20160331133914) do
     t.boolean  "public_builds",          default: true,     null: false
     t.string   "main_language"
     t.integer  "pushes_since_gc",        default: 0
+    t.boolean  "last_repo_check_failed", default: false
+    t.datetime "last_repo_check_at"
   end
   add_index "projects", ["builds_enabled", "shared_runners_enabled"], name: "index_projects_on_builds_enabled_and_shared_runners_enabled", using: :btree

doc/administration/repo_checks.md

+# Repo checks
+
+_**Note:** This feature was [introduced][ce-3232] in GitLab 8.7_
+
+---
+
+Git has a built-in mechanism [git fsck][git-fsck] to verify the
+integrity of all data commited to a repository. GitLab administrators can
+trigger such a check for a project via the admin panel. The checks run
+asynchronously so it may take a few minutes before the check result is
+visible on the project admin page. If the checks failed you can see their
+output on the admin log page under 'repocheck.log'.
+
+## Periodical checks
+
+GitLab periodically runs a repo check on all project repositories and
+wiki repositories in order to detect data corruption problems. A
+project will be checked no more than once per week. If any projects
+fail their repo checks all GitLab administrators will receive an email
+notification of the situation. This notification is sent out no more
+than once a day.
+
+
+## What to do if a check failed
+
+If the repo check fails for some repository you shouldlook up the error
+in repocheck.log (in the admin panel or on disk; see
+`/var/log/gitlab/gitlab-rails` for Omnibus installations or
+`/home/git/gitlab/log` for installations from source). Once you have
+resolved the issue use the admin panel to trigger a new repo check on
+the project. This will clear the 'check failed' state.
+
+If for some reason the periodical repo check caused a lot of false
+alarms you can choose to clear ALL repo check states from the admin
+project index page.
+
+---
+[ce-3232]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/3232 "Auto git fsck"
+[git-fsck]: https://www.kernel.org/pub/software/scm/git/docs/git-fsck.html "git fsck documentation"
\ No newline at end of file

lib/gitlab/repo_check_logger.rb

+module Gitlab
+  class RepoCheckLogger < Gitlab::Logger
+    def self.file_name_noext
+      'repocheck'
+    end
+  end
+end

spec/features/admin/admin_projects_spec.rb

 require 'spec_helper'
-describe "Admin::Projects", feature: true  do
+require 'rails_helper'
+describe "Admin Projects", feature: true  do
   before do
     @project = create(:project)
     login_as :admin
@@ -31,4 +32,38 @@ describe "Admin::Projects", feature: true  do
       expect(page).to have_content(@project.name)
     end
   end
+
+  feature 'repo checks' do
+    scenario 'trigger repo check' do
+      visit_admin_project_page
+
+      page.within('.repo-check') do
+        click_button 'Trigger repo check'
+      end
+
+      expect(page).to have_content('Repo check was triggered')
+    end
+
+    scenario 'see failed repo check' do
+      @project.update_column(:last_repo_check_failed, true)
+      visit_admin_project_page
+
+      expect(page).to have_content('Last repo check failed')
+    end
+
+    scenario 'clear repo checks', js: true do
+      @project.update_column(:last_repo_check_failed, true)
+      visit admin_namespaces_projects_path
+
+      page.within('.repo-check-states') do
+        click_link 'Clear all' # pop-up should be auto confirmed
+      end
+
+      expect(@project.reload.last_repo_check_failed).to eq(false)
+    end
+  end
+
+  def visit_admin_project_page
+    visit admin_namespace_project_path(@project.namespace, @project)
+  end
 end

spec/mailers/repo_check_mailer_spec.rb

+require 'rails_helper'
+
+describe RepoCheckMailer do
+  include EmailSpec::Matchers
+
+  describe '.notify' do
+    it 'emails all admins' do
+      admins = 3.times.map { create(:admin) }
+
+      mail = described_class.notify(1)
+
+      expect(mail).to deliver_to admins.map(&:email)
+    end
+
+    it 'mentions the number of failed checks' do
+      mail = described_class.notify(3)
+
+      expect(mail).to have_subject '3 projects failed their last repository check'
+    end
+  end
+end

spec/workers/repo_check_worker_spec.rb

+require 'spec_helper'
+
+describe RepoCheckWorker do
+  subject { RepoCheckWorker.new }
+
+  it 'prefers projects that have never been checked' do
+    projects = 3.times.map { create(:project) }
+    projects[0].update_column(:last_repo_check_at, 1.month.ago)
+    projects[2].update_column(:last_repo_check_at, 3.weeks.ago)
+
+    expect(subject.perform).to eq(projects.values_at(1, 0, 2).map(&:id))
+  end
+
+  it 'sorts projects by last_repo_check_at' do
+    projects = 3.times.map { create(:project) }
+    projects[0].update_column(:last_repo_check_at, 2.weeks.ago)
+    projects[1].update_column(:last_repo_check_at, 1.month.ago)
+    projects[2].update_column(:last_repo_check_at, 3.weeks.ago)
+
+    expect(subject.perform).to eq(projects.values_at(1, 2, 0).map(&:id))
+  end
+
+  it 'excludes projects that were checked recently' do
+    projects = 3.times.map { create(:project) }
+    projects[0].update_column(:last_repo_check_at, 2.days.ago)
+    projects[1].update_column(:last_repo_check_at, 1.month.ago)
+    projects[2].update_column(:last_repo_check_at, 3.days.ago)
+
+    expect(subject.perform).to eq([projects[1].id])
+  end
+end