Add repository languages for projects

Our friends at GitHub show the programming languages for a long time,
and inspired by that this commit means to create about the same
functionality.

Language detection is done through Linguist, as before, where the
difference is that we cache the result in the database. Also, Gitaly can
incrementaly scan a repository. This is done through a shell out, which
creates overhead of about 3s each run. For now this won't be improved.

Scans are triggered by pushed to the default branch, usually `master`.
However, one exception to this rule the charts page. If we're requesting
this expensive data anyway, we just cache it in the database.

Edge cases where there is no repository, or its empty are caught in the
Repository model. This makes use of Redis caching, which is probably
already loaded.

The added model is called RepositoryLanguage, which will make it harder
if/when GitLab supports multiple repositories per project. However, for
now I think this shouldn't be a concern. Also, Language could be
confused with the i18n languages and felt like the current name was
suiteable too.

Design of the Project#Show page is done with help from @dimitrieh. This
change is not visible to the end user unless detections are done.

spec/factories/repository_languages.rb

+FactoryBot.define do
+  factory :repository_language do
+    project
+    programming_language
+    share 98.5
+  end
+end

spec/lib/gitlab/import_export/all_models.yml

@@ -297,6 +297,7 @@ project:
 - settings
 - ci_cd_settings
 - import_export_upload
+- repository_languages
 award_emoji:
 - awardable
 - user

spec/lib/gitlab/language_detection_spec.rb

+require 'spec_helper'
+
+describe Gitlab::LanguageDetection do
+  set(:project) { create(:project, :repository) }
+  set(:ruby) { create(:programming_language, name: 'Ruby') }
+  set(:haskell) { create(:programming_language, name: 'Haskell') }
+  let(:repository) { project.repository }
+  let(:detection) do
+    [{ value: 66.63, label: "Ruby", color: "#701516", highlight: "#701516" },
+     { value: 12.96, label: "JavaScript", color: "#f1e05a", highlight: "#f1e05a" },
+     { value: 7.9, label: "Elixir", color: "#e34c26", highlight: "#e34c26" },
+     { value: 2.51, label: "CoffeeScript", color: "#244776", highlight: "#244776" },
+     { value: 1.51, label: "Go", color: "#2a4776", highlight: "#244776" },
+     { value: 1.1, label: "MepmepLang", color: "#2a4776", highlight: "#244776" }]
+  end
+  let(:repository_languages) do
+    [RepositoryLanguage.new(share: 10, programming_language: ruby)]
+  end
+
+  subject { described_class.new(repository, repository_languages) }
+
+  before do
+    allow(repository).to receive(:languages).and_return(detection)
+  end
+
+  describe '#languages' do
+    it 'returns the language names' do
+      expect(subject.languages).to eq(%w[Ruby JavaScript Elixir CoffeeScript Go])
+    end
+  end
+
+  describe '#insertions' do
+    let(:programming_languages) { [ruby, haskell] }
+    let(:detection) do
+      [{ value: 10, label: haskell.name, color: haskell.color }]
+    end
+
+    it 'only includes new languages' do
+      insertions = subject.insertions(programming_languages)
+
+      expect(insertions).not_to be_empty
+      expect(insertions.first[:project_id]).to be(project.id)
+      expect(insertions.first[:programming_language_id]).to be(haskell.id)
+      expect(insertions.first[:share]).to be(10)
+    end
+  end
+
+  describe '#updates' do
+    it 'updates the share of languages' do
+      first_update = subject.updates.first
+
+      expect(first_update).not_to be_nil
+      expect(first_update[:programming_language_id]).to eq(ruby.id)
+      expect(first_update[:share]).to eq(66.63)
+    end
+
+    it 'does not include languages to be removed' do
+      ids = subject.updates.map { |h| h[:programming_language_id] }
+
+      expect(ids).not_to include(haskell.id)
+    end
+
+    context 'when silent writes occur' do
+      let(:repository_languages) do
+        [RepositoryLanguage.new(share: 66.63, programming_language: ruby)]
+      end
+
+      it "doesn't include them in the result" do
+        expect(subject.updates).to be_empty
+      end
+    end
+  end
+
+  describe '#deletions' do
+    let(:repository_languages) do
+      [RepositoryLanguage.new(share: 10, programming_language: ruby),
+       RepositoryLanguage.new(share: 5, programming_language: haskell)]
+    end
+
+    it 'lists undetected languages' do
+      expect(subject.deletions).not_to be_empty
+      expect(subject.deletions).to include(haskell.id)
+    end
+  end
+end

spec/models/programming_language_spec.rb

+require 'spec_helper'
+
+describe ProgrammingLanguage do
+  it { is_expected.to respond_to(:name) }
+  it { is_expected.to respond_to(:color) }
+
+  it { is_expected.to validate_presence_of(:name) }
+  it { is_expected.to allow_value("#000000").for(:color) }
+  it { is_expected.not_to allow_value("000000").for(:color) }
+  it { is_expected.not_to allow_value("#0z0000").for(:color) }
+end

spec/models/project_spec.rb

@@ -69,6 +69,7 @@ describe Project do
     it { is_expected.to have_many(:pages_domains) }
     it { is_expected.to have_many(:labels).class_name('ProjectLabel') }
     it { is_expected.to have_many(:users_star_projects) }
+    it { is_expected.to have_many(:repository_languages) }
     it { is_expected.to have_many(:environments) }
     it { is_expected.to have_many(:deployments) }
     it { is_expected.to have_many(:todos) }

spec/models/repository_language_spec.rb

+require 'spec_helper'
+
+describe RepositoryLanguage do
+  let(:repository_language) { build(:repository_language) }
+
+  describe 'associations' do
+    it { is_expected.to belong_to(:project) }
+    it { is_expected.to belong_to(:programming_language) }
+  end
+
+  describe 'validations' do
+    it { is_expected.to allow_value(0).for(:share) }
+    it { is_expected.to allow_value(100.0).for(:share) }
+    it { is_expected.not_to allow_value(100.1).for(:share) }
+  end
+end

spec/services/git_push_service_spec.rb

@@ -3,8 +3,8 @@ require 'spec_helper'
 describe GitPushService, services: true do
   include RepoHelpers
-  let(:user)     { create(:user) }
-  let(:project)  { create(:project, :repository) }
+  set(:user)     { create(:user) }
+  set(:project)  { create(:project, :repository) }
   let(:blankrev) { Gitlab::Git::BLANK_SHA }
   let(:oldrev)   { sample_commit.parent_id }
   let(:newrev)   { sample_commit.id }

spec/services/projects/detect_repository_languages_service_spec.rb

+require 'spec_helper'
+
+describe Projects::DetectRepositoryLanguagesService, :clean_gitlab_redis_shared_state do
+  set(:project) { create(:project, :repository) }
+
+  subject { described_class.new(project, project.owner) }
+
+  before do
+    allow(Feature).to receive(:disabled?).and_return(false)
+  end
+
+  describe '#execute' do
+    context 'without previous detection' do
+      it 'inserts new programming languages in the database' do
+        subject.execute
+
+        expect(ProgrammingLanguage.exists?(name: 'Ruby')).to be(true)
+        expect(ProgrammingLanguage.count).to be(4)
+      end
+
+      it 'inserts the repository langauges' do
+        names = subject.execute.map(&:name)
+
+        expect(names).to eq(%w[Ruby JavaScript HTML CoffeeScript])
+      end
+    end
+
+    context 'with a previous detection' do
+      before do
+        subject.execute
+
+        allow(project.repository).to receive(:languages).and_return(
+          [{ value: 99.63, label: "Ruby", color: "#701516", highlight: "#701516" },
+           { value: 0.3, label: "D", color: "#701516", highlight: "#701516" }]
+        )
+      end
+
+      it 'updates the repository languages' do
+        repository_languages = subject.execute.map(&:name)
+
+        expect(repository_languages).to eq(%w[Ruby D])
+      end
+    end
+
+    context 'when no repository exists' do
+      set(:project) { create(:project) }
+
+      it 'has no languages' do
+        expect(subject.execute).to be_empty
+        expect(project.repository_languages).to be_empty
+      end
+    end
+  end
+end

spec/workers/detect_repository_languages_worker_spec.rb

+require 'spec_helper'
+
+describe DetectRepositoryLanguagesWorker do
+  set(:project) { create(:project) }
+  let(:user) { project.owner }
+
+  subject { described_class.new }
+
+  describe '#perform' do
+    it 'calls de DetectRepositoryLanguages service' do
+      service = double
+      allow(::Projects::DetectRepositoryLanguagesService).to receive(:new).and_return(service)
+      expect(service).to receive(:execute)
+
+      subject.perform(project.id, user.id)
+    end
+
+    context 'when invalid ids are used' do
+      it 'does not raise when the project could not be found' do
+        expect do
+          subject.perform(-1, user.id)
+        end.not_to raise_error
+      end
+
+      it 'does not raise when the user could not be found' do
+        expect do
+          subject.perform(project.id, -1)
+        end.not_to raise_error
+      end
+    end
+  end
+end