Skip to content
Snippets Groups Projects
Commit 9812e5dd authored by Nick Thomas's avatar Nick Thomas
Browse files

Merge branch 'zj-repository-languages' into 'master'

Add repository languages for projects

Closes #23931, #34671, #48647, and #47301

See merge request gitlab-org/gitlab-ce!19480
parents 92e079ed 79a5d768
No related branches found
No related tags found
1 merge request!10495Merge Requests - Assignee
Showing
with 269 additions and 2 deletions
Loading
Loading
@@ -754,6 +754,11 @@
}
}
 
.repository-languages-bar {
height: 6px;
margin-bottom: 8px;
}
pre.light-well {
border-color: $well-light-border;
}
Loading
Loading
module RepositoryLanguagesHelper
def repository_languages_bar(languages)
return if languages.none?
content_tag :div, class: 'progress repository-languages-bar' do
safe_join(languages.map { |lang| language_progress(lang) })
end
end
def language_progress(lang)
content_tag :div, nil,
class: "progress-bar has-tooltip",
style: "width: #{lang.share}%; background-color:#{lang.color}",
title: lang.name
end
end
Loading
Loading
@@ -122,6 +122,7 @@ class Namespace < ActiveRecord::Base
def to_param
full_path
end
alias_method :flipper_id, :to_param
 
def human_name
owner_name
Loading
Loading
class ProgrammingLanguage < ActiveRecord::Base
validates :name, presence: true
validates :color, allow_blank: false, color: true
end
Loading
Loading
@@ -192,6 +192,7 @@ class Project < ActiveRecord::Base
has_many :hooks, class_name: 'ProjectHook'
has_many :protected_branches
has_many :protected_tags
has_many :repository_languages, -> { order "share DESC" }
 
has_many :project_authorizations
has_many :authorized_users, through: :project_authorizations, source: :user, class_name: 'User'
Loading
Loading
Loading
Loading
@@ -235,6 +235,12 @@ class Repository
false
end
 
def languages
return [] if empty?
raw_repository.languages(root_ref)
end
# Makes sure a commit is kept around when Git garbage collection runs.
# Git GC will delete commits from the repository that are no longer in any
# branches or tags, but we want to keep some of these commits around, for
Loading
Loading
@@ -432,6 +438,8 @@ class Repository
# Runs code after a repository has been forked/imported.
def after_import
expire_content_cache
DetectRepositoryLanguagesWorker.perform_async(project.id, project.owner.id)
end
 
# Runs code after a new commit has been pushed.
Loading
Loading
class RepositoryLanguage < ActiveRecord::Base
belongs_to :project
belongs_to :programming_language
default_scope { includes(:programming_language) }
validates :project, presence: true
validates :share, inclusion: { in: 0..100, message: "The share of a lanuage is between 0 and 100" }
validates :programming_language, uniqueness: { scope: :project_id }
delegate :name, :color, to: :programming_language
end
Loading
Loading
@@ -85,6 +85,8 @@ class GitPushService < BaseService
 
types = Gitlab::FileDetector.types_in_paths(paths.to_a)
end
DetectRepositoryLanguagesWorker.perform_async(@project.id, current_user.id)
else
types = []
end
Loading
Loading
module Projects
class DetectRepositoryLanguagesService < BaseService
attr_reader :detected_repository_languages, :programming_languages
def execute
repository_languages = project.repository_languages
detection = Gitlab::LanguageDetection.new(repository, repository_languages)
matching_programming_languages = ensure_programming_languages(detection)
RepositoryLanguage.transaction do
project.repository_languages.where(programming_language_id: detection.deletions).delete_all
detection.updates.each do |update|
RepositoryLanguage
.arel_table.update_manager
.where(project_id: project.id)
.where(programming_language_id: update[:programming_language_id])
.set(share: update[:share])
end
Gitlab::Database.bulk_insert(
RepositoryLanguage.table_name,
detection.insertions(matching_programming_languages)
)
end
project.repository_languages.reload
end
private
def ensure_programming_languages(detection)
existing_languages = ProgrammingLanguage.where(name: detection.languages)
return existing_languages if detection.languages.size == existing_languages.size
missing_languages = detection.languages - existing_languages.map(&:name)
created_languages = missing_languages.map do |name|
create_language(name, detection.language_color(name))
end
existing_languages + created_languages
end
def create_language(name, color)
ProgrammingLanguage.transaction do
ProgrammingLanguage.where(name: name).first_or_create(color: color)
end
rescue ActiveRecord::RecordNotUnique
retry
end
end
end
Loading
Loading
@@ -18,10 +18,11 @@
= render "home_panel"
 
- if can?(current_user, :download_code, @project)
%nav.project-stats{ class: container_class }
%nav.project-stats{ class: [container_class, ("limit-container-width" unless fluid_layout)] }
= render 'stat_anchor_list', anchors: @project.statistics_anchors(show_auto_devops_callout: show_auto_devops_callout)
= render 'stat_anchor_list', anchors: @project.statistics_buttons(show_auto_devops_callout: show_auto_devops_callout)
- if Feature.enabled?(:repository_languages, @project.namespace.becomes(Namespace))
= repository_languages_bar(@project.repository_languages)
 
%div{ class: [container_class, ("limit-container-width" unless fluid_layout)] }
- if @project.archived?
Loading
Loading
Loading
Loading
@@ -123,3 +123,4 @@
- repository_update_remote_mirror
- create_note_diff_file
- delete_diff_files
- detect_repository_languages
class DetectRepositoryLanguagesWorker
include ApplicationWorker
include ExceptionBacktrace
include ExclusiveLeaseGuard
sidekiq_options retry: 1
LEASE_TIMEOUT = 300
attr_reader :project
def perform(project_id, user_id)
@project = Project.find_by(id: project_id)
user = User.find_by(id: user_id)
return unless project && user
return if Feature.disabled?(:repository_languages, project.namespace)
try_obtain_lease do
::Projects::DetectRepositoryLanguagesService.new(project, user).execute
end
end
private
def lease_timeout
LEASE_TIMEOUT
end
def lease_key
"gitlab:detect_repository_languages:#{project.id}"
end
end
---
title: Show repository languages for projects
merge_request: 19480
author:
type: added
Loading
Loading
@@ -77,3 +77,4 @@
- [repository_remove_remote, 1]
- [create_note_diff_file, 1]
- [delete_diff_files, 1]
- [detect_repository_languages, 1]
class AddRepositoryLanguages < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def up
create_table(:programming_languages) do |t|
t.string :name, null: false
t.string :color, null: false
t.datetime_with_timezone :created_at, null: false
end
create_table(:repository_languages, id: false) do |t|
t.references :project, null: false, foreign_key: { on_delete: :cascade }
t.references :programming_language, null: false
t.float :share, null: false
end
add_index :programming_languages, :name, unique: true
add_index :repository_languages, [:project_id, :programming_language_id],
unique: true, name: "index_repository_languages_on_project_and_languages_id"
end
def down
drop_table :repository_languages
drop_table :programming_languages
end
end
Loading
Loading
@@ -1502,6 +1502,14 @@ ActiveRecord::Schema.define(version: 20180726172057) do
add_index "personal_access_tokens", ["token"], name: "index_personal_access_tokens_on_token", unique: true, using: :btree
add_index "personal_access_tokens", ["user_id"], name: "index_personal_access_tokens_on_user_id", using: :btree
 
create_table "programming_languages", force: :cascade do |t|
t.string "name", null: false
t.string "color", null: false
t.datetime_with_timezone "created_at", null: false
end
add_index "programming_languages", ["name"], name: "index_programming_languages_on_name", unique: true, using: :btree
create_table "project_authorizations", id: false, force: :cascade do |t|
t.integer "user_id", null: false
t.integer "project_id", null: false
Loading
Loading
@@ -1788,6 +1796,14 @@ ActiveRecord::Schema.define(version: 20180726172057) do
add_index "remote_mirrors", ["last_successful_update_at"], name: "index_remote_mirrors_on_last_successful_update_at", using: :btree
add_index "remote_mirrors", ["project_id"], name: "index_remote_mirrors_on_project_id", using: :btree
 
create_table "repository_languages", id: false, force: :cascade do |t|
t.integer "project_id", null: false
t.integer "programming_language_id", null: false
t.float "share", null: false
end
add_index "repository_languages", ["project_id", "programming_language_id"], name: "index_repository_languages_on_project_and_languages_id", unique: true, using: :btree
create_table "resource_label_events", id: :bigserial, force: :cascade do |t|
t.integer "action", null: false
t.integer "issue_id"
Loading
Loading
@@ -2359,6 +2375,7 @@ ActiveRecord::Schema.define(version: 20180726172057) do
add_foreign_key "push_event_payloads", "events", name: "fk_36c74129da", on_delete: :cascade
add_foreign_key "releases", "projects", name: "fk_47fe2a0596", on_delete: :cascade
add_foreign_key "remote_mirrors", "projects", on_delete: :cascade
add_foreign_key "repository_languages", "projects", on_delete: :cascade
add_foreign_key "resource_label_events", "issues", on_delete: :cascade
add_foreign_key "resource_label_events", "labels", on_delete: :nullify
add_foreign_key "resource_label_events", "merge_requests", on_delete: :cascade
Loading
Loading
Loading
Loading
@@ -46,6 +46,10 @@ class Feature
get(key).enabled?(thing)
end
 
def disabled?(key, thing = nil)
!enabled?(key, thing)
end
def enable(key, thing = true)
get(key).enable(thing)
end
Loading
Loading
Loading
Loading
@@ -107,6 +107,7 @@ excluded_attributes:
- :storage_version
- :remote_mirror_available_overridden
- :description_html
- :repository_languages
snippets:
- :expired_at
merge_request_diff:
Loading
Loading
module Gitlab
class LanguageDetection
MAX_LANGUAGES = 5
def initialize(repository, repository_languages)
@repository = repository
@repository_languages = repository_languages
end
def languages
detection.keys
end
def language_color(name)
detection.dig(name, :color)
end
# Newly detected languages, returned in a structure accepted by
# Gitlab::Database.bulk_insert
def insertions(programming_languages)
lang_to_id = programming_languages.map { |p| [p.name, p.id] }.to_h
(languages - previous_language_names).map do |new_lang|
{
project_id: @repository.project.id,
share: detection[new_lang][:value],
programming_language_id: lang_to_id[new_lang]
}
end
end
# updates analyses which records only require updating of their share
def updates
to_update = @repository_languages.select do |lang|
detection.key?(lang.name) && detection[lang.name][:value] != lang.share
end
to_update.map do |lang|
{ programming_language_id: lang.programming_language_id, share: detection[lang.name][:value] }
end
end
# Returns the ids of the programming languages that do not occur in the detection
# as current repository languages
def deletions
@repository_languages.map do |repo_lang|
next if detection.key?(repo_lang.name)
repo_lang.programming_language_id
end.compact
end
private
def previous_language_names
@previous_language_names ||= @repository_languages.map(&:name)
end
def detection
@detection ||=
@repository
.languages
.first(MAX_LANGUAGES)
.map { |l| [l[:label], l] }
.to_h
end
end
end
FactoryBot.define do
factory :programming_language do
name 'Ruby'
color '#123456'
end
end
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment