Merge branch 'improve-contributions-calendar' into 'master'

Replace commits calendar with contributions calendar

* count opening of issues and merge requests
* dont trigger git repository - use events from database
* count pushes instead of commits for faster and easier counting
* much-much faster since does not affected by repository size

See merge request !420

CHANGELOG

@@ -28,6 +28,7 @@ v 7.10.0 (unreleased)
   - Restrict permissions on backup files
   - Improve oauth accounts UI in profile page
   - Add ability to unlink connected accounts
+  - Replace commits calendar with faster contribution calendar that includes issues and merge requests
 v 7.9.0
   - Add HipChat integration documentation (Stan Hu)

app/assets/javascripts/calendar.js.coffee

@@ -7,7 +7,7 @@ class @calendar
   constructor: (timestamps, starting_year, starting_month, calendar_activities_path) ->
     cal = new CalHeatMap()
     cal.init
-      itemName: ["commit"]
+      itemName: ["contribution"]
       data: timestamps
       start: new Date(starting_year, starting_month)
       domainLabelFormat: "%b"
@@ -27,7 +27,6 @@ class @calendar
       legendCellPadding: 3
       onClick: (date, count) ->
         formated_date = date.getFullYear() + "-" + (date.getMonth()+1) + "-" + date.getDate()
-        $(".calendar_commit_activity").fadeOut 400
         $.ajax
           url: calendar_activities_path
           data:
@@ -36,6 +35,4 @@ class @calendar
           dataType: "html"
           success: (data) ->
             $(".user-calendar-activities").html data
-            $(".calendar_commit_activity").find(".js-toggle-content").hide()
-            $(".calendar_commit_activity").fadeIn 400

app/assets/stylesheets/generic/calendar.scss

 .user-calendar-activities {
-  
-  .calendar_commit_activity {
-    padding: 5px 0 0;
-  }
-  
   .calendar_onclick_hr {
     padding: 0;
     margin: 10px 0;
   }
-  
-  .calendar_commit_date {
-    color: #999;
-  }
-  
-  .calendar_activity_summary {
-    font-size: 14px;
-  }
   .str-truncated {
     max-width: 70%;
@@ -31,14 +18,6 @@
       background-color: #ddd;
     }
   }
-  
-  .commit-row-message {
-    color: #333;
-    &:hover {
-      color: #444;
-      text-decoration: underline;
-    }
-  }
 }
 /**
 * This overwrites the default values of the cal-heatmap gem

app/controllers/users_controller.rb

@@ -4,10 +4,7 @@ class UsersController < ApplicationController
   layout :determine_layout
   def show
-    @contributed_projects = Project.
-      where(id: authorized_projects_ids & @user.contributed_projects_ids).
-      in_group_namespace.
-      includes(:namespace).
+    @contributed_projects = contributed_projects.joined(@user).
       reject(&:forked?)
     @projects = @user.personal_projects.
@@ -31,9 +28,7 @@ class UsersController < ApplicationController
   end
   def calendar
-    projects = Project.where(id: authorized_projects_ids & @user.contributed_projects_ids)
-
-    calendar = Gitlab::CommitsCalendar.new(projects, @user)
+    calendar = contributions_calendar
     @timestamps = calendar.timestamps
     @starting_year = calendar.starting_year
     @starting_month = calendar.starting_month
@@ -42,20 +37,13 @@ class UsersController < ApplicationController
   end
   def calendar_activities
-    projects = Project.where(id: authorized_projects_ids & @user.contributed_projects_ids)
-    date = Date.parse(params[:date]) rescue nil
-    if date
-      @calendar_activities = Gitlab::CommitsCalendar.get_commits_for_date(projects, @user, date)
-    else
-      @calendar_activities = {}
+    @calendar_date = Date.parse(params[:date]) rescue nil
+    @events = []
+    if @calendar_date
+      @events = contributions_calendar.events_by_date(@calendar_date)
     end
-    # get the total number of unique commits
-    @commit_count = @calendar_activities.values.flatten.map(&:id).uniq.count
-
-    @calendar_date = date
-
     render 'calendar_activities', layout: false
   end
@@ -82,4 +70,15 @@ class UsersController < ApplicationController
     @authorized_projects_ids ||=
       ProjectsFinder.new.execute(current_user).pluck(:id)
   end
+
+  def contributed_projects
+    @contributed_projects = Project.
+      where(id: authorized_projects_ids & @user.contributed_projects_ids).
+      includes(:namespace)
+  end
+
+  def contributions_calendar
+    @contributions_calendar ||= Gitlab::ContributionsCalendar.
+      new(contributed_projects.reject(&:forked?), @user)
+  end
 end

app/models/event.rb

@@ -55,6 +55,12 @@ class Event < ActiveRecord::Base
         order('id DESC').limit(100).
         update_all(updated_at: Time.now)
     end
+
+    def contributions
+      where("action = ? OR (target_type in (?) AND action in (?))",
+            Event::PUSHED, ["MergeRequest", "Issue"],
+            [Event::CREATED, Event::CLOSED, Event::MERGED])
+    end
   end
   def proper?

app/models/project_contributions.rb

-class ProjectContributions
-  attr_reader :project, :user
-
-  def initialize(project, user)
-    @project, @user = project, user
-  end
-
-  def commits_log
-    repository = project.repository
-
-    if !repository.exists? || repository.empty?
-      return {}
-    end
-
-    Rails.cache.fetch(cache_key) do
-      repository.commits_per_day_for_user(user)
-    end
-  end
-
-  def user_commits_on_date(date)
-    repository = @project.repository
-
-    if !repository.exists? || repository.empty?
-      return []
-    end
-    commits = repository.commits_by_user_on_date_log(@user, date)
-  end
-
-  def cache_key
-    "#{Date.today.to_s}-commits-log-#{project.id}-#{user.email}"
-  end
-end

app/models/repository.rb

@@ -149,41 +149,6 @@ class Repository
     end
   end
-  def timestamps_by_user_log(user)
-    author_emails = '(' + user.all_emails.map{ |e| Regexp.escape(e) }.join('|') + ')'
-    args = %W(git log -E --author=#{author_emails} --since=#{(Date.today - 1.year).to_s} --branches --pretty=format:%cd --date=short)
-    dates = Gitlab::Popen.popen(args, path_to_repo).first.split("\n")
-
-    if dates.present?
-      dates
-    else
-      []
-    end
-  end
-
-  def commits_by_user_on_date_log(user, date)
-    # format the date string for git
-    start_date = date.strftime("%Y-%m-%d 00:00:00")
-    end_date = date.strftime("%Y-%m-%d 23:59:59")
-
-    author_emails = '(' + user.all_emails.map{ |e| Regexp.escape(e) }.join('|') + ')'
-    args = %W(git log -E --author=#{author_emails} --after=#{start_date.to_s} --until=#{end_date.to_s} --branches --pretty=format:%h)
-    commits = Gitlab::Popen.popen(args, path_to_repo).first.split("\n")
-
-    commits.map! do |commit_id|
-      commit(commit_id)
-    end
-  end
-
-  def commits_per_day_for_user(user)
-    timestamps_by_user_log(user).
-      group_by { |commit_date| commit_date }.
-      inject({}) do |hash, (timestamp_date, commits)|
-        hash[timestamp_date] = commits.count
-        hash
-      end
-  end
-
   def lookup_cache
     @lookup_cache ||= {}
   end

app/models/user.rb

@@ -603,13 +603,10 @@ class User < ActiveRecord::Base
   end
   def contributed_projects_ids
-    Event.where(author_id: self).
+    Event.contributions.where(author_id: self).
       where("created_at > ?", Time.now - 1.year).
-      where("action = :pushed OR (target_type = 'MergeRequest' AND action = :created)",
-        pushed: Event::PUSHED, created: Event::CREATED).
       reorder(project_id: :desc).
       select(:project_id).
-      uniq
-      .map(&:project_id)
+      uniq.map(&:project_id)
   end
 end

app/views/users/_projects.html.haml

 - if @contributed_projects.present?
-  .panel.panel-default
+  .panel.panel-default.contributed-projects
     .panel-heading Projects contributed to
     = render 'shared/projects_list',
       projects: @contributed_projects.sort_by(&:star_count).reverse,

app/views/users/calendar.html.haml

-%h4 Commits calendar
+%h4
+  Contributions calendar
+  .pull-right
+    %small Issues, merge requests and push events
 #cal-heatmap.calendar
   :javascript
     new calendar(

app/views/users/calendar_activities.html.haml

-.calendar_commit_activity
-  %hr
-  %h4
-    Commit Activity
-  %strong
-    - if @commit_count == 0
-      no
-    - else
-      = @commit_count
-  %span.calendar_commit_date
-    unique
-    = 'commit'.pluralize(@commit_count)
-    on
-    = @calendar_date.strftime("%b %d, %Y") rescue ''
-  -unless @commit_count == 0
-    %hr
-  - @calendar_activities.each do |project, commits|
-    - next if commits.empty?
-    %div.js-toggle-container
+%h4.prepend-top-20
+  %span.light Contributions for
+  %strong #{@calendar_date.to_s(:short)}
+
+%ul.bordered-list
+  - @events.sort_by(&:created_at).each do |event|
+    %li
+      %span.light
+        %i.fa.fa-clock-o
+        = event.created_at.to_s(:time)
+      - if event.push?
+        #{event.action_name} #{event.ref_type} #{event.ref_name}
+      - else
+        = event_action_name(event)
+        - if event.target
+          %strong= link_to "##{event.target_iid}", [event.project.namespace.becomes(Namespace), event.project, event.target]
+
+      at
       %strong
-        = pluralize(commits.count, 'commit')
-        in project
-        = link_to project.name_with_namespace, project_path(project)
-        %a.text-expander.js-toggle-button …
-        %hr
-      %div.js-toggle-content
-        - commits.each do |commit|
-          %span.monospace
-            = commit.committed_date.strftime("%H:%M")
-            = link_to commit.short_id, namespace_project_commit_path(project.namespace, project, commit), class: "commit_short_id"
-          = link_to commit.message, namespace_project_commit_path(project.namespace, project, commit), class: "commit-row-message str-truncated"
-          %br
-        %hr
+        - if event.project
+          = link_to_project event.project
+        - else
+          = event.project_name

features/steps/user.rb

@@ -7,4 +7,37 @@ class Spinach::Features::User < Spinach::FeatureSteps
   step 'I should see user "John Doe" page' do
     expect(title).to match(/^\s*John Doe/)
   end
+
+  step '"John Doe" has contributions' do
+    user = User.find_by(name: 'John Doe')
+    project = contributed_project
+
+    # Issue controbution
+    issue_params = { title: 'Bug in old browser' }
+    Issues::CreateService.new(project, user, issue_params).execute
+
+    # Push code contribution
+    push_params = {
+      project: project,
+      action: Event::PUSHED,
+      author_id: user.id,
+      data: { commit_count: 3 }
+    }
+
+    Event.create(push_params)
+  end
+
+  step 'I should see contributed projects' do
+    within '.contributed-projects' do
+      page.should have_content(@contributed_project.name)
+    end
+  end
+
+  step 'I should see contributions calendar' do
+    page.should have_css('.cal-heatmap-container')
+  end
+
+  def contributed_project
+    @contributed_project ||= create(:project, :public)
+  end
 end

features/user.feature

@@ -67,3 +67,12 @@ Feature: User
     And I should see project "Enterprise"
     And I should not see project "Internal"
     And I should not see project "Community"
+
+  @javascript
+  Scenario: "John Doe" contribution profile
+    Given I sign in as a user
+    And "John Doe" has contributions
+    When I visit user "John Doe" page
+    Then I should see user "John Doe" page
+    And I should see contributed projects
+    And I should see contributions calendar

lib/gitlab/commits_calendar.rb

-module Gitlab
-  class CommitsCalendar
-    attr_reader :timestamps
-
-    def initialize(projects, user)
-      @timestamps = {}
-      date_timestamps = []
-
-      projects.reject(&:forked?).each do |project|
-        date_timestamps << ProjectContributions.new(project, user).commits_log
-      end
-
-      # Sumarrize commits from all projects per days
-      date_timestamps = date_timestamps.inject do |collection, date|
-        collection.merge(date) { |k, old_v, new_v| old_v + new_v }
-      end
-
-      date_timestamps ||= []
-      date_timestamps.each do |date, commits|
-        timestamp = Date.parse(date).to_time.to_i.to_s rescue nil
-        @timestamps[timestamp] = commits if timestamp
-      end
-    end
-
-    def self.get_commits_for_date(projects, user, date)
-      user_commits = {}
-      projects.reject(&:forked?).each do |project|
-        user_commits[project] = ProjectContributions.new(project, user).user_commits_on_date(date)
-      end
-      user_commits
-    end
-
-    def starting_year
-      (Time.now - 1.year).strftime("%Y")
-    end
-
-    def starting_month
-      Date.today.strftime("%m").to_i
-    end
-  end
-end

lib/gitlab/contributions_calendar.rb

+module Gitlab
+  class ContributionsCalendar
+    attr_reader :timestamps, :projects, :user
+
+    def initialize(projects, user)
+      @projects = projects
+      @user = user
+    end
+
+    def timestamps
+      return @timestamps if @timestamps.present?
+
+      @timestamps = {}
+      date_from = 1.year.ago
+      date_to = Date.today
+
+      events = Event.reorder(nil).contributions.where(author_id: user.id).
+        where("created_at > ?", date_from).where(project_id: projects).
+        group('date(created_at)').
+        select('date(created_at), count(id) as total_amount').
+        map(&:attributes)
+
+      dates = (1.year.ago.to_date..(Date.today + 1.day)).to_a
+
+      dates.each do |date|
+        date_id = date.to_time.to_i.to_s
+        @timestamps[date_id] = 0
+        day_events = events.find { |day_events| day_events["date"] == date }
+
+        if day_events
+          @timestamps[date_id] = day_events["total_amount"]
+        end
+      end
+
+      @timestamps
+    end
+
+    def events_by_date(date)
+      events = Event.contributions.where(author_id: user.id).
+        where("created_at > ? AND created_at < ?", date.beginning_of_day, date.end_of_day).
+        where(project_id: projects)
+
+      events.select do |event|
+        event.push? || event.issue? || event.merge_request?
+      end
+    end
+
+    def starting_year
+      (Time.now - 1.year).strftime("%Y")
+    end
+
+    def starting_month
+      Date.today.strftime("%m").to_i
+    end
+  end
+end

spec/controllers/users_controller_spec.rb

@@ -25,34 +25,21 @@ describe UsersController do
   end
   describe 'GET #calendar_activities' do
-    include RepoHelpers
-    let(:project) { create(:project) }
-    let(:calendar_user) { create(:user, email: sample_commit.author_email) }
-    let(:commit1) { '0ed8c6c6752e8c6ea63e7b92a517bf5ac1209c80' }
-    let(:commit2) { '7d3b0f7cff5f37573aea97cebfd5692ea1689924' }
+    let!(:project) { create(:project) }
+    let!(:user) { create(:user) }
     before do
       allow_any_instance_of(User).to receive(:contributed_projects_ids).and_return([project.id])
       project.team << [user, :developer]
     end
-    it 'assigns @commit_count' do
-      get :calendar_activities, username: calendar_user.username, date: '2014-07-31'
-      expect(assigns(:commit_count)).to eq(2)
-    end
-
     it 'assigns @calendar_date' do
-      get :calendar_activities, username: calendar_user.username, date: '2014-07-31'
+      get :calendar_activities, username: user.username, date: '2014-07-31'
       expect(assigns(:calendar_date)).to eq(Date.parse('2014-07-31'))
     end
-    it 'assigns @calendar_activities' do
-      get :calendar_activities, username: calendar_user.username, date: '2014-07-31'
-      expect(assigns(:calendar_activities).values.flatten.map(&:id)).to eq([commit1, commit2])
-    end
-
     it 'renders calendar_activities' do
-      get :calendar_activities, username: calendar_user.username
+      get :calendar_activities, username: user.username
       expect(response).to render_template('calendar_activities')
     end
   end

spec/models/repository_spec.rb

@@ -13,47 +13,16 @@ describe Repository do
     it { is_expected.not_to include('fix') }
   end
-  describe :last_commit_for_path do
-    subject { repository.last_commit_for_path(sample_commit.id, '.gitignore').id }
-
-    it { is_expected.to eq('c1acaa58bbcbc3eafe538cb8274ba387047b69f8') }
-  end
-
-  context :timestamps_by_user_log do
-    before do
-      Date.stub(:today).and_return(Date.new(2015, 03, 01))
-    end
-
-    describe 'single e-mail for user' do
-      let(:user) { create(:user, email: sample_commit.author_email) }
-
-      subject { repository.timestamps_by_user_log(user) }
-      it { is_expected.to eq(['2014-08-06', '2014-07-31', '2014-07-31']) }
-    end
-
-    describe 'multiple emails for user' do
-      let(:email_alias) { create(:email, email: another_sample_commit.author_email) }
-      let(:user) { create(:user, email: sample_commit.author_email, emails: [email_alias]) }
-
-      subject { repository.timestamps_by_user_log(user) }
-
-      it { is_expected.to eq(['2015-01-10', '2014-08-06', '2014-07-31', '2014-07-31']) }
-    end
+  describe :tag_names_contains do
+    subject { repository.tag_names_contains(sample_commit.id) }
+    it { is_expected.to include('v1.1.0') }
+    it { is_expected.not_to include('v1.0.0') }
   end
-  context :commits_by_user_on_date_log do
-
-    describe 'single e-mail for user' do
-      let(:user) { create(:user, email: sample_commit.author_email) }
-      let(:commit1) { '0ed8c6c6752e8c6ea63e7b92a517bf5ac1209c80' }
-      let(:commit2) { '7d3b0f7cff5f37573aea97cebfd5692ea1689924' }
-
-      subject { repository.commits_by_user_on_date_log(user,Date.new(2014, 07, 31)) }
-      it 'contains the exepected commits' do
-        expect(subject.flatten.map(&:id)).to eq([commit1, commit2])
-      end
-    end
+  describe :last_commit_for_path do
+    subject { repository.last_commit_for_path(sample_commit.id, '.gitignore').id }
+    it { is_expected.to eq('c1acaa58bbcbc3eafe538cb8274ba387047b69f8') }
   end
 end