Merge branch 'epic/sidekiq'

CHANGELOG

+v1.3.0
+  - Replace resque with sidekiq
+
 v1.2.0
   - Added Github web hook support
   - Added build schedule

Gemfile

@@ -26,9 +26,13 @@ gem "unicorn", "~> 4.4.0"
 # Haml
 gem 'haml-rails'
-# Jobs
-gem 'resque'
-gem 'resque-scheduler', :require => 'resque_scheduler'
+# Background jobs
+gem 'slim'
+gem 'sinatra', :require => nil
+gem 'sidekiq', '2.6.4'
+
+# Scheduled
+gem 'whenever', require: false
 # Format dates
 gem 'stamp'

Gemfile.lock

@@ -41,8 +41,12 @@ GEM
       rack-test (>= 0.5.4)
       selenium-webdriver (~> 2.0)
       xpath (~> 0.1.4)
+    celluloid (0.12.4)
+      facter (>= 1.6.12)
+      timers (>= 1.0.0)
     childprocess (0.3.5)
       ffi (~> 1.0, >= 1.0.6)
+    chronic (0.9.0)
     coderay (1.0.8)
     coffee-rails (3.2.2)
       coffee-script (>= 2.2.0)
@@ -51,6 +55,7 @@ GEM
       coffee-script-source
       execjs
     coffee-script-source (1.4.0)
+    connection_pool (1.0.0)
     daemons (1.1.9)
     devise (2.1.2)
       bcrypt-ruby (~> 3.0)
@@ -62,6 +67,7 @@ GEM
     eventmachine (1.0.0)
     execjs (1.4.0)
       multi_json (~> 1.0)
+    facter (1.6.17)
     factory_girl (4.1.0)
       activesupport (>= 3.0.0)
     factory_girl_rails (4.1.0)
@@ -106,7 +112,7 @@ GEM
       treetop (~> 1.4.8)
     method_source (0.8.1)
     mime-types (1.19)
-    multi_json (1.3.6)
+    multi_json (1.5.0)
     mysql2 (0.3.11)
     nokogiri (1.5.5)
     orm_adapter (0.4.0)
@@ -152,15 +158,6 @@ GEM
     redis (3.0.2)
     redis-namespace (1.2.1)
       redis (~> 3.0.0)
-    resque (1.23.0)
-      multi_json (~> 1.0)
-      redis-namespace (~> 1.0)
-      sinatra (>= 0.9.2)
-      vegas (~> 0.1.2)
-    resque-scheduler (2.0.0)
-      redis (>= 2.0.1)
-      resque (>= 1.20.0)
-      rufus-scheduler
     rspec (2.11.0)
       rspec-core (~> 2.11.0)
       rspec-expectations (~> 2.11.0)
@@ -175,8 +172,6 @@ GEM
       railties (>= 3.0)
       rspec (~> 2.11.0)
     rubyzip (0.9.9)
-    rufus-scheduler (2.0.17)
-      tzinfo (>= 0.3.23)
     sass (3.2.1)
     sass-rails (3.2.5)
       railties (~> 3.2.0)
@@ -190,10 +185,19 @@ GEM
     settingslogic (2.0.8)
     shoulda-matchers (1.3.0)
       activesupport (>= 3.0.0)
+    sidekiq (2.6.4)
+      celluloid (~> 0.12.0)
+      connection_pool (~> 1.0)
+      multi_json (~> 1)
+      redis (~> 3)
+      redis-namespace
     sinatra (1.3.3)
       rack (~> 1.3, >= 1.3.6)
       rack-protection (~> 1.2)
       tilt (~> 1.3, >= 1.3.3)
+    slim (1.3.6)
+      temple (~> 0.5.5)
+      tilt (~> 1.3.3)
     slop (3.3.3)
     sprockets (2.1.3)
       hike (~> 1.2)
@@ -201,6 +205,7 @@ GEM
       tilt (~> 1.1, != 1.3.0)
     stamp (0.3.0)
     state_machine (1.1.2)
+    temple (0.5.5)
     therubyracer (0.10.2)
       libv8 (~> 3.3.10)
     thin (1.5.0)
@@ -209,6 +214,7 @@ GEM
       rack (>= 1.0.0)
     thor (0.16.0)
     tilt (1.3.3)
+    timers (1.0.2)
     treetop (1.4.12)
       polyglot
       polyglot (>= 0.3.1)
@@ -220,10 +226,11 @@ GEM
       kgio (~> 2.6)
       rack
       raindrops (~> 0.7)
-    vegas (0.1.11)
-      rack (>= 1.0.0)
     warden (1.2.1)
       rack (>= 1.0)
+    whenever (0.8.2)
+      activesupport (>= 2.3.4)
+      chronic (>= 0.6.3)
     will_paginate (3.0.3)
     xpath (0.1.4)
       nokogiri (~> 1.3)
@@ -254,16 +261,18 @@ DEPENDENCIES
   rake
   rb-fsevent
   rb-inotify
-  resque
-  resque-scheduler
   rspec-rails
   sass-rails (~> 3.2.3)
   settingslogic
   shoulda-matchers
+  sidekiq (= 2.6.4)
+  sinatra
+  slim
   stamp
   state_machine
   therubyracer
   thin
   uglifier (>= 1.0.3)
   unicorn (~> 4.4.0)
+  whenever
   will_paginate (~> 3.0)

Procfile

 web: bundle exec rails s -p $PORT
-worker: bundle exec rake environment resque:work QUEUE=runner,scheduler_task VVERBOSE=1
-schedule: bundle exec rake environment resque:scheduler VVERBOSE=1 RAILS_ENV=development
+worker: bundle exec sidekiq -q runner,default

VERSION

-1.2.0
+1.3.0pre

app/controllers/projects_controller.rb

@@ -56,7 +56,7 @@ class ProjectsController < ApplicationController
     @build = @project.register_build(ref: params[:ref])
     if @build and @build.id
-      Resque.enqueue(Runner, @build.id)
+      Runner.perform_async(@build.id)
       redirect_to project_build_path(@project, @build)
     else
       redirect_to project_path(@project), notice: 'Branch is not defined for this project'
@@ -77,7 +77,7 @@ class ProjectsController < ApplicationController
    @build = @project.register_build(build_params)
    if @build
-     Resque.enqueue(Runner, @build.id)
+     Runner.perform_async(@build.id)
      head 200
    else
      head 500

app/models/build.rb

@@ -117,6 +117,7 @@ class Build < ActiveRecord::Base
 end
+
 # == Schema Information
 #
 # Table name: builds
@@ -126,9 +127,12 @@ end
 #  ref         :string(255)
 #  status      :string(255)
 #  finished_at :datetime
-#  trace       :text
+#  trace       :text(2147483647
 #  created_at  :datetime        not null
 #  updated_at  :datetime        not null
 #  sha         :string(255)
+#  started_at  :datetime
+#  tmp_file    :string(255)
+#  before_sha  :string(255)
 #

app/models/project.rb

@@ -13,11 +13,10 @@ class Project < ActiveRecord::Base
   validates_uniqueness_of :name
   validates :polling_interval,
-    format: { with: /^[1-9]\d{0,7}[m|h|d]$/ },
-    if: ->(project) { project.polling_interval.present? }
+    presence: true,
+    if: ->(project) { project.always_build.present? }
   before_validation :set_default_values
-  after_save :set_scheduler
   def set_default_values
     self.token = SecureRandom.hex(15) if self.token.blank?
@@ -122,20 +121,6 @@ class Project < ActiveRecord::Base
     self.token && self.token == token
   end
-  def set_scheduler
-    if self.always_build && self.polling_interval.present?
-      Resque.set_schedule(self.schedule_id, {
-                            :class => 'SchedulerJob',
-                            every: self.polling_interval,
-                            queue: 'scheduler_task',
-                            args: [:run, self.id],
-                            description: self.name
-                          })
-    else
-      Resque.remove_schedule(self.schedule_id)
-    end
-  end
-
   def schedule_id
     "project-#{id}"
   end
@@ -156,3 +141,22 @@ end
 #  token       :string(255)
 #  default_ref :string(255)
 #
+
+# == Schema Information
+#
+# Table name: projects
+#
+#  id               :integer(4)      not null, primary key
+#  name             :string(255)     not null
+#  path             :string(255)     not null
+#  timeout          :integer(4)      default(1800), not null
+#  scripts          :text            default(""), not null
+#  created_at       :datetime        not null
+#  updated_at       :datetime        not null
+#  token            :string(255)
+#  default_ref      :string(255)
+#  gitlab_url       :string(255)
+#  always_build     :boolean(1)      default(FALSE), not null
+#  polling_interval :integer(4)
+#
+

app/models/user.rb

@@ -6,6 +6,10 @@ class User < ActiveRecord::Base
   # Setup accessible (or protected) attributes for your model
   attr_accessible :email, :password, :password_confirmation, :remember_me
+
+  def admin?
+    true
+  end
 end
 # == Schema Information

app/views/layouts/application.html.haml

@@ -19,7 +19,7 @@
               %li
                 = link_to 'Users', users_path
               %li
-                = link_to 'Resque', resque_path
+                = link_to 'Background Jobs', resque_path
               %li
                 = link_to edit_user_path(current_user) do
                   %i.icon-user.icon-white

app/views/projects/_form.html.haml

@@ -55,12 +55,8 @@
         %br
         .field
           = f.label :polling_interval, "Build interval"
-          = f.text_field :polling_interval, placeholder: '5h'
-          %ul
-            %li m - minutes
-            %li h - hours
-            %li d - days
-          %p Ex. 30m means every 30 minutes and 2d - every two days
+          = f.number_field :polling_interval, placeholder: '5'
+          %span Hours
   .form-actions
     = f.submit 'Save', class: 'btn btn-primary'
     = link_to 'Cancel', projects_path, class: 'btn'

app/views/resques/show.html.haml

-%h3.page_title Resque
+%h3.page_title Background Jobs
 %br
 .bordered
   %iframe{src: ext_resque_url, width: '100%', height: 600, style: "border: none"}

config/initializers/2_resque.rb

-require Rails.root.join('lib', 'runner')
-require 'scheduler_job'
-
-# Custom Redis configuration
-config_file = Rails.root.join('config', 'resque.yml')
-
-if File.exists?(config_file)
-  resque_config = YAML.load_file(config_file)
-  Resque.redis = resque_config[Rails.env]
-end
-
-Resque.redis.namespace = 'resque:gitlab_ci'
-
-# Queues
-Resque.watch_queue(Runner.instance_variable_get("@queue"))
-
-# Authentication
-require 'resque/server'
-class Authentication
-  def initialize(app)
-    @app = app
-  end
-
-  def call(env)
-    account = env['warden'].authenticate!(:database_authenticatable, :rememberable, scope: :user)
-    raise "Access denied" if !account#.admin?
-    @app.call(env)
-  end
-end
-
-Resque::Server.use Authentication
-
-# Mailer
-# For future email notifications
-#Resque::Mailer.excluded_environments = []

config/initializers/2_sidekiq.rb

+# Custom Redis configuration
+config_file = Rails.root.join('config', 'resque.yml')
+
+resque_url = if File.exists?(config_file)
+               YAML.load_file(config_file)[Rails.env]
+             else
+               "localhost:6379"
+             end
+
+Sidekiq.configure_server do |config|
+  config.redis = {
+    url: "redis://#{resque_url}",
+    namespace: 'resque:gitlab_ci'
+  }
+end
+
+Sidekiq.configure_client do |config|
+  config.redis = {
+    url: "redis://#{resque_url}",
+    namespace: 'resque:gitlab_ci'
+  }
+end

config/routes.rb

+require 'sidekiq/web'
+
 GitlabCi::Application.routes.draw do
   # Optionally, enable Resque here
-  require 'resque/server'
-  require 'resque_scheduler'
-  require 'resque_scheduler/server'
-  mount Resque::Server => '/ext/resque', as: 'ext_resque'
+  constraint = lambda { |request| request.env["warden"].authenticate? and request.env['warden'].user.admin? }
+  constraints constraint do
+    mount Sidekiq::Web, at: "/ext/sidekiq", as: :ext_resque
+  end
   resources :projects do
     member do

config/schedule.rb

+# Use this file to easily define all of your cron jobs.
+#
+every 1.hour do
+  runner "Scheduler.new.perform"
+end

db/migrate/20130114153451_change_schedule_invertal.rb

+class ChangeScheduleInvertal < ActiveRecord::Migration
+  def up
+    change_column :projects, :polling_interval, :integer, null: true
+  end
+
+  def down
+    change_column :projects, :polling_interval, :string, null: true
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended to check this file into your version control system.
-ActiveRecord::Schema.define(:version => 20121224092350) do
+ActiveRecord::Schema.define(:version => 20130114153451) do
   create_table "builds", :force => true do |t|
     t.integer  "project_id"
@@ -28,17 +28,17 @@ ActiveRecord::Schema.define(:version => 20121224092350) do
   end
   create_table "projects", :force => true do |t|
-    t.string   "name",                               :null => false
-    t.string   "path",                               :null => false
-    t.integer  "timeout",          :default => 1800, :null => false
-    t.text     "scripts",                            :null => false
-    t.datetime "created_at",                         :null => false
-    t.datetime "updated_at",                         :null => false
+    t.string   "name",                                :null => false
+    t.string   "path",                                :null => false
+    t.integer  "timeout",          :default => 1800,  :null => false
+    t.text     "scripts",                             :null => false
+    t.datetime "created_at",                          :null => false
+    t.datetime "updated_at",                          :null => false
     t.string   "token"
     t.string   "default_ref"
     t.string   "gitlab_url"
-    t.boolean  "always_build",     :default => true
-    t.string   "polling_interval"
+    t.boolean  "always_build",     :default => false, :null => false
+    t.integer  "polling_interval"
   end
   create_table "users", :force => true do |t|

lib/runner.rb

@@ -2,22 +2,25 @@ require 'open3'
 require 'timeout'
 class Runner
+  include Sidekiq::Worker
+
   TIMEOUT = 1800
   attr_accessor :project, :build, :output
-  @queue = :runner
-  def self.perform(build_id)
-    new(Build.find(build_id)).run
+  sidekiq_options queue: :runner
+
+  def perform(build_id)
+    @build = Build.find(build_id)
+    @project = @build.project
+    @output = ''
+
+    run
   end
-  def initialize(build)
+  def initialize
     @logger = Logger.new(STDOUT)
     @logger.level = Logger::INFO
-
-    @build = build
-    @project = build.project
-    @output = ''
   end
   def run

lib/scheduler.rb

+class Scheduler
+  def perform
+    projects = Project.where(always_build: true).all
+    projects.each do |project|
+      last_build_time = project.last_build.created_at
+      interval = project.polling_interval
+      if (last_build_time + interval.hours) < Time.now
+        build = project.register_build(ref: project.tracked_refs.first)
+        if build and build.id
+          Runner.perform_async(build.id)
+        end
+      end
+    end
+  end
+end