Merge branch 'lfs-authenticate-support' into 'master'

Added LFS support to SSH

Required changes to GitLab Shell include the actual handling of the `git-lfs-authenticate` command and the retrieval of the correct credentials.

Needed for gitlab-org/gitlab-ce!6043

Related to gitlab-org/gitlab-ce#3589

> **Note:** gitlab-org/gitlab-ce!6043 needs to be merged before this one.

cc @jacobvosmaer-gitlab @marin @DouweM

See merge request !86

CHANGELOG

 v3.5.0
   - Add option to recover 2FA via SSH
+  - Added full support for `git-lfs-authenticate` to properly handle LFS requests and pass them on to Workhorse
 v3.4.0
   - Redis Sentinel support

lib/gitlab_lfs_authentication.rb

+require 'base64'
+require 'json'
+
+class GitlabLfsAuthentication
+  attr_accessor :username, :lfs_token, :repository_http_path
+
+  def initialize(username, lfs_token, repository_http_path)
+    @username = username
+    @lfs_token = lfs_token
+    @repository_http_path = repository_http_path
+  end
+
+  def self.build_from_json(json)
+    begin
+      values = JSON.parse(json)
+      self.new(values['username'], values['lfs_token'], values['repository_http_path'])
+    rescue
+      nil
+    end
+  end
+
+  def authentication_payload
+    authorization = {
+      header: {
+        Authorization: "Basic #{Base64.strict_encode64("#{username}:#{lfs_token}")}"
+      },
+      href: "#{repository_http_path}/info/lfs/"
+    }
+
+    JSON.generate(authorization)
+  end
+end

lib/gitlab_net.rb

@@ -6,6 +6,7 @@ require_relative 'gitlab_config'
 require_relative 'gitlab_logger'
 require_relative 'gitlab_access'
 require_relative 'gitlab_redis'
+require_relative 'gitlab_lfs_authentication'
 require_relative 'httpunix'
 class GitlabNet
@@ -15,15 +16,12 @@ class GitlabNet
   READ_TIMEOUT = 300
   def check_access(cmd, repo, actor, changes, protocol)
-    project_name = repo.gsub("'", "")
-    project_name = project_name.gsub(/\.git\Z/, "")
-    project_name = project_name.gsub(/\A\//, "")
     changes = changes.join("\n") unless changes.kind_of?(String)
     params = {
       action: cmd,
       changes: changes,
-      project: project_name,
+      project: project_name(repo),
       protocol: protocol
     }
@@ -49,6 +47,19 @@ class GitlabNet
     JSON.parse(resp.body) rescue nil
   end
+  def lfs_authenticate(key, repo)
+    params = {
+      project: project_name(repo),
+      key_id: key.gsub('key-', '')
+    }
+
+    resp = post("#{host}/lfs_authenticate", params)
+
+    if resp.code == '200'
+      GitlabLfsAuthentication.build_from_json(resp.body)
+    end
+  end
+
   def broadcast_message
     resp = get("#{host}/broadcast_message")
     JSON.parse(resp.body) rescue {}
@@ -107,6 +118,12 @@ class GitlabNet
   protected
+  def project_name(repo)
+    project_name = repo.gsub("'", "")
+    project_name = project_name.gsub(/\.git\Z/, "")
+    project_name.gsub(/\A\//, "")
+  end
+
   def config
     @config ||= GitlabConfig.new
   end

lib/gitlab_shell.rb

@@ -11,7 +11,7 @@ class GitlabShell
   API_COMMANDS = %w(2fa_recovery_codes)
   GL_PROTOCOL = 'ssh'.freeze
-  attr_accessor :key_id, :repo_name, :command
+  attr_accessor :key_id, :repo_name, :command, :git_access
   attr_reader :repo_path
   def initialize(key_id)
@@ -117,6 +117,11 @@ class GitlabShell
       $logger.info "gitlab-shell: executing git-annex command <#{parsed_args.join(' ')}> for #{log_username}."
       exec_cmd(*parsed_args)
+
+    elsif @command == 'git-lfs-authenticate'
+      $logger.info "gitlab-shell: Processing LFS authentication for #{log_username}."
+      lfs_authenticate
+
     else
       $logger.info "gitlab-shell: executing git command <#{@command} #{repo_path}> for #{log_username}."
       exec_cmd(@command, repo_path)
@@ -184,6 +189,14 @@ class GitlabShell
     non_dashed[0, 2] == %w{git-annex-shell gcryptsetup}
   end
+  def lfs_authenticate
+    lfs_access = api.lfs_authenticate(@key_id, @repo_name)
+
+    return unless lfs_access
+
+    puts lfs_access.authentication_payload
+  end
+
   private
   def continue?(question)

spec/gitlab_lfs_authentication_spec.rb

+require 'spec_helper'
+require 'gitlab_lfs_authentication'
+require 'json'
+
+describe GitlabLfsAuthentication do
+  subject do
+    GitlabLfsAuthentication.build_from_json(
+      JSON.generate(
+        {
+          username: 'dzaporozhets',
+          lfs_token: 'wsnys8Zm8Jn7zyhHTAAK',
+          repository_http_path: 'http://gitlab.dev/repo'
+        }
+      )
+    )
+  end
+
+  describe '#build_from_json' do
+    it { subject.username.should == 'dzaporozhets' }
+    it { subject.lfs_token.should == 'wsnys8Zm8Jn7zyhHTAAK' }
+    it { subject.repository_http_path.should == 'http://gitlab.dev/repo' }
+  end
+
+  describe '#authentication_payload' do
+    result = "{\"header\":{\"Authorization\":\"Basic ZHphcG9yb3poZXRzOndzbnlzOFptOEpuN3p5aEhUQUFL\"},\"href\":\"http://gitlab.dev/repo/info/lfs/\"}"
+
+    it { subject.authentication_payload.should eq(result) }
+
+    it 'should be a proper JSON' do
+      payload = subject.authentication_payload
+      json_payload = JSON.parse(payload)
+
+      json_payload['header']['Authorization'].should eq('Basic ZHphcG9yb3poZXRzOndzbnlzOFptOEpuN3p5aEhUQUFL')
+      json_payload['href'].should eq('http://gitlab.dev/repo/info/lfs/')
+    end
+  end
+end

spec/gitlab_net_spec.rb

@@ -38,6 +38,7 @@ describe GitlabNet, vcr: true do
       VCR.use_cassette("discover-ok") do
         user = gitlab_net.discover('key-126')
         user['name'].should == 'Dmitriy Zaporozhets'
+        user['username'].should == 'dzaporozhets'
       end
     end
@@ -56,6 +57,19 @@ describe GitlabNet, vcr: true do
     end
   end
+  describe '#lfs_authenticate' do
+    context 'lfs authentication succeeded' do
+      it 'should return the correct data' do
+        VCR.use_cassette('lfs-authenticate-ok') do
+          lfs_access = gitlab_net.lfs_authenticate('key-126', 'gitlab/gitlabhq.git')
+          lfs_access.username.should == 'dzaporozhets'
+          lfs_access.lfs_token.should == 'wsnys8Zm8Jn7zyhHTAAK'
+          lfs_access.repository_http_path.should == 'http://gitlab.dev/gitlab/gitlabhq.git'
+        end
+      end
+    end
+  end
+
   describe :broadcast_message do
     context "broadcast message exists" do
       it 'should return message' do

spec/gitlab_shell_spec.rb

@@ -112,6 +112,32 @@ describe GitlabShell do
       its(:repo_name) { should == 'dzaporozhets/gitlab.git' }
       its(:command) { should == 'git-annex-shell' }
     end
+
+    describe 'git-lfs' do
+      let(:repo_name) { 'dzaporozhets/gitlab.git' }
+      let(:ssh_args) { %W(git-lfs-authenticate dzaporozhets/gitlab.git download) }
+
+      before do
+        subject.send :parse_cmd, ssh_args
+      end
+
+      its(:repo_name) { should == 'dzaporozhets/gitlab.git' }
+      its(:command) { should == 'git-lfs-authenticate' }
+      its(:git_access) { should == 'git-upload-pack' }
+    end
+
+    describe 'git-lfs old clients' do
+      let(:repo_name) { 'dzaporozhets/gitlab.git' }
+      let(:ssh_args) { %W(git-lfs-authenticate dzaporozhets/gitlab.git download long_oid) }
+
+      before do
+        subject.send :parse_cmd, ssh_args
+      end
+
+      its(:repo_name) { should == 'dzaporozhets/gitlab.git' }
+      its(:command) { should == 'git-lfs-authenticate' }
+      its(:git_access) { should == 'git-upload-pack' }
+    end
   end
   describe :exec do

spec/vcr_cassettes/lfs-authenticate-ok.yml

+---
+http_interactions:
+- request:
+    method: post
+    uri: https://dev.gitlab.org/api/v3/internal/lfs_authenticate
+    body:
+      encoding: US-ASCII
+      string: project=gitlab%2Fgitlabhq&key_id=126&secret_token=a123
+    headers:
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Server:
+      - nginx/1.1.19
+      Date:
+      - Wed, 03 Sep 2014 11:27:35 GMT
+      Content-Type:
+      - application/json
+      Content-Length:
+      - '56'
+      Connection:
+      - keep-alive
+      Status:
+      - 200 OK
+      Etag:
+      - '"1d75c1cf3d4bfa4d2b7bb6a0bcfd7f55"'
+      Cache-Control:
+      - max-age=0, private, must-revalidate
+      X-Request-Id:
+      - ef4513ae-0424-4941-8be0-b5a3a7b4bf12
+      X-Runtime:
+      - '0.016934'
+    body:
+      encoding: UTF-8
+      string: '{"username":"dzaporozhets","lfs_token":"wsnys8Zm8Jn7zyhHTAAK","repository_http_path":"http://gitlab.dev/gitlab/gitlabhq.git"}'
+    http_version:
+  recorded_at: Wed, 03 Sep 2014 11:27:35 GMT
+recorded_with: VCR 2.4.0