Run/time a script in production to estimate performance for rev-list / blob fetch (For JEJ)

I'd like to get an estimate for how long a couple of things might take in production:

Rev-list

Benchmark.bm do |b|
  b.report('rev_list: ') do
    project = Project.find_by_full_path('gitlab-org/gitlab-ce')
    path_to_repo = project.repository.path_to_repo
    newrev = '84c2085e94ddc3b813b18b0208b30b31fb66dac4'
    base_rev = '84c2085e94ddc3b813b18b0208b30b31fb66dac4~1000'
    args = [Gitlab.config.git.bin_path, "--git-dir=#{path_to_repo}", 'rev-list', newrev, '--not', base_rev, '--objects']
    Gitlab::Popen.popen(args, nil, Gitlab::Git::Env.all.stringify_keys)
  end
end

Blob fetch

Snippet for test_blob_fetch.rb: https://gitlab.com/snippets/1676165

require './test_blob_fetch.rb'

project = Project.find_by_full_path('jamedjo/engine')

new_objects = ["4b67e82213a2ce0f8b4ed1d65bef48ef8b8be1f3",
"21d117dd22a6fc1e39e1a7d82184359fb40af3f3",
"56b13c63a5aaca766831bafa942ea9fcaf9f7d9c",
"df780cc64ca73b477816f2bc6ed377f70d686e44",
"76360011149eb1c1bcdacb359e081fb959d8add1",
"31662eda42accab19789d2e04a9d08217fa76b4f",
"49914c0b0f1461543263acbf7082a4f9e0387282",
"54c5f340c18cc0494fc0983e152b78e9c0cddbf6",
"5ae08eefdcdbdd821ce32082589879348ec6dbc7",
"d6a9a74d40c3f380a69096a82a849454c35ee0d9",
"24a8e87939aa53cdd833f6be7610cb4972e063ad",
"ce5363de5916038291484584fc935f58f765b208",
"b13f643410b7c25a3269992163b0fbbbdcc086a6",
"738d6c095a76859f99f6852f36bb2ea180027f9e",
"f5e81fc0276f53e256959224b0a59aeba7e12708",
"93d0e703e013922701e64e5e85e0c7d575377fe8",
"278e95c4c05407c5b27d0b5dbbed5351b2ceb545",
"cc7fcaa135d01df224b301d6a5e4de3388d349c9",
"8b19dd8432ee674a98302a1ae4a3269558cd1fbd",
"bf8312bfb54045f62bf8905fbf61cc2fdabab60a",
"8e665c70a2ffb1e1bc17cf619d8f6bcefe575c4b",
"4939c9a12f27552f987df6212c463559e8046e51",
"f93e3a1a1525fb5b91020da86e44810c87a2d7bc",
"e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"]

Benchmark.bm do |b|
  b.report('blob_fetch: ') do
    TestBlobFetch.new_lfs_blobs(project, new_objects)
  end
end

removed assignee

changed the description

irb(main):001:0> Benchmark.bm do |b|
irb(main):002:1*   b.report('rev_list: ') do
irb(main):003:2*     project = Project.find_by_full_path('gitlab-org/gitlab-ce')
irb(main):004:2>     path_to_repo = project.repository.path_to_repo
irb(main):005:2>     newrev = '84c2085e94ddc3b813b18b0208b30b31fb66dac4'
irb(main):006:2>     base_rev = '84c2085e94ddc3b813b18b0208b30b31fb66dac4~1000'
irb(main):007:2>     args = [Gitlab.config.git.bin_path, "--git-dir=#{path_to_repo}", 'rev-list', newrev, '--not', base_rev, '--objects']
irb(main):008:2>     Gitlab::Popen.popen(args, nil, Gitlab::Git::Env.all.stringify_keys)
irb(main):009:2>   end
irb(main):010:1> end
       user     system      total        real
rev_list:   0.100000   0.040000   0.840000 (  2.102528)
=> [#<Benchmark::Tms:0x00007f5478539318 @label="rev_list: ", @real=2.102527630981058, @cstime=0.15999999999999992, @cutime=0.54, @stime=0.03999999999999915, @utime=0.09999999999999787, @total=0.839999999999997>]

the second one fails on require, though:

irb(main):011:0> require './test_blob_fetch.rb'
LoadError: cannot load such file -- ./test_blob_fetch.rb

assigned to @ilyaf

changed milestone to %WoW ending 2017-09-26

added requires production access unblocks others unscheduled labels

I put the snippet to server locally, here's the second query:

irb(main):001:0> require '/tmp/test_blob_fetch.rb'
=> true
irb(main):002:0> project = Project.find_by_full_path('jamedjo/engine')
=> #<Project id:4202644 jamedjo/engine>
irb(main):003:0>
irb(main):004:0* new_objects = ["4b67e82213a2ce0f8b4ed1d65bef48ef8b8be1f3",
irb(main):005:1* "21d117dd22a6fc1e39e1a7d82184359fb40af3f3",
irb(main):006:1* "56b13c63a5aaca766831bafa942ea9fcaf9f7d9c",
irb(main):007:1* "df780cc64ca73b477816f2bc6ed377f70d686e44",
irb(main):008:1* "76360011149eb1c1bcdacb359e081fb959d8add1",
irb(main):009:1* "31662eda42accab19789d2e04a9d08217fa76b4f",
irb(main):010:1* "49914c0b0f1461543263acbf7082a4f9e0387282",
irb(main):011:1* "54c5f340c18cc0494fc0983e152b78e9c0cddbf6",
irb(main):012:1* "5ae08eefdcdbdd821ce32082589879348ec6dbc7",
irb(main):013:1* "d6a9a74d40c3f380a69096a82a849454c35ee0d9",
irb(main):014:1* "24a8e87939aa53cdd833f6be7610cb4972e063ad",
irb(main):015:1* "ce5363de5916038291484584fc935f58f765b208",
irb(main):016:1* "b13f643410b7c25a3269992163b0fbbbdcc086a6",
irb(main):017:1* "738d6c095a76859f99f6852f36bb2ea180027f9e",
irb(main):018:1* "f5e81fc0276f53e256959224b0a59aeba7e12708",
irb(main):019:1* "93d0e703e013922701e64e5e85e0c7d575377fe8",
irb(main):020:1* "278e95c4c05407c5b27d0b5dbbed5351b2ceb545",
irb(main):021:1* "cc7fcaa135d01df224b301d6a5e4de3388d349c9",
irb(main):022:1* "8b19dd8432ee674a98302a1ae4a3269558cd1fbd",
irb(main):023:1* "bf8312bfb54045f62bf8905fbf61cc2fdabab60a",
irb(main):024:1* "8e665c70a2ffb1e1bc17cf619d8f6bcefe575c4b",
irb(main):025:1* "4939c9a12f27552f987df6212c463559e8046e51",
irb(main):026:1* "f93e3a1a1525fb5b91020da86e44810c87a2d7bc",
irb(main):027:1* "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"]
=> ["4b67e82213a2ce0f8b4ed1d65bef48ef8b8be1f3", "21d117dd22a6fc1e39e1a7d82184359fb40af3f3", "56b13c63a5aaca766831bafa942ea9fcaf9f7d9c", "df780cc64ca73b477816f2bc6ed377f70d686e44", "76360011149eb1c1bcdacb359e081fb959d8add1", "31662eda42accab19789d2e04a9d08217fa76b4f", "49914c0b0f1461543263acbf7082a4f9e0387282", "54c5f340c18cc0494fc0983e152b78e9c0cddbf6", "5ae08eefdcdbdd821ce32082589879348ec6dbc7", "d6a9a74d40c3f380a69096a82a849454c35ee0d9", "24a8e87939aa53cdd833f6be7610cb4972e063ad", "ce5363de5916038291484584fc935f58f765b208", "b13f643410b7c25a3269992163b0fbbbdcc086a6", "738d6c095a76859f99f6852f36bb2ea180027f9e", "f5e81fc0276f53e256959224b0a59aeba7e12708", "93d0e703e013922701e64e5e85e0c7d575377fe8", "278e95c4c05407c5b27d0b5dbbed5351b2ceb545", "cc7fcaa135d01df224b301d6a5e4de3388d349c9", "8b19dd8432ee674a98302a1ae4a3269558cd1fbd", "bf8312bfb54045f62bf8905fbf61cc2fdabab60a", "8e665c70a2ffb1e1bc17cf619d8f6bcefe575c4b", "4939c9a12f27552f987df6212c463559e8046e51", "f93e3a1a1525fb5b91020da86e44810c87a2d7bc", "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"]
irb(main):028:0>
irb(main):029:0* Benchmark.bm do |b|
irb(main):030:1*   b.report('blob_fetch: ') do
irb(main):031:2*     TestBlobFetch.new_lfs_blobs(project, new_objects)
irb(main):032:2>   end
irb(main):033:1> end
       user     system      total        real
blob_fetch:   0.050000   0.010000   0.060000 (  0.130903)
=> [#<Benchmark::Tms:0x00007f12ac4531f0 @label="blob_fetch: ", @real=0.13090256496798247, @cstime=0.0, @cutime=0.0, @stime=0.009999999999999787, @utime=0.05000000000000071, @total=0.0600000000000005>]

Locally with the same project the second test was almost instant:

[52] pry(main)> Benchmark.bm do |b|
[52] pry(main)*   b.report('blob_fetch: ') do  
[52] pry(main)*     TestBlobFetch.new_lfs_blobs(project, new_objects)
[52] pry(main)*   end    
[52] pry(main)* end  
       user     system      total        real
blob_fetch:   0.000000   0.000000   0.000000 (  0.000006)
=> [#<Benchmark::Tms:0x007f9456766b68  @cstime=0.0,  @cutime=0.0,  @label="blob_fetch: ",  @real=6.4889900386333466e-06,  @stime=0.0,  @total=0.0,  @utime=0.0>]

For the first test I ran it locally with a slightly different set of commits:

[24] pry(main)> Benchmark.bm do |b|
[24] pry(main)*   b.report('rev_list: ') do  
[24] pry(main)*     project = Project.find_by_full_path('root/gitlab-ce-local')    
[24] pry(main)*     path_to_repo = project.repository.path_to_repo    
[24] pry(main)*     newrev = 'master'    
[24] pry(main)*     base_rev = 'master~1000'    
[24] pry(main)*     args = [Gitlab.config.git.bin_path, "--git-dir=#{path_to_repo}", 'rev-list', newrev, '--not', base_rev, '--objects']    
[24] pry(main)*     Gitlab::Popen.popen(args, nil, Gitlab::Git::Env.all.stringify_keys)    
[24] pry(main)*   end    
[24] pry(main)* end  
       user     system      total        real
rev_list:
  0.010000   0.010000   0.450000 (  0.456111)
=> [#<Benchmark::Tms:0x007f944dd39fd0  @cstime=0.010000000000000009,  @cutime=0.41999999999999993,  @label="rev_list: ",  @real=0.45611068699508905,  @stime=0.010000000000000231,  @total=0.4500000000000004,  @utime=0.010000000000000231>]

As discussed in slack, we may have to run few more queries, so I'm not closing this issue now. Feel free to update or close it when you know for sure, James.

changed milestone to %WoW ending 2017-10-03

@jamedjo Can this be closed or you expect more queries?

I'll need to do a longer running test for new_lfs_blobs to get more accurate timings. Aiming to get you something to run today

changed milestone to %WoW