From 63e03dada7e754a92ca088c683f4189424ab34b1 Mon Sep 17 00:00:00 2001
From: Lin Jen-Shin <godfat@godfat.org>
Date: Wed, 21 Sep 2016 16:12:32 +0800
Subject: [PATCH] Make various trace methods take last_lines argument:
So that we could read last few lines rather than read
the entire file which could be huge.
---
app/controllers/projects/builds_controller.rb | 4 +-
app/models/ci/build.rb | 20 ++++----
lib/gitlab/ci/trace_reader.rb | 49 +++++++++++++++++++
spec/lib/gitlab/ci/trace_reader_spec.rb | 40 +++++++++++++++
4 files changed, 103 insertions(+), 10 deletions(-)
create mode 100644 lib/gitlab/ci/trace_reader.rb
create mode 100644 spec/lib/gitlab/ci/trace_reader_spec.rb
diff --git a/app/controllers/projects/builds_controller.rb b/app/controllers/projects/builds_controller.rb
index f13fb1df373..6069e620ba2 100644
--- a/app/controllers/projects/builds_controller.rb
+++ b/app/controllers/projects/builds_controller.rb
@@ -43,7 +43,9 @@ class Projects::BuildsController < Projects::ApplicationController
def trace
respond_to do |format|
format.json do
- render json: @build.trace_with_state(params[:state].presence).merge!(id: @build.id, status: @build.status)
+ state = params[:state].presence
+ render json: @build.trace_with_state(state: state).
+ merge!(id: @build.id, status: @build.status)
end
end
end
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index f9aa1984c1f..13f101cbecb 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -128,13 +128,14 @@ module Ci
latest_builds.where('stage_idx < ?', stage_idx)
end
- def trace_html
- trace_with_state[:html] || ''
+ def trace_html(**args)
+ trace_with_state(**args)[:html] || ''
end
- def trace_with_state(state = nil)
- if trace.present?
- Ci::Ansi2html.convert(trace, state)
+ def trace_with_state(state: nil, last_lines: nil)
+ trace_ansi = trace(last_lines)
+ if trace_ansi.present?
+ Ci::Ansi2html.convert(trace_ansi, state)
else
{}
end
@@ -220,9 +221,10 @@ module Ci
raw_trace.present?
end
- def raw_trace
+ def raw_trace(last_lines: nil)
if File.exist?(trace_file_path)
- File.read(trace_file_path)
+ Gitlab::Ci::TraceReader.new(trace_file_path).
+ read(last_lines: last_lines)
else
# backward compatibility
read_attribute :trace
@@ -237,8 +239,8 @@ module Ci
project.ci_id && File.exist?(old_path_to_trace)
end
- def trace
- result = raw_trace
+ def trace(last_lines: nil)
+ result = raw_trace(last_lines)
if project && result.present? && project.runners_token.present?
result.gsub(project.runners_token, 'xxxxxx')
else
diff --git a/lib/gitlab/ci/trace_reader.rb b/lib/gitlab/ci/trace_reader.rb
new file mode 100644
index 00000000000..37e51536e8f
--- /dev/null
+++ b/lib/gitlab/ci/trace_reader.rb
@@ -0,0 +1,49 @@
+module Gitlab
+ module Ci
+ # This was inspired from: http://stackoverflow.com/a/10219411/1520132
+ class TraceReader
+ BUFFER_SIZE = 4096
+
+ attr_accessor :path, :buffer_size
+
+ def initialize(new_path, buffer_size: BUFFER_SIZE)
+ self.path = new_path
+ self.buffer_size = Integer(buffer_size)
+ end
+
+ def read(last_lines: nil)
+ if last_lines
+ read_last_lines(last_lines)
+ else
+ File.read(path)
+ end
+ end
+
+ def read_last_lines(max_lines)
+ File.open(path) do |file|
+ chunks = []
+ pos = lines = 0
+ max = file.size
+
+ # We want an extra line to make sure fist line has full contents
+ while lines <= max_lines && pos < max
+ pos += buffer_size
+
+ buf = if pos <= max
+ file.seek(-pos, IO::SEEK_END)
+ file.read(buffer_size)
+ else # Reached the head, read only left
+ file.seek(0)
+ file.read(buffer_size - (pos - max))
+ end
+
+ lines += buf.count("\n")
+ chunks.unshift(buf)
+ end
+
+ chunks.join.lines.last(max_lines).join
+ end
+ end
+ end
+ end
+end
diff --git a/spec/lib/gitlab/ci/trace_reader_spec.rb b/spec/lib/gitlab/ci/trace_reader_spec.rb
new file mode 100644
index 00000000000..f06d78694d6
--- /dev/null
+++ b/spec/lib/gitlab/ci/trace_reader_spec.rb
@@ -0,0 +1,40 @@
+require 'spec_helper'
+
+describe Gitlab::Ci::TraceReader do
+ let(:path) { __FILE__ }
+ let(:lines) { File.readlines(path) }
+ let(:bytesize) { lines.sum(&:bytesize) }
+
+ it 'returns last few lines' do
+ 10.times do
+ subject = build_subject
+ last_lines = random_lines
+
+ expected = lines.last(last_lines).join
+
+ expect(subject.read(last_lines: last_lines)).to eq(expected)
+ end
+ end
+
+ it 'returns everything if trying to get too many lines' do
+ expect(build_subject.read(last_lines: lines.size * 2)).to eq(lines.join)
+ end
+
+ it 'raises an error if not passing an integer for last_lines' do
+ expect do
+ build_subject.read(last_lines: lines)
+ end.to raise_error(ArgumentError)
+ end
+
+ def random_lines
+ Random.rand(lines.size) + 1
+ end
+
+ def random_buffer
+ Random.rand(bytesize) + 1
+ end
+
+ def build_subject
+ described_class.new(__FILE__, buffer_size: random_buffer)
+ end
+end
--
GitLab