Skip to content
Snippets Groups Projects
Commit a9c868d1 authored by Michael Kozono's avatar Michael Kozono
Browse files

Bulk insert uploads

parent 908aacdd
No related branches found
No related tags found
No related merge requests found
Loading
Loading
@@ -56,7 +56,8 @@ module Gitlab
uploader: uploader,
model_type: model_type,
model_id: model_id,
size: file_size
size: file_size,
checksum: checksum
}
end
 
Loading
Loading
@@ -90,10 +91,13 @@ module Gitlab
end
 
def file_size
absolute_path = File.join(CarrierWave.root, path)
File.size(absolute_path)
end
 
def checksum
Digest::SHA256.file(absolute_path).hexdigest
end
# Not including a leading slash
def path_relative_to_upload_dir
base = %r{\A#{Regexp.escape(Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR)}/}
Loading
Loading
@@ -120,55 +124,14 @@ module Gitlab
project = Project.find_by_full_path(full_path)
project.id.to_s
end
end
# Copy-pasted class for less fragile migration
class Upload < ActiveRecord::Base
self.table_name = 'uploads' # This is the only line different from copy-paste
# Upper limit for foreground checksum processing
CHECKSUM_THRESHOLD = 100.megabytes
belongs_to :model, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations
before_save :calculate_checksum, if: :foreground_checksum?
after_commit :schedule_checksum, unless: :foreground_checksum?
 
def absolute_path
return path unless relative_path?
uploader_class.absolute_path(self)
end
def calculate_checksum
return unless exist?
self.checksum = Digest::SHA256.file(absolute_path).hexdigest
rescue StandardError
schedule_checksum
end
def exist?
File.exist?(absolute_path)
end
private
def foreground_checksum?
size <= CHECKSUM_THRESHOLD
end
def schedule_checksum
UploadChecksumWorker.perform_async(id)
end
def relative_path?
!path.start_with?('/')
File.join(CarrierWave.root, path)
end
end
 
def uploader_class
Object.const_get(uploader)
end
class Upload < ActiveRecord::Base
self.table_name = 'uploads'
end
 
def perform(start_id, end_id)
Loading
Loading
@@ -207,9 +170,11 @@ module Gitlab
end
 
def insert(files)
files.each do |file|
Upload.create!(file.to_h)
rows = files.map do |file|
file.to_h.merge(created_at: 'NOW()')
end
Gitlab::Database.bulk_insert('uploads', rows)
end
 
def drop_temp_table_if_finished
Loading
Loading
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment