Add untracked files to uploads

lib/gitlab/background_migration/populate_untracked_uploads.rb

@@ -4,27 +4,149 @@ module Gitlab
       class UnhashedUploadFile < ActiveRecord::Base
         self.table_name = 'unhashed_upload_files'
+        # Ends with /:random_hex/:filename
+        FILE_UPLOADER_PATH_PATTERN = /\/\h+\/[^\/]+\z/
+
+        # These regex patterns are tested against a relative path, relative to
+        # the upload directory.
+        # For convenience, if there exists a capture group in the pattern, then
+        # it indicates the model_id.
+        PATH_PATTERNS = [
+          {
+            pattern: /\A-\/system\/appearance\/logo\/(\d+)/,
+            uploader: 'AttachmentUploader',
+            model_type: 'Appearance',
+          },
+          {
+            pattern: /\A-\/system\/appearance\/header_logo\/(\d+)/,
+            uploader: 'AttachmentUploader',
+            model_type: 'Appearance',
+          },
+          {
+            pattern: /\A-\/system\/note\/attachment\/(\d+)/,
+            uploader: 'AttachmentUploader',
+            model_type: 'Note',
+          },
+          {
+            pattern: /\A-\/system\/user\/avatar\/(\d+)/,
+            uploader: 'AvatarUploader',
+            model_type: 'User',
+          },
+          {
+            pattern: /\A-\/system\/group\/avatar\/(\d+)/,
+            uploader: 'AvatarUploader',
+            model_type: 'Group',
+          },
+          {
+            pattern: /\A-\/system\/project\/avatar\/(\d+)/,
+            uploader: 'AvatarUploader',
+            model_type: 'Project',
+          },
+          {
+            pattern: FILE_UPLOADER_PATH_PATTERN,
+            uploader: 'FileUploader',
+            model_type: 'Project'
+          },
+        ]
+
         scope :untracked, -> { where(tracked: false) }
         def ensure_tracked!
-          # TODO
-          # unless unhashed_upload_file.in_uploads?
-          #   unhashed_upload_file.add_to_uploads
-          # end
-          #
-          # unhashed_upload_file.mark_as_tracked
+          return if persisted? && tracked?
+
+          unless in_uploads?
+            add_to_uploads
+          end
+
+          mark_as_tracked
         end
-        def model_id
-          # TODO
+        def in_uploads?
+          # Even though we are checking relative paths, path is enough to
+          # uniquely identify uploads. There is no ambiguity between
+          # FileUploader paths and other Uploader paths because we use the /-/
+          # separator kind of like an escape character. Project full_path will
+          # never conflict with an upload path starting with "uploads/-/".
+          Upload.exists?(path: upload_path)
         end
-        def model_type
-          # TODO
+        def add_to_uploads
+          Upload.create!(
+            path: upload_path,
+            uploader: uploader,
+            model_type: model_type,
+            model_id: model_id,
+            size: file_size
+          )
+        end
+
+        def mark_as_tracked
+          self.tracked = true
+          self.save!
+        end
+
+        def upload_path
+          # UnhashedUploadFile#path is absolute, but Upload#path depends on uploader
+          if uploader == 'FileUploader'
+            # Path relative to project directory in uploads
+            matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH_PATTERN)
+            matchd[0].sub(/\A\//, '') # remove leading slash
+          else
+            path_relative_to_carrierwave_root
+          end
         end
         def uploader
-          # TODO
+          PATH_PATTERNS.each do |path_pattern_map|
+            if path_relative_to_upload_dir.match(path_pattern_map[:pattern])
+              return path_pattern_map[:uploader]
+            end
+          end
+        end
+
+        def model_type
+          PATH_PATTERNS.each do |path_pattern_map|
+            if path_relative_to_upload_dir.match(path_pattern_map[:pattern])
+              return path_pattern_map[:model_type]
+            end
+          end
+        end
+
+        def model_id
+          PATH_PATTERNS.each do |path_pattern_map|
+            matchd = path_relative_to_upload_dir.match(path_pattern_map[:pattern])
+
+            # If something is captured (matchd[1] is not nil), it is a model_id
+            return matchd[1] if matchd && matchd[1]
+          end
+
+          # Only the FileUploader pattern will not match an ID
+          file_uploader_model_id
+        end
+
+        def file_size
+          File.size(path)
+        end
+
+        # Not including a leading slash
+        def path_relative_to_upload_dir
+          @path_relative_to_upload_dir ||= path.sub(/\A#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}\//, '')
+        end
+
+        # Not including a leading slash
+        def path_relative_to_carrierwave_root
+          "uploads/#{path_relative_to_upload_dir}"
+        end
+
+        private
+
+        def file_uploader_model_id
+          pattern_to_capture_full_path = /\A(.+)#{FILE_UPLOADER_PATH_PATTERN}/
+          matchd = path_relative_to_upload_dir.match(pattern_to_capture_full_path)
+          raise "Could not capture project full_path from a FileUploader path: \"#{path_relative_to_upload_dir}\"" unless matchd
+          full_path = matchd[1]
+          project = Project.find_by_full_path(full_path)
+          project.id.to_s
         end
       end

spec/migrations/track_untracked_uploads_spec.rb

@@ -2,6 +2,10 @@ require 'spec_helper'
 require Rails.root.join('db', 'post_migrate', '20171103140253_track_untracked_uploads')
 describe TrackUntrackedUploads, :migration, :sidekiq do
+  class UnhashedUploadFile < ActiveRecord::Base
+    self.table_name = 'unhashed_upload_files'
+  end
+
   matcher :be_scheduled_migration do
     match do |migration|
       BackgroundMigrationWorker.jobs.any? do |job|
@@ -30,10 +34,6 @@ describe TrackUntrackedUploads, :migration, :sidekiq do
   end
   it 'has a path field long enough for really long paths' do
-    class UnhashedUploadFile < ActiveRecord::Base
-      self.table_name = 'unhashed_upload_files'
-    end
-
     migrate!
     max_length_namespace_path = max_length_project_path = max_length_filename = 'a' * 255
@@ -57,7 +57,8 @@ describe TrackUntrackedUploads, :migration, :sidekiq do
       uploaded_file = fixture_file_upload(fixture)
       user1.update(avatar: uploaded_file)
       project1.update(avatar: uploaded_file)
-      UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload
+      upload_result = UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload
+      @project1_markdown_upload_path = upload_result[:url].sub(/\A\/uploads\//, '')
       appearance.update(logo: uploaded_file)
       # Untracked, by doing normal file upload then deleting records from DB
@@ -65,48 +66,62 @@ describe TrackUntrackedUploads, :migration, :sidekiq do
       user2.update(avatar: uploaded_file)
       user2.uploads.delete_all
       project2.update(avatar: uploaded_file)
-      UploadService.new(project2, uploaded_file, FileUploader).execute # Markdown upload
+      upload_result = UploadService.new(project2, uploaded_file, FileUploader).execute # Markdown upload
+      @project2_markdown_upload_path = upload_result[:url].sub(/\A\/uploads\//, '')
       project2.uploads.delete_all
       appearance.update(header_logo: uploaded_file)
       appearance.uploads.last.destroy
     end
-    it 'schedules background migrations' do
+    it 'tracks untracked migrations' do
       Sidekiq::Testing.inline! do
         migrate!
         # Tracked uploads still exist
-        expect(user1.uploads.first.attributes).to include({
-          "path" => "uploads/-/system/user/avatar/1/rails_sample.jpg",
+        expect(user1.reload.uploads.first.attributes).to include({
+          "path" => "uploads/-/system/user/avatar/#{user1.id}/rails_sample.jpg",
           "uploader" => "AvatarUploader"
         })
-        expect(project1.uploads.first.attributes).to include({
-          "path" => "uploads/-/system/project/avatar/1/rails_sample.jpg",
+        expect(project1.reload.uploads.first.attributes).to include({
+          "path" => "uploads/-/system/project/avatar/#{project1.id}/rails_sample.jpg",
           "uploader" => "AvatarUploader"
         })
-        expect(appearance.uploads.first.attributes).to include({
-          "path" => "uploads/-/system/appearance/logo/1/rails_sample.jpg",
+        expect(appearance.reload.uploads.first.attributes).to include({
+          "path" => "uploads/-/system/appearance/logo/#{appearance.id}/rails_sample.jpg",
           "uploader" => "AttachmentUploader"
         })
-        expect(project1.uploads.last.path).to match(/\w+\/rails_sample\.jpg/)
-        expect(project1.uploads.last.uploader).to eq('FileUploader')
+        expect(project1.uploads.last.attributes).to include({
+          "path" => @project1_markdown_upload_path,
+          "uploader" => "FileUploader"
+        })
         # Untracked uploads are now tracked
-        expect(user2.uploads.first.attributes).to include({
-          "path" => "uploads/-/system/user/avatar/2/rails_sample.jpg",
+        expect(user2.reload.uploads.first.attributes).to include({
+          "path" => "uploads/-/system/user/avatar/#{user2.id}/rails_sample.jpg",
           "uploader" => "AvatarUploader"
         })
-        expect(project2.uploads.first.attributes).to include({
-          "path" => "uploads/-/system/project/avatar/2/rails_sample.jpg",
+        expect(project2.reload.uploads.first.attributes).to include({
+          "path" => "uploads/-/system/project/avatar/#{project2.id}/rails_sample.jpg",
           "uploader" => "AvatarUploader"
         })
-        expect(appearance.uploads.count).to eq(2)
+        expect(appearance.reload.uploads.count).to eq(2)
         expect(appearance.uploads.last.attributes).to include({
-          "path" => "uploads/-/system/appearance/header_logo/1/rails_sample.jpg",
+          "path" => "uploads/-/system/appearance/header_logo/#{appearance.id}/rails_sample.jpg",
           "uploader" => "AttachmentUploader"
         })
-        expect(project2.uploads.last.path).to match(/\w+\/rails_sample\.jpg/)
-        expect(project2.uploads.last.uploader).to eq('FileUploader')
+        expect(project2.uploads.last.attributes).to include({
+          "path" => @project2_markdown_upload_path,
+          "uploader" => "FileUploader"
+        })
+      end
+    end
+
+    it 'all UnhashedUploadFile records are marked as tracked' do
+      Sidekiq::Testing.inline! do
+        migrate!
+
+        expect(UnhashedUploadFile.count).to eq(8)
+        expect(UnhashedUploadFile.count).to eq(UnhashedUploadFile.where(tracked: true).count)
       end
     end
   end