Remove test gitaly when older than version file

scripts/gitaly-test-build

 #!/usr/bin/env ruby
+require 'fileutils'
+
 # This script assumes tmp/tests/gitaly already contains the correct
 # Gitaly version. We just have to compile it and run its 'bundle
 # install'. We have this separate script for that because weird things
@@ -7,4 +9,11 @@
 # called 'bundle install' using a different Gemfile, as happens with
 # gitlab-ce and gitaly.
-abort 'gitaly build failed' unless system('make', chdir: 'tmp/tests/gitaly')
+dir = 'tmp/tests/gitaly'
+
+abort 'gitaly build failed' unless system('make', chdir: dir)
+
+# Make the 'gitaly' executable look newer than 'GITALY_SERVER_VERSION'.
+# Without this a gitaly executable created in the setup-test-env job
+# will look stale compared to GITALY_SERVER_VERSION.
+FileUtils.touch(File.join(dir, 'gitaly'), mtime: Time.now + (1 << 24))

spec/support/test_env.rb

@@ -133,6 +133,12 @@ module TestEnv
   def setup_gitaly
     socket_path = Gitlab::GitalyClient.address('default').sub(/\Aunix:/, '')
     gitaly_dir = File.dirname(socket_path)
+
+    if gitaly_dir_stale?(gitaly_dir)
+      puts "rm -rf #{gitaly_dir}"
+      FileUtils.rm_rf(gitaly_dir) 
+    end
+
     gitaly_needs_update = component_needs_update?(gitaly_dir,
       Gitlab::GitalyClient.expected_server_version)
@@ -143,6 +149,11 @@ module TestEnv
     start_gitaly(gitaly_dir)
   end
+  def gitaly_dir_stale?(dir)
+    gitaly_executable = File.join(dir, 'gitaly')
+    !File.exist?(gitaly_executable) || (File.mtime(gitaly_executable) < File.mtime(Rails.root.join('GITALY_SERVER_VERSION')))
+  end
+
   def start_gitaly(gitaly_dir)
     if ENV['CI'].present?
       # Gitaly has been spawned outside this process already