Geo testbed still attempting to sync projects on a non-existent shard
I see these in the sync log messages:
{"severity":"ERROR","time":"2017-10-08T13:13:37.573Z","class":"Geo::WikiSyncService","project_id":517975,"project_path":"ilstudio/evrosibtrade-layout","message":"Error syncing wiki repository","error":"error: cannot open /var/opt/gitlab/git-data-doesnt-exist/repositories/namespace/some-project.wiki.git/FETCH_HEAD: Read-only file system\n"}
irb(main):001:0> p = Project.find(517975)
=> #<Project id:517975 namespace/some-project>
irb(main):002:0> p.repository.path
=> "/var/opt/gitlab/git-data-doesnt-exist/repositories/namespace/some-project.git"
irb(main):003:0> p.repository.storage
=> "nfs-file04"
Here are the stats from the /-/metrics
endpoint:
curl http://localhost:8080/-/metrics
# TYPE db_ping_timeout gauge
db_ping_timeout 0
# TYPE db_ping_success gauge
db_ping_success 1
# TYPE db_ping_latency_seconds gauge
db_ping_latency_seconds 0.0007195472717285156
# TYPE redis_ping_timeout gauge
redis_ping_timeout 0
# TYPE redis_ping_success gauge
redis_ping_success 1
# TYPE redis_ping_latency_seconds gauge
redis_ping_latency_seconds 0.001233816146850586
# TYPE redis_cache_ping_timeout gauge
redis_cache_ping_timeout 0
# TYPE redis_cache_ping_success gauge
redis_cache_ping_success 1
# TYPE redis_cache_ping_latency_seconds gauge
redis_cache_ping_latency_seconds 0.00025582313537597656
# TYPE redis_queues_ping_timeout gauge
redis_queues_ping_timeout 0
# TYPE redis_queues_ping_success gauge
redis_queues_ping_success 1
# TYPE redis_queues_ping_latency_seconds gauge
redis_queues_ping_latency_seconds 0.0005891323089599609
# TYPE redis_shared_state_ping_timeout gauge
redis_shared_state_ping_timeout 0
# TYPE redis_shared_state_ping_success gauge
redis_shared_state_ping_success 1
# TYPE redis_shared_state_ping_latency_seconds gauge
redis_shared_state_ping_latency_seconds 0.00029158592224121094
# TYPE filesystem_access_latency_seconds gauge
filesystem_access_latency_seconds{shard="default"} 0.0064144134521484375
# TYPE filesystem_accessible gauge
filesystem_accessible{shard="default"} 1
# TYPE filesystem_write_latency_seconds gauge
filesystem_write_latency_seconds{shard="default"} 0.01152181625366211
# TYPE filesystem_writable gauge
filesystem_writable{shard="default"} 1
# TYPE filesystem_read_latency_seconds gauge
filesystem_read_latency_seconds{shard="default"} 0.007086277008056641
# TYPE filesystem_readable gauge
filesystem_readable{shard="default"} 1
# TYPE filesystem_circuitbreaker_latency_seconds gauge
filesystem_circuitbreaker_latency_seconds{shard="default"} 2.0157339572906494
# TYPE filesystem_circuitbreaker gauge
filesystem_circuitbreaker{shard="default"} 1
filesystem_access_latency_seconds{shard="nfs-file01"} 0.007112979888916016
filesystem_accessible{shard="nfs-file01"} 1
filesystem_write_latency_seconds{shard="nfs-file01"} 0.008136987686157227
filesystem_writable{shard="nfs-file01"} 0
filesystem_read_latency_seconds{shard="nfs-file01"} 0.0036199092864990234
filesystem_readable{shard="nfs-file01"} 0
filesystem_circuitbreaker_latency_seconds{shard="nfs-file01"} 0.7329363822937012
filesystem_circuitbreaker{shard="nfs-file01"} 1
filesystem_access_latency_seconds{shard="nfs-file02"} 0.004650592803955078
filesystem_accessible{shard="nfs-file02"} 1
filesystem_write_latency_seconds{shard="nfs-file02"} 0.007066488265991211
filesystem_writable{shard="nfs-file02"} 0
filesystem_read_latency_seconds{shard="nfs-file02"} 0.002657175064086914
filesystem_readable{shard="nfs-file02"} 0
filesystem_circuitbreaker_latency_seconds{shard="nfs-file02"} 0.8388466835021973
filesystem_circuitbreaker{shard="nfs-file02"} 1
filesystem_access_latency_seconds{shard="nfs-file03"} 0.006426334381103516
filesystem_accessible{shard="nfs-file03"} 1
filesystem_write_latency_seconds{shard="nfs-file03"} 0.0049474239349365234
filesystem_writable{shard="nfs-file03"} 0
filesystem_read_latency_seconds{shard="nfs-file03"} 0.0047457218170166016
filesystem_readable{shard="nfs-file03"} 0
filesystem_circuitbreaker_latency_seconds{shard="nfs-file03"} 0.6728601455688477
filesystem_circuitbreaker{shard="nfs-file03"} 1
filesystem_access_latency_seconds{shard="nfs-file04"} 0.0062389373779296875
filesystem_accessible{shard="nfs-file04"} 1
filesystem_write_latency_seconds{shard="nfs-file04"} 0.004507780075073242
filesystem_writable{shard="nfs-file04"} 0
filesystem_read_latency_seconds{shard="nfs-file04"} 0.004731178283691406
filesystem_readable{shard="nfs-file04"} 0
filesystem_circuitbreaker_latency_seconds{shard="nfs-file04"} 0.8704860210418701
filesystem_circuitbreaker{shard="nfs-file04"} 1
filesystem_access_latency_seconds{shard="nfs-file05"} 0.004483222961425781
filesystem_accessible{shard="nfs-file05"} 1
filesystem_write_latency_seconds{shard="nfs-file05"} 0.0027244091033935547
filesystem_writable{shard="nfs-file05"} 0
filesystem_read_latency_seconds{shard="nfs-file05"} 0.0028748512268066406
filesystem_readable{shard="nfs-file05"} 0
filesystem_circuitbreaker_latency_seconds{shard="nfs-file05"} 0.8405416011810303
filesystem_circuitbreaker{shard="nfs-file05"} 1
filesystem_access_latency_seconds{shard="nfs-file06"} 0.0044405460357666016
filesystem_accessible{shard="nfs-file06"} 1
filesystem_write_latency_seconds{shard="nfs-file06"} 0.005105495452880859
filesystem_writable{shard="nfs-file06"} 0
filesystem_read_latency_seconds{shard="nfs-file06"} 0.005211591720581055
filesystem_readable{shard="nfs-file06"} 0
filesystem_circuitbreaker_latency_seconds{shard="nfs-file06"} 0.7428178787231445
filesystem_circuitbreaker{shard="nfs-file06"} 1
filesystem_access_latency_seconds{shard="nfs-file07"} 0.004784345626831055
filesystem_accessible{shard="nfs-file07"} 1
filesystem_write_latency_seconds{shard="nfs-file07"} 0.0039365291595458984
filesystem_writable{shard="nfs-file07"} 0
filesystem_read_latency_seconds{shard="nfs-file07"} 0.0037963390350341797
filesystem_readable{shard="nfs-file07"} 0
filesystem_circuitbreaker_latency_seconds{shard="nfs-file07"} 0.8251218795776367
filesystem_circuitbreaker{shard="nfs-file07"} 1
filesystem_access_latency_seconds{shard="nfs-file08"} 0.006285429000854492
filesystem_accessible{shard="nfs-file08"} 1
filesystem_write_latency_seconds{shard="nfs-file08"} 0.005956411361694336
filesystem_writable{shard="nfs-file08"} 1
filesystem_read_latency_seconds{shard="nfs-file08"} 0.00606536865234375
filesystem_readable{shard="nfs-file08"} 1
filesystem_circuitbreaker_latency_seconds{shard="nfs-file08"} 0.763981819152832
filesystem_circuitbreaker{shard="nfs-file08"} 1
filesystem_access_latency_seconds{shard="nfs-file09"} 0.004219770431518555
filesystem_accessible{shard="nfs-file09"} 1
filesystem_write_latency_seconds{shard="nfs-file09"} 0.0028460025787353516
filesystem_writable{shard="nfs-file09"} 0
filesystem_read_latency_seconds{shard="nfs-file09"} 0.0031037330627441406
filesystem_readable{shard="nfs-file09"} 0
filesystem_circuitbreaker_latency_seconds{shard="nfs-file09"} 0.8361780643463135
filesystem_circuitbreaker{shard="nfs-file09"} 1
filesystem_access_latency_seconds{shard="nfs-file10"} 0.004065513610839844
filesystem_accessible{shard="nfs-file10"} 1
filesystem_write_latency_seconds{shard="nfs-file10"} 0.0032258033752441406
filesystem_writable{shard="nfs-file10"} 0
filesystem_read_latency_seconds{shard="nfs-file10"} 0.002960681915283203
filesystem_readable{shard="nfs-file10"} 0
filesystem_circuitbreaker_latency_seconds{shard="nfs-file10"} 0.7646541595458984
filesystem_circuitbreaker{shard="nfs-file10"} 1
filesystem_access_latency_seconds{shard="nfs-file11"} 0.0060236454010009766
filesystem_accessible{shard="nfs-file11"} 1
filesystem_write_latency_seconds{shard="nfs-file11"} 0.0027778148651123047
filesystem_writable{shard="nfs-file11"} 0
filesystem_read_latency_seconds{shard="nfs-file11"} 0.005401134490966797
filesystem_readable{shard="nfs-file11"} 0
filesystem_circuitbreaker_latency_seconds{shard="nfs-file11"} 0.8191945552825928
filesystem_circuitbreaker{shard="nfs-file11"} 1
filesystem_access_latency_seconds{shard="nfs-file12"} 0.006497621536254883
filesystem_accessible{shard="nfs-file12"} 1
filesystem_write_latency_seconds{shard="nfs-file12"} 0.0052030086517333984
filesystem_writable{shard="nfs-file12"} 0
filesystem_read_latency_seconds{shard="nfs-file12"} 0.0025615692138671875
filesystem_readable{shard="nfs-file12"} 0
filesystem_circuitbreaker_latency_seconds{shard="nfs-file12"} 0.765749454498291
filesystem_circuitbreaker{shard="nfs-file12"} 1
# HELP http_requests_total Multiprocess metric
# TYPE http_requests_total counter
http_requests_total{method="get"} 61.0
# HELP unicorn_active_connections Multiprocess metric
# TYPE unicorn_active_connections gauge
unicorn_active_connections{type="tcp",address="127.0.0.1:8080"} 1.0
unicorn_active_connections{type="unix",address="/var/opt/gitlab/gitlab-rails/sockets/gitlab.socket"} 0.0
# HELP unicorn_queued_connections Multiprocess metric
# TYPE unicorn_queued_connections gauge
unicorn_queued_connections{type="tcp",address="127.0.0.1:8080"} 0.0
unicorn_queued_connections{type="unix",address="/var/opt/gitlab/gitlab-rails/sockets/gitlab.socket"} 0.0
# HELP http_request_duration_seconds Multiprocess metric
# TYPE http_request_duration_seconds histogram
http_request_duration_seconds_sum{method="get",status="200"} 168.76109409332275
http_request_duration_seconds_count{method="get",status="200"} 46.0
http_request_duration_seconds_bucket{method="get",status="200",le="0.05"} 0.0
http_request_duration_seconds_bucket{method="get",status="200",le="0.1"} 1.0
http_request_duration_seconds_bucket{method="get",status="200",le="0.25"} 1.0
http_request_duration_seconds_bucket{method="get",status="200",le="0.5"} 1.0
http_request_duration_seconds_bucket{method="get",status="200",le="0.7"} 1.0
http_request_duration_seconds_bucket{method="get",status="200",le="1"} 1.0
http_request_duration_seconds_bucket{method="get",status="200",le="2.5"} 8.0
http_request_duration_seconds_bucket{method="get",status="200",le="5"} 39.0
http_request_duration_seconds_bucket{method="get",status="200",le="10"} 45.0
http_request_duration_seconds_bucket{method="get",status="200",le="25"} 46.0
http_request_duration_seconds_sum{method="get",status="304"} 73.52354216575623
http_request_duration_seconds_count{method="get",status="304"} 14.0
http_request_duration_seconds_bucket{method="get",status="304",le="0.05"} 0.0
http_request_duration_seconds_bucket{method="get",status="304",le="0.1"} 0.0
http_request_duration_seconds_bucket{method="get",status="304",le="0.25"} 0.0
http_request_duration_seconds_bucket{method="get",status="304",le="0.5"} 0.0
http_request_duration_seconds_bucket{method="get",status="304",le="0.7"} 0.0
http_request_duration_seconds_bucket{method="get",status="304",le="1"} 0.0
http_request_duration_seconds_bucket{method="get",status="304",le="2.5"} 1.0
http_request_duration_seconds_bucket{method="get",status="304",le="5"} 11.0
http_request_duration_seconds_bucket{method="get",status="304",le="10"} 13.0
http_request_duration_seconds_bucket{method="get",status="304",le="25"} 14.0