Unable to getback the repos from @hashed

Hi,
I upgraded my installation to 13.6.3 recently from 12.10.14 and I see my repos still in @hashed folder.

I tried “gitlab-rake gitlab:storage:rollback_to_legacy” . Yet the repos did not come back to legacy storage. I get the message “Enqueuing rollback of 4 projects in batches of 200. Done!”. However the repos do not move to legacy storage.

Please advice.

Thanks,

The log in sideqik/current is as follows

{“severity”:“INFO”,“time”:“2020-12-18T01:11:53.793Z”,“class”:“HashedStorage::RollbackerWorker”,“args”:[“3”,“4”],“retry”:3,“queue”:“hashed_storage:hashed_storage_rollbacker”,“version”:0,“queue_namespace”:“hashed_storage”,“jid”:“aa08b77fc68d5e435d614fce”,“created_at”:“2020-12-18T01:11:53.784Z”,“correlation_id”:“532645af2335eaa29555b7aaf24e0722”,“enqueued_at”:“2020-12-18T01:11:53.792Z”,“pid”:28997,“message”:“HashedStorage::RollbackerWorker JID-aa08b77fc68d5e435d614fce: start”,“job_status”:“start”,“scheduling_latency_s”:0.001344}
{“severity”:“INFO”,“time”:“2020-12-18T01:11:53.810Z”,“class”:“HashedStorage::RollbackerWorker”,“args”:[“3”,“4”],“retry”:3,“queue”:“hashed_storage:hashed_storage_rollbacker”,“version”:0,“queue_namespace”:“hashed_storage”,“jid”:“aa08b77fc68d5e435d614fce”,“created_at”:“2020-12-18T01:11:53.784Z”,“correlation_id”:“532645af2335eaa29555b7aaf24e0722”,“enqueued_at”:“2020-12-18T01:11:53.792Z”,“pid”:28997,“message”:“HashedStorage::RollbackerWorker JID-aa08b77fc68d5e435d614fce: done: 0.016395 sec”,“job_status”:“done”,“scheduling_latency_s”:0.001344,“redis_calls”:7,“redis_duration_s”:0.0019,“redis_read_bytes”:234,“redis_write_bytes”:1438,“redis_cache_calls”:1,“redis_cache_duration_s”:0.000552,“redis_cache_read_bytes”:203,“redis_cache_write_bytes”:110,“redis_queues_calls”:4,“redis_queues_duration_s”:0.000874,“redis_queues_read_bytes”:24,“redis_queues_write_bytes”:1234,“redis_shared_state_calls”:2,“redis_shared_state_duration_s”:0.000474,“redis_shared_state_read_bytes”:7,“redis_shared_state_write_bytes”:94,“db_count”:2,“db_write_count”:0,“db_cached_count”:0,“duration_s”:0.016395,“cpu_s”:0.014954,“completed_at”:“2020-12-18T01:11:53.810Z”,“db_duration_s”:0.001445}

{“severity”:“WARN”,“time”:“2020-12-18T01:24:03.958Z”,“error_class”:“Projects::HashedStorage::RepositoryInUseError”,“error_message”:“Target repository ‘@hashed/4b/22/4b227777d4dd1fc61c6f884f48641d02b4d121d3fd328cb08b5531fcacdabf8a’ cannot be made read-only: Git transfer in progress”,“context”:“Job raised exception”,“class”:“HashedStorage::ProjectRollbackWorker”,“args”:[“4”],“retry”:3,“queue”:“hashed_storage:hashed_storage_project_rollback”,“version”:0,“queue_namespace”:“hashed_storage”,“jid”:“775b7df8bdf91d14c4e3d8b2”,“created_at”:“2020-12-18T01:11:53.807Z”,“meta.caller_id”:“HashedStorage::RollbackerWorker”,“meta.feature_category”:“source_code_management”,“correlation_id”:“e9dfedad2aad94f469080a0135f9974d”,“enqueued_at”:“2020-12-18T01:24:03.935Z”,“failed_at”:“2020-12-18T01:21:57.042Z”,“retry_count”:2,“retried_at”:“2020-12-18T01:23:16.156Z”,“db_count”:3,“db_write_count”:2,“db_cached_count”:0,“error_backtrace”:[“app/services/projects/hashed_storage/base_repository_service.rb:87:in rescue in try_to_set_repository_read_only!'","app/services/projects/hashed_storage/base_repository_service.rb:81:in try_to_set_repository_read_only!’”,“app/services/projects/hashed_storage/rollback_repository_service.rb:7:in execute'","app/services/projects/hashed_storage/rollback_service.rb:22:in execute’”,“app/workers/hashed_storage/project_rollback_worker.rb:22:in block in perform'","app/services/concerns/exclusive_lease_guard.rb:29:in try_obtain_lease’”,“app/workers/hashed_storage/project_rollback_worker.rb:16:in perform'","lib/gitlab/metrics/sidekiq_middleware.rb:18:in block in call’”,“lib/gitlab/metrics/transaction.rb:61:in run'","lib/gitlab/metrics/sidekiq_middleware.rb:18:in call’”,“lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb:16:in perform'","lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb:40:in perform’”,“lib/gitlab/sidekiq_middleware/duplicate_jobs/server.rb:8:in call'","lib/gitlab/sidekiq_middleware/worker_context.rb:9:in wrap_in_optional_context’”,“lib/gitlab/sidekiq_middleware/worker_context/server.rb:17:in block in call'","lib/gitlab/application_context.rb:54:in block in use’”,“lib/gitlab/application_context.rb:54:in use'","lib/gitlab/application_context.rb:21:in with_context’”,“lib/gitlab/sidekiq_middleware/worker_context/server.rb:15:in call'","lib/gitlab/sidekiq_status/server_middleware.rb:7:in call’”,“lib/gitlab/sidekiq_versioning/middleware.rb:9:in call'","lib/gitlab/sidekiq_middleware/admin_mode/server.rb:8:in call’”,“lib/gitlab/sidekiq_middleware/instrumentation_logger.rb:7:in call'","lib/gitlab/sidekiq_middleware/batch_loader.rb:7:in call’”,“lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb:7:in call'","lib/gitlab/sidekiq_middleware/request_store_middleware.rb:10:in block in call’”,“lib/gitlab/with_request_store.rb:17:in enabling_request_store'","lib/gitlab/with_request_store.rb:10:in with_request_store’”,“lib/gitlab/sidekiq_middleware/request_store_middleware.rb:9:in call'","lib/gitlab/sidekiq_middleware/server_metrics.rb:35:in call’”,“lib/gitlab/sidekiq_middleware/monitor.rb:8:in block in call'","lib/gitlab/sidekiq_daemon/monitor.rb:49:in within_job’”,“lib/gitlab/sidekiq_middleware/monitor.rb:7:in call'","lib/gitlab/sidekiq_logging/structured_logger.rb:18:in call’”]}

I have one project that is still in @hashed and I want it rolledback to legacy storage.

Please help.