Okay, I got the error today and I grepped the logs for the ID and found this:
"request_urgency": "low",
"target_duration_s": 5,
"redis_calls": 53,
"redis_duration_s": 0.128596,
"redis_read_bytes": 1981,
"redis_write_bytes": 7949,
"redis_cache_calls": 51,
"redis_cache_duration_s": 0.037397,
"redis_cache_read_bytes": 1798,
"redis_cache_write_bytes": 6738,
"redis_shared_state_calls": 1,
"redis_shared_state_duration_s": 0.021431,
"redis_shared_state_write_bytes": 53,
"redis_sessions_calls": 1,
"redis_sessions_duration_s": 0.069768,
"redis_sessions_read_bytes": 183,
"redis_sessions_write_bytes": 1158,
"db_count": 57,
"db_write_count": 0,
"db_cached_count": 8,
"db_replica_count": 0,
"db_primary_count": 57,
"db_main_count": 57,
"db_main_replica_count": 0,
"db_replica_cached_count": 0,
"db_primary_cached_count": 8,
"db_main_cached_count": 8,
"db_main_replica_cached_count": 0,
"db_replica_wal_count": 0,
"db_primary_wal_count": 0,
"db_main_wal_count": 0,
"db_main_replica_wal_count": 0,
"db_replica_wal_cached_count": 0,
"db_primary_wal_cached_count": 0,
"db_main_wal_cached_count": 0,
"db_main_replica_wal_cached_count": 0,
"db_replica_duration_s": 0.0,
"db_primary_duration_s": 0.19,
"db_main_duration_s": 0.19,
"db_main_replica_duration_s": 0.0,
"cpu_s": 2.548205,
"mem_objects": 648314,
"mem_bytes": 108218366,
"mem_mallocs": 419364,
"mem_total_bytes": 134150926,
"pid": 3518858,
"worker_id": "puma_1",
"rate_limiting_gates": [],
"exception.class": "Rack::Timeout::RequestTimeoutException",
"exception.message": "Request ran for longer than 60000ms ",
"exception.backtrace": ["app/views/projects/show.html.haml:18", "app/controllers/application_controller.rb:142:in `render'", "app/controllers/application_controller.rb:582:in `block in allow_gitaly_ref_name_caching'", "lib/gitlab/gitaly_client.rb:323:in `allow_ref_name_caching'", "app/controllers/application_controller.rb:581:in `allow_gitaly_ref_name_caching'", "app/controllers/application_controller.rb:531:in `set_current_admin'", "lib/gitlab/session.rb:11:in `with_session'", "app/controllers/application_controller.rb:522:in `set_session_storage'", "lib/gitlab/i18n.rb:107:in `with_locale'", "lib/gitlab/i18n.rb:113:in `with_user_locale'", "app/controllers/application_controller.rb:516:in `set_locale'", "app/controllers/application_controller.rb:510:in `set_current_context'", "lib/gitlab/middleware/memory_report.rb:13:in `call'", "lib/gitlab/middleware/speedscope.rb:13:in `call'", "lib/gitlab/database/load_balancing/rack_middleware.rb:23:in `call'", "lib/gitlab/jira/middleware.rb:19:in `call'", "lib/gitlab/middleware/go.rb:20:in `call'", "lib/gitlab/etag_caching/middleware.rb:21:in `call'", "lib/gitlab/middleware/query_analyzer.rb:11:in `block in call'", "lib/gitlab/database/query_analyzer.rb:37:in `within'", "lib/gitlab/middleware/query_analyzer.rb:11:in `call'", "lib/gitlab/middleware/multipart.rb:173:in `call'", "lib/gitlab/middleware/read_only/controller.rb:50:in `call'", "lib/gitlab/middleware/read_only.rb:18:in `call'", "lib/gitlab/middleware/same_site_cookies.rb:27:in `call'", "lib/gitlab/middleware/handle_malformed_strings.rb:21:in `call'", "lib/gitlab/middleware/basic_health_check.rb:25:in `call'", "lib/gitlab/middleware/handle_ip_spoof_attack_error.rb:25:in `call'", "lib/gitlab/middleware/request_context.rb:21:in `call'", "lib/gitlab/middleware/webhook_recursion_detection.rb:15:in `call'", "config/initializers/fix_local_cache_middleware.rb:11:in `call'", "lib/gitlab/middleware/compressed_json.rb:26:in `call'", "lib/gitlab/middleware/rack_multipart_tempfile_factory.rb:19:in `call'", "lib/gitlab/middleware/sidekiq_web_static.rb:20:in `call'", "lib/gitlab/metrics/requests_rack_middleware.rb:77:in `call'", "lib/gitlab/middleware/release_env.rb:13:in `call'"],
"db_duration_s": 0.34026,
"view_duration_s": 0.0,
"duration_s": 34.30009
On refresh it worked, so for some reason my system is just hitting the 60 second timeout. Any way to increase it a bit?
@tnir does this help? What’s interesting is I would have expected the durations in the json to be greater than 60 but it seems duration_s is only 34.
I found this thread and I increased my CPUs to 4 within Proxmox LXC. I guess I’ll see if that helps.