After upgrade to 15.11.2 - All webhook calls are now getting the error : Hook execution failed: Net::OpenTimeout
Does anyone know what might be causing this?
For extra info, we run over a proxy and the webhooks have to follow the no_proxy rules.
gitlab_rails['env'] = {
"http_proxy" => "http://internal-proxy.com:8080",
"https_proxy" => "http://internal-proxy.com:8080",
"no_proxy" => ".my-domain.com"
}
Error from exceptions.json
{
"severity": "ERROR",
"time": "2023-05-09T07:26:10.012Z",
"correlation_id": "01GZZPE66A110QNEEX51G08DM4",
"exception.class": "Net::OpenTimeout",
"exception.message": "Net::OpenTimeout",
"exception.backtrace": [
"lib/gitlab/net_http_adapter.rb:21:in `connect'",
"lib/gitlab/http.rb:55:in `perform_request'",
"app/services/web_hook_service.rb:123:in `make_request'",
"app/services/web_hook_service.rb:72:in `execute'",
"app/workers/web_hook_worker.rb:27:in `perform'",
"lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb:26:in `call'",
"lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb:16:in `perform'",
"lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb:44:in `perform'",
"lib/gitlab/sidekiq_middleware/duplicate_jobs/server.rb:8:in `call'",
"lib/gitlab/sidekiq_middleware/worker_context.rb:9:in `wrap_in_optional_context'",
"lib/gitlab/sidekiq_middleware/worker_context/server.rb:19:in `block in call'",
"lib/gitlab/application_context.rb:118:in `block in use'",
"lib/gitlab/application_context.rb:118:in `use'",
"lib/gitlab/application_context.rb:57:in `with_context'",
"lib/gitlab/sidekiq_middleware/worker_context/server.rb:17:in `call'",
"lib/gitlab/sidekiq_status/server_middleware.rb:7:in `call'",
"lib/gitlab/sidekiq_versioning/middleware.rb:9:in `call'",
"lib/gitlab/sidekiq_middleware/query_analyzer.rb:7:in `block in call'",
"lib/gitlab/database/query_analyzer.rb:37:in `within'",
"lib/gitlab/sidekiq_middleware/query_analyzer.rb:7:in `call'",
"lib/gitlab/sidekiq_middleware/admin_mode/server.rb:14:in `call'",
"lib/gitlab/sidekiq_middleware/instrumentation_logger.rb:9:in `call'",
"lib/gitlab/sidekiq_middleware/batch_loader.rb:7:in `call'",
"lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb:7:in `call'",
"lib/gitlab/sidekiq_middleware/request_store_middleware.rb:10:in `block in call'",
"lib/gitlab/with_request_store.rb:17:in `enabling_request_store'",
"lib/gitlab/with_request_store.rb:10:in `with_request_store'",
"lib/gitlab/sidekiq_middleware/request_store_middleware.rb:9:in `call'",
"lib/gitlab/sidekiq_middleware/server_metrics.rb:76:in `block in call'",
"lib/gitlab/sidekiq_middleware/server_metrics.rb:103:in `block in instrument'",
"lib/gitlab/metrics/background_transaction.rb:33:in `run'",
"lib/gitlab/sidekiq_middleware/server_metrics.rb:103:in `instrument'",
"lib/gitlab/sidekiq_middleware/server_metrics.rb:75:in `call'",
"lib/gitlab/sidekiq_middleware/monitor.rb:10:in `block in call'",
"lib/gitlab/sidekiq_daemon/monitor.rb:46:in `within_job'",
"lib/gitlab/sidekiq_middleware/monitor.rb:9:in `call'",
"lib/gitlab/sidekiq_middleware/size_limiter/server.rb:13:in `call'",
"lib/gitlab/sidekiq_logging/structured_logger.rb:21:in `call'"
],
"user.username": "redacted",
"tags.program": "sidekiq",
"tags.locale": "en",
"tags.feature_category": "integrations",
"tags.correlation_id": "01GZZPE66A110QNEEX51G08DM4",
"extra.sidekiq": {
"retry": 4,
"queue": "web_hook",
"version": 0,
"dead": false,
"args": [
"37",
"[FILTERED]",
"push_hooks",
"{}"
],
"class": "WebHookWorker",
"jid": "c3dc6726fb18e22600814f75",
"created_at": 1683617159.9390328,
"correlation_id": "01GZZPE66A110QNEEX51G08DM4",
"meta.caller_id": "PostReceive",
"meta.remote_ip": "127.0.0.1",
"meta.feature_category": "integrations",
"meta.user": "redacted",
"meta.user_id": 47,
"meta.project": "redacted/redacted",
"meta.root_namespace": "redacted",
"meta.client_id": "user/47",
"meta.root_caller_id": "POST /api/:version/internal/post_receive",
"meta.related_class": "ProjectHook",
"worker_data_consistency": "delayed",
"wal_locations": {},
"wal_location_source": "primary",
"idempotency_key": "resque:gitlab:duplicate:web_hook:7563dd7048d8b156468fa2e3569f86b2cf3909d9b9de3a6569f8df74ae3c6924",
"size_limiter": "validated",
"enqueued_at": 1683617159.9427748
}
}