Two days ago, we upgraded gitlab from 14.2.3 to 14.10.2. That upgrade appeared to go fine. Yesterday we were using gitab with no problem. Today when you do actions in the UI, you frequently (say 80% of the time) get logged out and sent to the login page. The error being recorded (which sometimes shows in the UI) is: Could not authenticate you from Ldapmain because “Actioncontroller::invalidauthenticitytoken”.
Below is a snippet from production_json.log on gitlab-webservice-default, container “webservice”.
Sometimes when you log in, requests to our LDAP server work, but after you click on one or two pages, an error like below will occur and you will get redirected to the login page. And login will frequently fail.
I’ve been trying to figure out the cause of this with no luck so far.
I have no idea how to check the authenticity_token.
One thing I noticed in the stack trace below is:
“lib/gitlab/jira/middleware.rb:19:in `call’”,
I don’t know if the jira middleware is significant. We use web hooks to post items to Jira, but I wouldn’t think this is getting invoked as this error occurs when I’m simply trying to get to a page.
Any assistance that can be provided would be appreciated.
{
"method": "POST",
"path": "/users/auth/ldapmain/callback",
"format": "html",
"controller": "Ldap::OmniauthCallbacksController",
"action": "ldapmain",
"status": 422,
"time": "2022-05-12T17:55:00.539Z",
"params": [
{
"key": "authenticity_token",
"value": "[FILTERED]"
},
{
"key": "username",
"value": "dwwarr"
},
{
"key": "password",
"value": "[FILTERED]"
}
],
"remote_ip": "10.244.7.6",
"ua": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.4 Safari/605.1.15",
"queue_duration_s": 0.008207,
"request_urgency": "default",
"target_duration_s": 1,
"redis_calls": 2,
"redis_duration_s": 0.000504,
"redis_write_bytes": 135,
"redis_sessions_calls": 2,
"redis_sessions_duration_s": 0.000504,
"redis_sessions_write_bytes": 135,
"db_count": 0,
"db_write_count": 0,
"db_cached_count": 0,
"db_replica_count": 0,
"db_primary_count": 0,
"db_main_count": 0,
"db_main_replica_count": 0,
"db_replica_cached_count": 0,
"db_primary_cached_count": 0,
"db_main_cached_count": 0,
"db_main_replica_cached_count": 0,
"db_replica_wal_count": 0,
"db_primary_wal_count": 0,
"db_main_wal_count": 0,
"db_main_replica_wal_count": 0,
"db_replica_wal_cached_count": 0,
"db_primary_wal_cached_count": 0,
"db_main_wal_cached_count": 0,
"db_main_replica_wal_cached_count": 0,
"db_replica_duration_s": 0.0,
"db_primary_duration_s": 0.0,
"db_main_duration_s": 0.0,
"db_main_replica_duration_s": 0.0,
"cpu_s": 0.015219,
"mem_objects": 7572,
"mem_bytes": 429496,
"mem_mallocs": 1714,
"mem_total_bytes": 732376,
"pid": 36,
"correlation_id": "01G2WPHRS4NZHADQZZAANR8R2B",
"exception.class": "ActionController::InvalidAuthenticityToken",
"exception.message": "ActionController::InvalidAuthenticityToken",
"exception.backtrace": [
"lib/gitlab/metrics/elasticsearch_rack_middleware.rb:16:in `call'",
"lib/gitlab/middleware/rails_queue_duration.rb:33:in `call'",
"lib/gitlab/middleware/memory_report.rb:13:in `call'",
"lib/gitlab/middleware/speedscope.rb:13:in `call'",
"lib/gitlab/request_profiler/middleware.rb:17:in `call'",
"lib/gitlab/database/load_balancing/rack_middleware.rb:23:in `call'",
"lib/gitlab/metrics/rack_middleware.rb:16:in `block in call'",
"lib/gitlab/metrics/web_transaction.rb:46:in `run'",
"lib/gitlab/metrics/rack_middleware.rb:16:in `call'",
"lib/gitlab/jira/middleware.rb:19:in `call'",
"lib/gitlab/middleware/go.rb:20:in `call'",
"lib/gitlab/etag_caching/middleware.rb:21:in `call'",
"lib/gitlab/middleware/query_analyzer.rb:11:in `block in call'",
"lib/gitlab/database/query_analyzer.rb:46:in `within'",
"lib/gitlab/middleware/query_analyzer.rb:11:in `call'",
"lib/gitlab/middleware/multipart.rb:173:in `call'",
"lib/gitlab/middleware/read_only/controller.rb:50:in `call'",
"lib/gitlab/middleware/read_only.rb:18:in `call'",
"lib/gitlab/middleware/same_site_cookies.rb:27:in `call'",
"lib/gitlab/middleware/handle_malformed_strings.rb:21:in `call'",
"lib/gitlab/middleware/basic_health_check.rb:25:in `call'",
"lib/gitlab/middleware/handle_ip_spoof_attack_error.rb:25:in `call'",
"lib/gitlab/middleware/request_context.rb:21:in `call'",
"lib/gitlab/middleware/webhook_recursion_detection.rb:15:in `call'",
"config/initializers/fix_local_cache_middleware.rb:11:in `call'",
"lib/gitlab/middleware/compressed_json.rb:26:in `call'",
"lib/gitlab/middleware/rack_multipart_tempfile_factory.rb:19:in `call'",
"lib/gitlab/middleware/sidekiq_web_static.rb:20:in `call'",
"lib/gitlab/metrics/requests_rack_middleware.rb:77:in `call'",
"lib/gitlab/middleware/release_env.rb:13:in `call'"
],
"db_duration_s": 0.0,
"view_duration_s": 0.0,
"duration_s": 0.0013
}