My problem is that after the update from 15.9.11 to 16.1.0 GitLab-runner brings HTTP code 500 in GitLab when I try to start a CI.
A reset of the runner-token did not help
The WebUI, git push, git clone, … works fine.
the runner-log:
gitlab-runner | Checking for jobs... received job=1711 repo_url=http://git.****.local/******/*******.git runner=844FkkyFx
gitlab-runner | WARNING: Container based cache volumes creation is disabled. Will not create volume for "/cache" job=1711 project=243 runner=844FkkyFx
gitlab-runner | WARNING: Appending trace to coordinator... failed code=500 job=1711 job-log= job-status= runner=844FkkyFx sent-log=0-977 status=500 Internal Server Error update-interval=0s
gitlab-runner | ERROR: Could not create cache adapter error=cache factory not found: factory for cache adapter "" was not registered
gitlab-runner | WARNING: Appending trace to coordinator... failed code=500 job=1711 job-log= job-status= runner=844FkkyFx sent-log=0-1729 status=500 Internal Server Error update-interval=0s
in the Gitlab log I noticed that
{"time":"2023-07-11T13:09:03.536Z","severity":"INFO","duration_s":0.07094,"db_duration_s":0.02727,"view_duration_s":0.04367,"status":500,"method":"PATCH","path":"/api/v4/jobs/1706/trace","params":[{"key":"debug_trace","value":"[FILTERED]"}],"host":"git.aa.local","remote_ip":"172.21.0.1, 172.21.0.1, 127.0.0.1","ua":"gitlab-runner 16.1.0 (16-1-stable; go1.19.9; linux/amd64)","route":"/api/:version/jobs/:id/trace","exception.class":"OpenSSL::Cipher::CipherError","exception.message":"","exception.backtrace":["lib/gitlab/crypto_helper.rb:28:in `aes256_gcm_decrypt'","app/models/concerns/token_authenticatable_strategies/encryption_helper.rb:16:in `decrypt_token'","app/models/concerns/token_authenticatable_strategies/encrypted.rb:78:in `get_encrypted_token'","app/models/concerns/token_authenticatable_strategies/encrypted.rb:113:in `token_set?'","app/models/concerns/token_authenticatable_strategies/base.rb:50:in `ensure_token!'","app/models/concerns/token_authenticatable.rb:54:in `block in add_authentication_token_field'","app/models/project.rb:2197:in `runners_token'","app/models/ci/build.rb:983:in `block in hide_secrets'","app/models/ci/build.rb:982:in `hide_secrets'","lib/gitlab/ci/trace.rb:69:in `block in append'","lib/gitlab/ci/trace.rb:184:in `unsafe_write!'","lib/gitlab/ci/trace.rb:96:in `block in write'","lib/gitlab/exclusive_lease_helpers.rb:38:in `in_lock'","lib/gitlab/ci/trace.rb:227:in `in_write_lock'","lib/gitlab/ci/trace.rb:95:in `write'","lib/gitlab/ci/trace.rb:65:in `append'","app/services/ci/append_build_trace_service.rb:38:in `execute'","lib/api/ci/runner.rb:279:in `block (2 levels) in <class:Runner>'","ee/lib/gitlab/middleware/ip_restrictor.rb:14:in `block in call'","ee/lib/gitlab/ip_address_state.rb:10:in `with'","ee/lib/gitlab/middleware/ip_restrictor.rb:13:in `call'","lib/api/api_guard.rb:219:in `call'","lib/gitlab/metrics/elasticsearch_rack_middleware.rb:16:in `call'","lib/gitlab/middleware/memory_report.rb:13:in `call'","lib/gitlab/middleware/speedscope.rb:13:in `call'","lib/gitlab/database/load_balancing/rack_middleware.rb:23:in `call'","lib/gitlab/middleware/rails_queue_duration.rb:33:in `call'","lib/gitlab/metrics/rack_middleware.rb:16:in `block in call'","lib/gitlab/metrics/web_transaction.rb:46:in `run'","lib/gitlab/metrics/rack_middleware.rb:16:in `call'","lib/gitlab/jira/middleware.rb:19:in `call'","lib/gitlab/middleware/go.rb:20:in `call'","lib/gitlab/etag_caching/middleware.rb:21:in `call'","lib/gitlab/middleware/query_analyzer.rb:11:in `block in call'","lib/gitlab/database/query_analyzer.rb:37:in `within'","lib/gitlab/middleware/query_analyzer.rb:11:in `call'","lib/gitlab/middleware/multipart.rb:173:in `call'","lib/gitlab/middleware/read_only/controller.rb:50:in `call'","lib/gitlab/middleware/read_only.rb:18:in `call'","lib/gitlab/middleware/same_site_cookies.rb:27:in `call'","lib/gitlab/middleware/basic_health_check.rb:25:in `call'","lib/gitlab/middleware/handle_malformed_strings.rb:21:in `call'","lib/gitlab/middleware/handle_ip_spoof_attack_error.rb:25:in `call'","lib/gitlab/middleware/request_context.rb:15:in `call'","lib/gitlab/middleware/webhook_recursion_detection.rb:15:in `call'","config/initializers/fix_local_cache_middleware.rb:11:in `call'","lib/gitlab/middleware/compressed_json.rb:44:in `call'","lib/gitlab/middleware/rack_multipart_tempfile_factory.rb:19:in `call'","lib/gitlab/middleware/sidekiq_web_static.rb:20:in `call'","lib/gitlab/metrics/requests_rack_middleware.rb:79:in `call'","lib/gitlab/middleware/release_env.rb:13:in `call'"],"queue_duration_s":0.016038,"redis_calls":10,"redis_duration_s":0.005992,"redis_read_bytes":586,"redis_write_bytes":1041,"redis_cache_calls":4,"redis_cache_duration_s":0.004328,"redis_cache_read_bytes":382,"redis_cache_write_bytes":538,"redis_feature_flag_calls":1,"redis_feature_flag_duration_s":0.000362,"redis_feature_flag_read_bytes":202,"redis_feature_flag_write_bytes":60,"redis_shared_state_calls":5,"redis_shared_state_duration_s":0.001302,"redis_shared_state_read_bytes":2,"redis_shared_state_write_bytes":443,"db_count":10,"db_write_count":0,"db_cached_count":0,"db_replica_count":0,"db_primary_count":10,"db_main_count":6,"db_ci_count":4,"db_main_replica_count":0,"db_ci_replica_count":0,"db_replica_cached_count":0,"db_primary_cached_count":0,"db_main_cached_count":0,"db_ci_cached_count":0,"db_main_replica_cached_count":0,"db_ci_replica_cached_count":0,"db_replica_wal_count":0,"db_primary_wal_count":0,"db_main_wal_count":0,"db_ci_wal_count":0,"db_main_replica_wal_count":0,"db_ci_replica_wal_count":0,"db_replica_wal_cached_count":0,"db_primary_wal_cached_count":0,"db_main_wal_cached_count":0,"db_ci_wal_cached_count":0,"db_main_replica_wal_cached_count":0,"db_ci_replica_wal_cached_count":0,"db_replica_duration_s":0.0,"db_primary_duration_s":0.028,"db_main_duration_s":0.019,"db_ci_duration_s":0.008,"db_main_replica_duration_s":0.0,"db_ci_replica_duration_s":0.0,"cpu_s":0.056708,"mem_objects":14777,"mem_bytes":962904,"mem_mallocs":3797,"mem_total_bytes":1553984,"pid":659,"worker_id":"puma_3","rate_limiting_gates":[],"correlation_id":"01H52H3MTJN55TJ4FHG748N28J","meta.caller_id":"PATCH /api/:version/jobs/:id/trace","meta.remote_ip":"172.21.0.1","meta.feature_category":"continuous_integration","meta.project":"bbbbb/website","meta.root_namespace":"bbbbb","meta.client_id":"user/4","meta.pipeline_id":797,"meta.job_id":1706,"content_length":"4090","content_range":"0-4089","request_urgency":"low","target_duration_s":5,"response_bytes":39}
both the Gitlab-runner and Gitlab (ee) are running in Docker containers on an Ubuntu 20.04 server.
My docker-compose.yml
version: '3.4'
services:
gitlab:
mem_limit: 8GB # <= Memory limitation
image: gitlab/gitlab-ee
container_name: 'gitlab'
restart: always
environment:
GITLAB_ROOT_PASSWORD: ********************************
GITLAB_OMNIBUS_CONFIG: |
external_url 'http://git.***.local/'
letsencrypt['acme_staging_endpoint'] = 'https://step-ca:9000/acme/acme/directory'
letsencrypt['acme_production_endpoint'] = 'https://step-ca:9000/acme/acme/directory'
letsencrypt['auto_renew'] = true
nginx['real_ip_trusted_addresses'] = [ '192.168.212.0/24', '172.21.0.0/16' ]
nginx['listen_port'] = 80
nginx['listen_https'] = false
# nginx['redirect_http_to_https'] = true
nginx['proxy_set_headers'] = {
"X-Forwarded-Proto" => "https",
"X-Forwarded-Ssl" => "on"
}
prometheus_monitoring['enable'] = false
node_exporter['listen_address'] = '0.0.0.0:9100'
gitlab_workhorse['prometheus_listen_addr'] = "0.0.0.0:9229"
sidekiq['listen_address'] = '0.0.0.0'
redis_exporter['listen_address'] = '0.0.0.0:9121'
gitlab_rails['monitoring_whitelist'] = ['127.0.0.0/8', '192.168.212.107', '172.12.0.0/16', '192.168.0.0/16', '10.0.0.0/8']
# gitaly['logging_level'] = 'error' ##deaktiviert 230627 AZ: not supported since 16.0.0
gitlab_shell['log_level'] = 'ERROR'
registry['log_level'] = 'ERROR'
registry_external_url 'http://git.**.local:5005'
registry['enable'] = true
gitlab_rails['artifacts_path'] = "/mnt/storage/artifacts" #
gitlab_rails['artifacts_enabled'] = true #
# gitlab_rails['artifacts_object_store_remote_directory'] = "artifacts"
#gitlab_rails['artifacts_object_store_direct_upload'] = true
gitlab_rails['artifacts_object_store_enabled'] = false #220602 AZ change to false for update to 15.0
# gitlab_rails['initial_license_file'] = "/etc/gitlab/Gitlab.gitlab-license" # 221104 AZ Lizenzdatei
logging['logrotate_frequency'] = "daily" # rotate logs daily
logging['logrotate_maxsize'] = 20 * 1024 * 1024 # logs will be rotated when they grow bigger than size specified for `maxsize`, even before the specified time interval (daily, weekly, monthly, or yearly)
sidekiq['max_concurrency'] = 10 # default value 50 ist set from AZ: https://docs.gitlab.com/omnibus/settings/memory_constrained_envs.html
sidekiq['enable'] = true
sidekiq['queue_selector'] = true
sidekiq['queue_groups'] = [
"feature_category=global_search",
"feature_category!=global_search"
]
# SMPT
gitlab_rails['smtp_enable'] = true
gitlab_rails['smtp_address'] = "smtp.****.***"
gitlab_rails['smtp_port'] = 587
gitlab_rails['smtp_user_name'] = "noreply@*****.***"
gitlab_rails['smtp_password'] = "*********"
gitlab_rails['smtp_domain'] = "*****.***"
gitlab_rails['smtp_authentication'] = "login"
gitlab_rails['smtp_enable_starttls_auto'] = true
gitlab_rails['smtp_tls'] = false
gitlab_rails['smtp_openssl_verify_mode'] = 'none'
# LDAP
## For setting up LDAP
## see https://gitlab.com/gitlab-org/omnibus-gitlab/blob/629def0a7a26e7c2326566f0758d4a27857b52a3/README.md#setting-up-ldap-sign-in
gitlab_rails['ldap_enabled'] = true
gitlab_rails['ldap_servers'] = YAML.load <<-EOS # {
main: #'main' is the GitLab 'provider ID' of this LDAP server
label: 'LDAP'
host: '***.***.***.***'
port: 389
uid: 'sAMAccountName'
method: 'plain' # "tls" or "ssl" or "plain"
bind_dn: 'readonly@*****.***'
password: '************'
active_directory: true
allow_username_or_email_login: true
block_auto_created_users: false
base: 'OU=SBSUsers,OU=Users,OU=MyBusiness,DC=*****,DC=***'
attributes:
username: ['uid', 'userid', 'sAMAccountName']
email: ['mail', 'email', 'userPrincipalName']
name: 'cn'
first_name: 'givenName'
last_name: 'sn'
EOS
#gitlab_rails['ldap_sync_worker_cron'] = "* */12 * * *"
expose:
- "80"
- "443"
- "22"
- "9090"
- "9100"
- "9168"
- "9121"
- "9187"
ports:
- '5005:5005'
volumes:
- ./data/config:/etc/gitlab
- ./logs:/var/log # for debuging AZ 20230711
- ./data/data:/var/opt/gitlab
- ./data/license.key:/etc/gitlab/Gitlab.gitlab-license # intitial Lizenzdatei 221104 AZ
- ../elasticsearch/data/certs/es01/es01.crt:/etc/gitlab/trusted-certs/es01.crt # ssl for ElasticSearch (advancedSearch)
- ./data/artifacts:/mnt/storage/artifacts
- /etc/ssl/certs/ca-certificates.crt:/etc/ssl/certs/ca-certificates.crt:ro
networks:
npm-network:
internal:
ipv4_address: 192.168.0.2
# healthcheck:
# disable: true
gitlab-runner:
image: gitlab/gitlab-runner #v15.11.1
container_name: 'gitlab-runner'
restart: always
volumes:
- ./data/config-runner:/etc/gitlab-runner
- ./data/config-runner/resolv.conf:/etc/resolv.conf
- /var/run/docker.sock:/var/run/docker.sock
- /etc/ssl/certs/ca-certificates.crt:/etc/ssl/certs/ca-certificates.crt:ro
networks:
npm-network:
internal:
ipv4_address: 192.168.0.3
environment:
- CI_SERVER_URL=http://git.*****.local
- REGISTRATION_TOKEN=***************************
networks:
npm-network:
external: true
internal:
internal: true
ipam:
driver: default
config:
- subnet: 192.168.0.0/24