Hi!
I want to use Gitlab inside a docker container but can’t. It starts to randomly throw 500 / 502 errors and loop. Sometime it ends in Docker container in “unhealthy” mode and then need to be restarted.
This is the compose file
# Node 1
# https://hub.docker.com/r/gitlab/gitlab-ce
# https://docs.gitlab.com/omnibus/settings/nginx.html#using-a-non-bundled-web-server
version: "3"
services:
gitlab:
image: gitlab/gitlab-ce:latest
privileged: true
cpuset: 16-23,48-55
restart: always
hostname: "redacted"
environment:
- TZ=Europe/redacted
- EXTERNAL_URL=https://redacted
- REGISTRY_STORAGE_DELETE_ENABLED=true
ports:
- "7001:22"
- "7513:80"
- "7514:443"
- "5050:5050"
volumes:
- /mnt/redacted/config:/etc/gitlab
- /mnt/redacted/logs:/var/log/gitlab
- /mnt/redacted/data:/var/opt/gitlab
runner_1:
image: gitlab/gitlab-runner:latest
cpuset: 16-23,48-55
restart: always
environment:
- TZ=Europe/redacted
- CI_SERVER_URL=https://redacted
volumes:
- /mnt/redacted/runners/1/config:/etc/gitlab-runner
- /mnt/redacted/runners/1/certs:/certs/client
- /mnt/redacted/runners/1/cache:/cache
- /var/run/docker.sock:/var/run/docker.sock:rw
depends_on:
- gitlab
runner_2:
image: gitlab/gitlab-runner:latest
cpuset: 16-23,48-55
restart: always
environment:
- TZ=Europe/redacted
- CI_SERVER_URL=https://redacted
volumes:
- /mnt/redacted/runners/2/config:/etc/gitlab-runner
- /mnt/redacted/runners/2/certs:/certs/client
- /mnt/redacted/runners/2/cache:/cache
- /var/run/docker.sock:/var/run/docker.sock:rw
depends_on:
- gitlab
The error row that i see in Portainer is:
{
"severity": "DEBUG",
"time": "2022-01-22T15:25:18.199Z",
"correlation_id": "01FT168JHXX8Y11E18S5XRZ1SZ",
"message": "error Can't reserve 4096 bytes for memory-mapped file in /dev/shm/gitlab/puma/counter_puma_56-4.db backtrace /opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/prometheus-client-mmap-0.15.0/lib/prometheus/client/helper/mmaped_file.rb:21:in `initialize'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/prometheus-client-mmap-0.15.0/lib/prometheus/client/helper/mmaped_file.rb:21:in `initialize'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/prometheus-client-mmap-0.15.0/lib/prometheus/client/helper/mmaped_file.rb:38:in `new'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/prometheus-client-mmap-0.15.0/lib/prometheus/client/helper/mmaped_file.rb:38:in `open'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/prometheus-client-mmap-0.15.0/lib/prometheus/client/helper/mmaped_file.rb:50:in `open_exclusive_file'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/prometheus-client-mmap-0.15.0/lib/prometheus/client/mmaped_value.rb:119:in `unsafe_initialize_file'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/prometheus-client-mmap-0.15.0/lib/prometheus/client/mmaped_value.rb:107:in `block in initialize_file'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/prometheus-client-mmap-0.15.0/lib/prometheus/client/mmaped_value.rb:106:in `synchronize'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/prometheus-client-mmap-0.15.0/lib/prometheus/client/mmaped_value.rb:106:in `initialize_file'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/prometheus-client-mmap-0.15.0/lib/prometheus/client/mmaped_value.rb:26:in `initialize'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/prometheus-client-mmap-0.15.0/lib/prometheus/client/uses_value_type.rb:12:in `new'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/prometheus-client-mmap-0.15.0/lib/prometheus/client/uses_value_type.rb:12:in `value_object'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/prometheus-client-mmap-0.15.0/lib/prometheus/client/counter.rb:23:in `default'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/prometheus-client-mmap-0.15.0/lib/prometheus/client/metric.rb:21:in `block in initialize'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/prometheus-client-mmap-0.15.0/lib/prometheus/client/counter.rb:17:in `block in increment'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/prometheus-client-mmap-0.15.0/lib/prometheus/client/metric.rb:76:in `synchronize'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/prometheus-client-mmap-0.15.0/lib/prometheus/client/metric.rb:76:in `synchronize'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/prometheus-client-mmap-0.15.0/lib/prometheus/client/counter.rb:17:in `increment'\n/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/cluster/rack_timeout_observer.rb:24:in `log_timeout_exception'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/rack-timeout-0.5.2/lib/rack/timeout/core.rb:192:in `block in notify_state_change_observers'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/rack-timeout-0.5.2/lib/rack/timeout/core.rb:192:in `each'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/rack-timeout-0.5.2/lib/rack/timeout/core.rb:192:in `notify_state_change_observers'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/rack-timeout-0.5.2/lib/rack/timeout/core.rb:169:in `_set_state!'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/rack-timeout-0.5.2/lib/rack/timeout/core.rb:113:in `block in call'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/rack-timeout-0.5.2/lib/rack/timeout/core.rb:127:in `block in call'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/rack-timeout-0.5.2/lib/rack/timeout/support/timeout.rb:19:in `timeout'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/rack-timeout-0.5.2/lib/rack/timeout/core.rb:122:in `call'\n/opt/gitlab/embedded/service/gitlab-rails/config/initializers/fix_local_cache_middleware.rb:11:in `call'\n/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/middleware/compressed_json.rb:26:in `call'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/actionpack-6.1.4.1/lib/action_dispatch/middleware/executor.rb:14:in `call'\n/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/middleware/rack_multipart_tempfile_factory.rb:19:in `call'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/rack-2.2.3/lib/rack/sendfile.rb:110:in `call'\n/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/middleware/sidekiq_web_static.rb:20:in `call'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/actionpack-6.1.4.1/lib/action_dispatch/middleware/host_authorization.rb:92:in `call'\n/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/metrics/requests_rack_middleware.rb:75:in `call'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/gitlab-labkit-0.21.1/lib/labkit/middleware/rack.rb:19:in `block in call'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/gitlab-labkit-0.21.1/lib/labkit/context.rb:39:in `with_context'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/gitlab-labkit-0.21.1/lib/labkit/middleware/rack.rb:18:in `call'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/actionpack-6.1.4.1/lib/action_dispatch/middleware/request_id.rb:26:in `call'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/sentry-raven-3.1.2/lib/raven/integrations/rack.rb:51:in `call'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/railties-6.1.4.1/lib/rails/engine.rb:539:in `call'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/railties-6.1.4.1/lib/rails/railtie.rb:207:in `public_send'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/railties-6.1.4.1/lib/rails/railtie.rb:207:in `method_missing'\n/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/middleware/release_env.rb:13:in `call'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/rack-2.2.3/lib/rack/urlmap.rb:74:in `block in call'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/rack-2.2.3/lib/rack/urlmap.rb:58:in `each'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/rack-2.2.3/lib/rack/urlmap.rb:58:in `call'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/puma-5.5.2/lib/puma/configuration.rb:249:in `call'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/puma-5.5.2/lib/puma/request.rb:77:in `block in handle_request'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/puma-5.5.2/lib/puma/thread_pool.rb:340:in `with_force_shutdown'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/puma-5.5.2/lib/puma/request.rb:76:in `handle_request'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/puma-5.5.2/lib/puma/server.rb:447:in `process_client'\n/opt/gitlab/embedded/lib/ruby/gems/2.7.0/gems/puma-5.5.2/lib/puma/thread_pool.rb:147:in `block in spawn_thread'"
}
And
==> /var/log/gitlab/gitlab-rails/application.log <==,
2022-01-22T15:37:16.398Z: error Can't reserve 4096 bytes for memory-mapped file in /dev/shm/gitlab/sidekiq/histogram_sidekiq_0-3930.db while creating instance of Prometheus::Client::MmapedValue defaulting to SimpleValue
The array has 24TB of space, the machine has total of 32 cores, and 64GB of RAM. So this isn’t anything space related. All other containers run fine.