Hi,
This is a copy of a problem that I posted at Stackoverflow.here:
getting 502 errors with Gitlab
It would be good to get some kind of guidance on how to resolve the problem.
The Gitlab server is no longer functioning. When typing in
the sysetem never comes back and I get a connection timed out
error
I have tried to run the following:
574 sudo strace -tt -T -f -s 1024 -p 4332 -o /tmp/unicorn.txt
575 sudo strace -tt -T -f -s 1024 -p 8080 -o /tmp/unicorn.txt
as part of troubleshooting by following the steps here:
https://docs.gitlab.com/ce/administration/troubleshooting/debug.html
The result is always the same, something like:
dgmufasa@mycomputer:~$ sudo strace -tt -T -f -s 1024 -p 4332 -o /tmp/unicorn.txt
strace: attach: ptrace(PTRACE_ATTACH, ...): No such process
Below is more information. What can I do to solve the problem?
TIA
sudo gitlab-ctl show-config
{
"gitlab": {
"gitlab-shell": {
"secret_token": "secret_token",
"auth_file": "/var/opt/gitlab/.ssh/authorized_keys"
},
"gitlab-rails": {
"gitlab_default_theme": 4,
"backup_upload_connection": {
"provider": "AWS",
"region": "us-east-2",
"aws_access_key_id": "aws_access_key",
"aws_secret_access_key": "aws_secret_key"
},
"backup_upload_remote_directory": "myhost-gitlab-bckup-0001",
"backup_multipart_chunk_size": 104857600,
"backup_encryption": "AES256",
"smtp_enable": true,
"smtp_address": "smtp.gmail.com",
"smtp_port": 587,
"smtp_user_name": "myemail@gmail.com",
"smtp_password": "mypassword",
"smtp_domain": "smtp.gmail.com",
"smtp_authentication": "login",
"smtp_enable_starttls_auto": true,
"smtp_tls": false,
"smtp_openssl_verify_mode": "peer",
"secret_key_base": "secret_key_base",
"db_key_base": "dev_key_base",
"otp_key_base": "otp_key_base",
"openid_connect_signing_key": "signing_key"
"stuck_ci_jobs_worker_cron": null,
"gitlab_host": "mycomputer.com",
"gitlab_email_from": "gitlab@mycomputer.com",
"gitlab_https": false,
"gitlab_port": 4332,
"shared_path": "/var/opt/gitlab/gitlab-rails/shared",
"artifacts_path": "/var/opt/gitlab/gitlab-rails/shared/artifacts",
"lfs_storage_path": "/var/opt/gitlab/gitlab-rails/shared/lfs-objects",
"pages_path": "/var/opt/gitlab/gitlab-rails/shared/pages",
"repositories_storages": {
"default": {
"path": "/var/opt/gitlab/git-data/repositories",
"gitaly_address": "unix:/var/opt/gitlab/gitaly/gitaly.socket",
"failure_count_threshold": 10,
"failure_wait_time": 30,
"failure_reset_time": 1800,
"storage_timeout": 30
}
},
"trusted_proxies": [
],
"db_username": "gitlab",
"db_host": "/var/opt/gitlab/postgresql",
"db_port": 5432
},
"gitlab-workhorse": {
"secret_token": "secret_token",
"auth_socket": "/var/opt/gitlab/gitlab-rails/sockets/gitlab.socket"
},
"logging": {
},
"redis": {
},
"postgresql": {
},
"unicorn": {
"port": 4333
},
"mailroom": {
},
"gitlab-pages": {
},
"prometheus": {
"flags": {
"web.listen-address": "localhost:9090",
"storage.local.path": "/var/opt/gitlab/prometheus/data",
"storage.local.chunk-encoding-version": "2",
"storage.local.target-heap-size": "106218700",
"config.file": "/var/opt/gitlab/prometheus/prometheus.yml"
}
},
"external-url": "http://mycomputer.com:4332",
"registry-external-url": null,
"mattermost-external-url": null,
"pages-external-url": null,
"runtime-dir": "/run",
"bootstrap": {
},
"omnibus-gitconfig": {
},
"manage-accounts": {
},
"manage-storage-directories": {
},
"user": {
"home": "/var/opt/gitlab",
"git_user_email": "gitlab@mycomputer.com"
},
"gitlab-ci": {
},
"sidekiq": {
},
"mattermost-nginx": {
"listen_port": null
},
"pages-nginx": {
"listen_port": null
},
"registry-nginx": {
},
"remote-syslog": {
},
"logrotate": {
},
"high-availability": {
},
"web-server": {
},
"gitaly": {
"storage": [
{
"name": "default",
"path": "/var/opt/gitlab/git-data/repositories"
}
]
},
"node-exporter": {
"flags": {
"web.listen-address": "localhost:9100",
"collector.textfile.directory": "/var/opt/gitlab/node-exporter/textfile_collector"
}
},
"redis-exporter": {
"flags": {
"web.listen-address": "localhost:9121",
"redis.addr": "unix:///var/opt/gitlab/redis/redis.socket"
}
},
"postgres-exporter": {
"flags": {
"web.listen-address": "localhost:9187",
"extend.query-path": "/var/opt/gitlab/postgres-exporter/queries.yaml"
}
},
"gitlab-monitor": {
},
"prometheus-monitoring": {
},
"pgbouncer": {
},
"sentinel": {
},
"mattermost": {
"email_invite_salt": "invite_salt",
"file_public_link_salt": "file_salt",
"sql_at_rest_encrypt_key": "sql_rest_key",
"sql_data_source": "user=gitlab_mattermost host=/var/opt/gitlab/postgresql port=5432 dbname=mattermost_production",
"sql_data_source_replicas": [
"user=gitlab_mattermost host=/var/opt/gitlab/postgresql port=5432 dbname=mattermost_production"
]
},
"nginx": {
"custom_gitlab_server_config": "location ^~ /.well-known { root /var/www/letsencrypt; }",
"proxy_set_headers": {
"Host": "$http_host_with_default",
"X-Real-IP": "$remote_addr",
"X-Forwarded-For": "$proxy_add_x_forwarded_for",
"Upgrade": "$http_upgrade",
"Connection": "$connection_upgrade",
"X-Forwarded-Proto": "http"
},
"real_ip_trusted_addresses": [
],
"listen_port": 4332
}
},
"roles": {
"application": {
},
"redis-sentinel": {
},
"redis-master": {
},
"redis-slave": {
},
"geo-primary": {
},
"geo-secondary": {
}
},
"registry": {
"http_secret": "http_secret",
"internal_certificate":"internal_certificate",
"internal_key": "internal_key"
},
"repmgr": {
},
"repmgrd": {
},
"consul": {
}
}
restarting the system
dgmufasa@mycomputer:~$ sudo gitlab-ctl restart
ok: run: gitaly: (pid 5286) 0s
ok: run: gitlab-monitor: (pid 5296) 1s
ok: run: gitlab-workhorse: (pid 5300) 0s
ok: run: logrotate: (pid 5315) 0s
ok: run: nginx: (pid 5329) 1s
ok: run: node-exporter: (pid 5336) 0s
ok: run: postgres-exporter: (pid 5341) 0s
ok: run: postgresql: (pid 5350) 0s
ok: run: prometheus: (pid 5394) 0s
ok: run: redis: (pid 5404) 1s
ok: run: redis-exporter: (pid 5409) 0s
ok: run: sidekiq: (pid 5418) 1s
ok: run: unicorn: (pid 5427) 0s
I executed: dgmufasa@mycomputer:~$ sudo tail -10 /var/log/gitlab/nginx/gitlab_access.log
192.222.22.22 - - [24/Jan/2018:21:55:24 -0600] "GET /users/sign_in HTTP/1.1" 302 83 "" "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0"
192.222.22.22 - - [24/Jan/2018:21:55:28 -0600] "GET / HTTP/1.1" 200 7845 "" "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0"
192.222.22.22 - - [24/Jan/2018:21:58:50 -0600] "GET /CCLUB-WORK/ccindb-restapi-fieldworkserver HTTP/1.1" 502 2916 "http://mycomputer:4332/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0"
192.222.22.22 - - [24/Jan/2018:21:58:57 -0600] "GET /CCLUB-WORK/ccindb-restapi-fieldworkserver HTTP/1.1" 502 2916 "http://mycomputer:4332/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0"
192.222.22.22 - - [24/Jan/2018:21:59:00 -0600] "GET /CCLUB-WORK/ccindb-restapi-fieldworkserver HTTP/1.1" 502 2916 "http://mycomputer:4332/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0"
192.222.22.22 - - [24/Jan/2018:22:00:51 -0600] "GET /CCLUB-WORK/ccindb-restapi-fieldworkserver HTTP/1.1" 499 0 "http://mycomputer:4332/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0"