GitLab registry 502 Bad Gateway

Hi folks,

I’ve the latest GitLab-Docker running gitlab/gitlab-ce:14.10.2-ce.0. This instance sits behind a nginxPM reverse proxy (which handles the certificates and traffic). The default https traffics is forwarded to the proxy and then forwarded to the correct docker instances.
I want to activate the container repository under it’s own domain like registry.gitlab.domain.tld
On the management overview i see that it is enabled, but when i try to connect to it, i get a HTTP 502 Bad Request error code.

Here is my Docker Compose

version: '3.5'
services:
  gitlab:
    image: gitlab/gitlab-ce:14.10.2-ce.0
    container_name: gitlab
    hostname: gitlab.domain.tld
    restart: unless-stopped
    environment:
      GITLAB_OMNIBUS_CONFIG: |
        gitlab_rails['gitlab_shell_ssh_port'] = 4222
    ports:
      - "4004:80"
      - "4222:22"
      - "4005:5000" // i guess this is the correct registry port???
    volumes:
      - /volume1/docker/gitlab/config:/etc/gitlab
      - /volume1/docker/gitlab/data:/var/opt/gitlab
      - /volume1/docker/gitlab/logs:/var/log/gitlab

    networks:
      - gitlab

  gitlab-runner:
    image: gitlab/gitlab-runner:alpine
    container_name: gitlab-runner
    restart: unless-stopped
    depends_on:
      - gitlab
    volumes:
      - /volume1/docker/gitlab/runner:/etc/gitlab-runner
      - /var/run/docker.sock:/var/run/docker.sock
    networks:
      - gitlab

networks:
  gitlab:

Proxy
gitlab.domain.tld points to http://host:4004
repository.gitlab.domain.tld points to https://host:4005

SSL Certificates
I have followed the docs for setting up the registy under its own domain via Omnibus. Basically copied the SSL certs from the reverseproxy to
/etc/gitlab/ssl/repository/fullchain.pem and
/etc/gitlab/ssl/repository/privatekey.pem according tho this post. Access rights are correct:

-rwxr-xr-x 1 root root 3.4K May 15 20:54 fullchain.pem
-rwxr-xr-x 1 root root  306 May 15 20:55 privatekey.pem

gitlab.rb changes

## Container Registry settings
registry_external_url 'https://registry.gitlab.domain.tld'
gitlab_rails['registry_enabled'] = true
gitlab_rails['registry_host'] = "registry.gitlab.domain.tld"
### Settings used by Registry application
registry['enable'] = true
registry['registry_http_addr'] = "localhost:5000"

## Registry NGINX
registry_nginx['ssl_certificate'] = "/etc/gitlab/ssl/registry/fullchain.pem"
registry_nginx['ssl_certificate_key'] = "/etc/gitlab/ssl/registry/privatekey.pem"

gitlab-ctl show-config output

root@gitlab:/# gitlab-ctl show-config
Starting Chef Infra Client, version 15.17.4
resolving cookbooks for run list: ["gitlab::show_config"]
Synchronizing Cookbooks:
  - package (0.1.0)
  - gitlab (0.0.1)
  - logrotate (0.1.0)
  - postgresql (0.1.0)
  - redis (0.1.0)
  - monitoring (0.1.0)
  - registry (0.1.0)
  - gitlab-pages (0.1.0)
  - praefect (0.1.0)
  - mattermost (0.1.0)
  - nginx (0.1.0)
  - gitaly (0.1.0)
  - acme (4.1.3)
  - gitlab-kas (0.1.0)
  - consul (0.1.0)
  - runit (5.1.3)
  - crond (0.1.0)
  - letsencrypt (0.1.0)
Installing Cookbook Gems:
Compiling Cookbooks...
{
  "gitlab": {
    "gitlab-shell": {
      "secret_token": "4a398f405ce82c08c7d22dbbf...d806c1c579e31ae45275f7d58d6afb3",
      "auth_file": "/var/opt/gitlab/.ssh/authorized_keys"
    },
    "gitlab-rails": {
      "gitlab_shell_ssh_port": 4222,
      "registry_enabled": true,
      "registry_host": "registry.gitlab.domain.tld",
      "secret_key_base": "785800870d...e07f7f002d5c0465ac004",
      "db_key_base": "ab860d2cdec4...70dc564401e8e3ff6ace8a6925c7e8c2928f",
      "otp_key_base": "db0c7a8b2595...67bee0624",
      "encrypted_settings_key_base": "3f929e57952b6da11e34e1cdd5...ef3f",
      "openid_connect_signing_key": "",
      "store_initial_root_password": true,
      "db_username": "gitlab",
      "db_host": "/var/opt/gitlab/postgresql",
      "db_port": 5432,
      "databases": {
        "main": {
          "enable": true,
          "db_adapter": "postgresql",
          "db_encoding": "unicode",
          "db_database": "gitlabhq_production",
          "db_username": "gitlab",
          "db_load_balancing": {
            "hosts": [

            ]
          },
          "db_host": "/var/opt/gitlab/postgresql",
          "db_port": 5432,
          "db_sslcompression": 0,
          "db_prepared_statements": false,
          "db_database_tasks": true,
          "db_statements_limit": 1000
        }
      },
      "gitlab_url": "http://gitlab.domain.tld",
      "gitlab_host": "gitlab.domain.tld",
      "gitlab_email_from": "gitlab@gitlab.domain.tld",
      "gitlab_https": false,
      "gitlab_port": 80,
      "shared_path": "/var/opt/gitlab/gitlab-rails/shared",
      "artifacts_path": "/var/opt/gitlab/gitlab-rails/shared/artifacts",
      "external_diffs_storage_path": "/var/opt/gitlab/gitlab-rails/shared/external-diffs",
      "lfs_storage_path": "/var/opt/gitlab/gitlab-rails/shared/lfs-objects",
      "uploads_storage_path": "/opt/gitlab/embedded/service/gitlab-rails/public",
      "packages_storage_path": "/var/opt/gitlab/gitlab-rails/shared/packages",
      "dependency_proxy_storage_path": "/var/opt/gitlab/gitlab-rails/shared/dependency_proxy",
      "terraform_state_storage_path": "/var/opt/gitlab/gitlab-rails/shared/terraform_state",
      "ci_secure_files_storage_path": "/var/opt/gitlab/gitlab-rails/shared/ci_secure_files",
      "encrypted_settings_path": "/var/opt/gitlab/gitlab-rails/shared/encrypted_settings",
      "pages_path": "/var/opt/gitlab/gitlab-rails/shared/pages",
      "pages_local_store_path": "/var/opt/gitlab/gitlab-rails/shared/pages",
      "repositories_storages": {
        "default": {
          "path": "/var/opt/gitlab/git-data/repositories",
          "gitaly_address": "unix:/var/opt/gitlab/gitaly/gitaly.socket"
        }
      },
      "trusted_proxies": [

      ],
      "incoming_email_log_file": "/var/log/gitlab/mailroom/mail_room_json.log",
      "service_desk_email_log_file": "/var/log/gitlab/mailroom/mail_room_json.log",
      "max_request_duration_seconds": 57,
      "registry_api_url": "http://localhost:5000",
      "registry_path": "/var/opt/gitlab/gitlab-rails/shared/registry",
      "gitlab_kas_enabled": false
    },
    "gitlab-workhorse": {
      "secret_token": "njs20y7...YLCELzMrkHnQ=",
      "sockets_directory": "/var/opt/gitlab/gitlab-workhorse/sockets",
      "listen_addr": "/var/opt/gitlab/gitlab-workhorse/sockets/socket"
    },
    "logging": {

    },
    "unicorn": {

    },
    "puma": {

    },
    "mailroom": {

    },
    "external-url": "http://gitlab.domain.tld",
    "registry-external-url": "https://registry.gitlab.domain.tld",
    "runtime-dir": "/dev/shm",
    "actioncable": {

    },
    "bootstrap": {

    },
    "omnibus-gitconfig": {

    },
    "manage-accounts": {
      "enable": false
    },
    "manage-storage-directories": {

    },
    "user": {
      "home": "/var/opt/gitlab",
      "git_user_email": "gitlab@gitlab.domain.tld"
    },
    "gitlab-ci": {

    },
    "sidekiq": {

    },
    "mattermost-nginx": {

    },
    "pages-nginx": {

    },
    "registry-nginx": {
      "ssl_certificate": "/etc/gitlab/ssl/registry/fullchain.pem",
      "ssl_certificate_key": "/etc/gitlab/ssl/registry/privatekey.pem",
      "listen_port": 443,
      "https": true,
      "proxy_set_headers": {
        "Host": "$http_host",
        "X-Real-IP": "$remote_addr",
        "X-Forwarded-For": "$proxy_add_x_forwarded_for",
        "X-Forwarded-Proto": "https",
        "X-Forwarded-Ssl": "on"
      }
    },
    "remote-syslog": {

    },
    "high-availability": {

    },
    "web-server": {

    },
    "prometheus-monitoring": {

    },
    "storage-check": {
      "target": "unix:///var/opt/gitlab/gitlab-rails/sockets/gitlab.socket"
    },
    "nginx": {
      "proxy_set_headers": {
        "Host": "$http_host_with_default",
        "X-Real-IP": "$remote_addr",
        "X-Forwarded-For": "$proxy_add_x_forwarded_for",
        "Upgrade": "$http_upgrade",
        "Connection": "$connection_upgrade",
        "X-Forwarded-Proto": "http"
      },
      "real_ip_trusted_addresses": [

      ],
      "listen_port": 80
    }
  },
  "roles": {
    "application": {

    },
    "redis-sentinel": {

    },
    "redis-master": {

    },
    "redis-replica": {

    },
    "geo-primary": {

    },
    "geo-secondary": {

    },
    "monitoring": {

    },
    "postgres": {

    },
    "pgbouncer": {

    },
    "patroni": {

    },
    "consul": {

    },
    "pages": {

    },
    "sidekiq": {

    },
    "spamcheck": {

    }
  },
  "monitoring": {
    "prometheus": {
      "alertmanagers": [

      ],
      "flags": {
        "web.listen-address": "localhost:9090",
        "storage.tsdb.path": "/var/opt/gitlab/prometheus/data",
        "config.file": "/var/opt/gitlab/prometheus/prometheus.yml"
      }
    },
    "grafana": {
      "secret_key": "a06427f6...0ea9805b9",
      "gitlab_secret": "9a5120ab8276f...3ee993a327",
      "gitlab_application_id": "e5524...5e515cee16b21531",
      "admin_password": "85ae69f...5874d6",
      "datasources": [
        {
          "name": "GitLab Omnibus",
          "type": "prometheus",
          "access": "proxy",
          "url": "http://localhost:9090",
          "isDefault": true
        }
      ]
    },
    "alertmanager": {
      "flags": {
        "web.listen-address": "localhost:9093",
        "storage.path": "/var/opt/gitlab/alertmanager/data",
        "config.file": "/var/opt/gitlab/alertmanager/alertmanager.yml"
      }
    },
    "node-exporter": {
      "enable": false,
      "flags": {
        "web.listen-address": "localhost:9100",
        "collector.mountstats": true,
        "collector.runit": true,
        "collector.runit.servicedir": "/opt/gitlab/sv",
        "collector.textfile.directory": "/var/opt/gitlab/node-exporter/textfile_collector"
      }
    },
    "redis-exporter": {
      "flags": {
        "web.listen-address": "localhost:9121",
        "redis.addr": "unix:///var/opt/gitlab/redis/redis.socket"
      }
    },
    "postgres-exporter": {
      "flags": {
        "web.listen-address": "localhost:9187",
        "extend.query-path": "/var/opt/gitlab/postgres-exporter/queries.yaml"
      }
    },
    "pgbouncer-exporter": {

    },
    "gitlab-exporter": {
      "probe_sidekiq": true
    }
  },
  "letsencrypt": {
    "auto_enabled": false,
    "enable": false
  },
  "package": {
    "detect_init": false,
    "modify_kernel_parameters": false
  },
  "registry": {
    "enable": true,
    "http_secret": "",
    "registry_http_addr": "localhost:5000",
    "token_realm": "http://gitlab.domain.tld",
    "storage_delete_enabled": true,
    "health_storagedriver_enabled": true,
    "storage": {
      "filesystem": {
        "rootdirectory": "/var/opt/gitlab/gitlab-rails/shared/registry"
      },
      "cache": {
        "blobdescriptor": "inmemory"
      },
      "delete": {
        "enabled": true
      }
    }
  },
  "redis": {
    "rename_commands": {
      "KEYS": ""
    }
  },
  "postgresql": {
    "shared_buffers": "1MB",
    "internal_certificate": "",
    "connect_port": 5432,
    "wal_keep_size": 160
  },
  "gitlab-kas": {
    "api_secret_key": "NG...NmE=",
    "private_api_secret_key": "MmI3N...NzQ=",
    "gitlab_address": "http://gitlab.domain.tld"
  },
  "gitlab-pages": {
    "api_secret_key": "A1+...+rc0c="
  },
  "pgbouncer": {

  },
  "repmgr": {

  },
  "repmgrd": {

  },
  "consul": {

  },
  "spamcheck": {

  },
  "patroni": {

  },
  "gitaly": {
    "storage": [
      {
        "name": "default",
        "path": "/var/opt/gitlab/git-data/repositories"
      }
    ]
  },
  "praefect": {

  },
  "crond": {

  },
  "logrotate": {

  },
  "mattermost": {
    "email_invite_salt": "092e...e2fb",
    "file_public_link_salt": "af...1f3",
    "sql_at_rest_encrypt_key": "461...8ef38",
    "sql_data_source": "user=gitlab_mattermost host=/var/opt/gitlab/postgresql port=5432 dbname=mattermost_production"
  }
}
Converging 0 resources

Running handlers:
Running handlers complete
Chef Infra Client finished, 0/0 resources updated in 10 seconds
root@gitlab:/# gitlab-ctl show-config
Starting Chef Infra Client, version 15.17.4
resolving cookbooks for run list: ["gitlab::show_config"]
Synchronizing Cookbooks:
  - package (0.1.0)
  - gitlab (0.0.1)
  - logrotate (0.1.0)
  - postgresql (0.1.0)
  - redis (0.1.0)
  - monitoring (0.1.0)
  - registry (0.1.0)
  - gitlab-pages (0.1.0)
  - praefect (0.1.0)
  - mattermost (0.1.0)
  - nginx (0.1.0)
  - gitaly (0.1.0)
  - acme (4.1.3)
  - gitlab-kas (0.1.0)
  - consul (0.1.0)
  - runit (5.1.3)
  - crond (0.1.0)
  - letsencrypt (0.1.0)
Installing Cookbook Gems:
Compiling Cookbooks...
{
  "gitlab": {
    "gitlab-shell": {
      "secret_token": "4a398f405ce82c08c7d22dbbf...d806c1c579e31ae45275f7d58d6afb3",
      "auth_file": "/var/opt/gitlab/.ssh/authorized_keys"
    },
    "gitlab-rails": {
      "gitlab_shell_ssh_port": 4222,
      "registry_enabled": true,
      "registry_host": "registry.gitlab.domain.tld",
      "secret_key_base": "785800870d...e07f7f002d5c0465ac004",
      "db_key_base": "ab860d2cdec4...70dc564401e8e3ff6ace8a6925c7e8c2928f",
      "otp_key_base": "db0c7a8b2595...67bee0624",
      "encrypted_settings_key_base": "3f929e57952b6da11e34e1cdd5...ef3f",
      "openid_connect_signing_key": "",
      "store_initial_root_password": true,
      "db_username": "gitlab",
      "db_host": "/var/opt/gitlab/postgresql",
      "db_port": 5432,
      "databases": {
        "main": {
          "enable": true,
          "db_adapter": "postgresql",
          "db_encoding": "unicode",
          "db_database": "gitlabhq_production",
          "db_username": "gitlab",
          "db_load_balancing": {
            "hosts": [

            ]
          },
          "db_host": "/var/opt/gitlab/postgresql",
          "db_port": 5432,
          "db_sslcompression": 0,
          "db_prepared_statements": false,
          "db_database_tasks": true,
          "db_statements_limit": 1000
        }
      },
      "gitlab_url": "http://gitlab.domain.tld",
      "gitlab_host": "gitlab.domain.tld",
      "gitlab_email_from": "gitlab@gitlab.domain.tld",
      "gitlab_https": false,
      "gitlab_port": 80,
      "shared_path": "/var/opt/gitlab/gitlab-rails/shared",
      "artifacts_path": "/var/opt/gitlab/gitlab-rails/shared/artifacts",
      "external_diffs_storage_path": "/var/opt/gitlab/gitlab-rails/shared/external-diffs",
      "lfs_storage_path": "/var/opt/gitlab/gitlab-rails/shared/lfs-objects",
      "uploads_storage_path": "/opt/gitlab/embedded/service/gitlab-rails/public",
      "packages_storage_path": "/var/opt/gitlab/gitlab-rails/shared/packages",
      "dependency_proxy_storage_path": "/var/opt/gitlab/gitlab-rails/shared/dependency_proxy",
      "terraform_state_storage_path": "/var/opt/gitlab/gitlab-rails/shared/terraform_state",
      "ci_secure_files_storage_path": "/var/opt/gitlab/gitlab-rails/shared/ci_secure_files",
      "encrypted_settings_path": "/var/opt/gitlab/gitlab-rails/shared/encrypted_settings",
      "pages_path": "/var/opt/gitlab/gitlab-rails/shared/pages",
      "pages_local_store_path": "/var/opt/gitlab/gitlab-rails/shared/pages",
      "repositories_storages": {
        "default": {
          "path": "/var/opt/gitlab/git-data/repositories",
          "gitaly_address": "unix:/var/opt/gitlab/gitaly/gitaly.socket"
        }
      },
      "trusted_proxies": [

      ],
      "incoming_email_log_file": "/var/log/gitlab/mailroom/mail_room_json.log",
      "service_desk_email_log_file": "/var/log/gitlab/mailroom/mail_room_json.log",
      "max_request_duration_seconds": 57,
      "registry_api_url": "http://localhost:5000",
      "registry_path": "/var/opt/gitlab/gitlab-rails/shared/registry",
      "gitlab_kas_enabled": false
    },
    "gitlab-workhorse": {
      "secret_token": "njs20y7...YLCELzMrkHnQ=",
      "sockets_directory": "/var/opt/gitlab/gitlab-workhorse/sockets",
      "listen_addr": "/var/opt/gitlab/gitlab-workhorse/sockets/socket"
    },
    "logging": {

    },
    "unicorn": {

    },
    "puma": {

    },
    "mailroom": {

    },
    "external-url": "http://gitlab.domain.tld",
    "registry-external-url": "https://registry.gitlab.domain.tld",
    "runtime-dir": "/dev/shm",
    "actioncable": {

    },
    "bootstrap": {

    },
    "omnibus-gitconfig": {

    },
    "manage-accounts": {
      "enable": false
    },
    "manage-storage-directories": {

    },
    "user": {
      "home": "/var/opt/gitlab",
      "git_user_email": "gitlab@gitlab.domain.tld"
    },
    "gitlab-ci": {

    },
    "sidekiq": {

    },
    "mattermost-nginx": {

    },
    "pages-nginx": {

    },
    "registry-nginx": {
      "ssl_certificate": "/etc/gitlab/ssl/registry/fullchain.pem",
      "ssl_certificate_key": "/etc/gitlab/ssl/registry/privatekey.pem",
      "listen_port": 443,
      "https": true,
      "proxy_set_headers": {
        "Host": "$http_host",
        "X-Real-IP": "$remote_addr",
        "X-Forwarded-For": "$proxy_add_x_forwarded_for",
        "X-Forwarded-Proto": "https",
        "X-Forwarded-Ssl": "on"
      }
    },
    "remote-syslog": {

    },
    "high-availability": {

    },
    "web-server": {

    },
    "prometheus-monitoring": {

    },
    "storage-check": {
      "target": "unix:///var/opt/gitlab/gitlab-rails/sockets/gitlab.socket"
    },
    "nginx": {
      "proxy_set_headers": {
        "Host": "$http_host_with_default",
        "X-Real-IP": "$remote_addr",
        "X-Forwarded-For": "$proxy_add_x_forwarded_for",
        "Upgrade": "$http_upgrade",
        "Connection": "$connection_upgrade",
        "X-Forwarded-Proto": "http"
      },
      "real_ip_trusted_addresses": [

      ],
      "listen_port": 80
    }
  },
  "roles": {
    "application": {

    },
    "redis-sentinel": {

    },
    "redis-master": {

    },
    "redis-replica": {

    },
    "geo-primary": {

    },
    "geo-secondary": {

    },
    "monitoring": {

    },
    "postgres": {

    },
    "pgbouncer": {

    },
    "patroni": {

    },
    "consul": {

    },
    "pages": {

    },
    "sidekiq": {

    },
    "spamcheck": {

    }
  },
  "monitoring": {
    "prometheus": {
      "alertmanagers": [

      ],
      "flags": {
        "web.listen-address": "localhost:9090",
        "storage.tsdb.path": "/var/opt/gitlab/prometheus/data",
        "config.file": "/var/opt/gitlab/prometheus/prometheus.yml"
      }
    },
    "grafana": {
      "secret_key": "a06427f6...0ea9805b9",
      "gitlab_secret": "9a5120ab8276f...3ee993a327",
      "gitlab_application_id": "e5524...5e515cee16b21531",
      "admin_password": "85ae69f...5874d6",
      "datasources": [
        {
          "name": "GitLab Omnibus",
          "type": "prometheus",
          "access": "proxy",
          "url": "http://localhost:9090",
          "isDefault": true
        }
      ]
    },
    "alertmanager": {
      "flags": {
        "web.listen-address": "localhost:9093",
        "storage.path": "/var/opt/gitlab/alertmanager/data",
        "config.file": "/var/opt/gitlab/alertmanager/alertmanager.yml"
      }
    },
    "node-exporter": {
      "enable": false,
      "flags": {
        "web.listen-address": "localhost:9100",
        "collector.mountstats": true,
        "collector.runit": true,
        "collector.runit.servicedir": "/opt/gitlab/sv",
        "collector.textfile.directory": "/var/opt/gitlab/node-exporter/textfile_collector"
      }
    },
    "redis-exporter": {
      "flags": {
        "web.listen-address": "localhost:9121",
        "redis.addr": "unix:///var/opt/gitlab/redis/redis.socket"
      }
    },
    "postgres-exporter": {
      "flags": {
        "web.listen-address": "localhost:9187",
        "extend.query-path": "/var/opt/gitlab/postgres-exporter/queries.yaml"
      }
    },
    "pgbouncer-exporter": {

    },
    "gitlab-exporter": {
      "probe_sidekiq": true
    }
  },
  "letsencrypt": {
    "auto_enabled": false,
    "enable": false
  },
  "package": {
    "detect_init": false,
    "modify_kernel_parameters": false
  },
  "registry": {
    "enable": true,
    "http_secret": "",
    "registry_http_addr": "localhost:5000",
    "token_realm": "http://gitlab.domain.tld",
    "storage_delete_enabled": true,
    "health_storagedriver_enabled": true,
    "storage": {
      "filesystem": {
        "rootdirectory": "/var/opt/gitlab/gitlab-rails/shared/registry"
      },
      "cache": {
        "blobdescriptor": "inmemory"
      },
      "delete": {
        "enabled": true
      }
    }
  },
  "redis": {
    "rename_commands": {
      "KEYS": ""
    }
  },
  "postgresql": {
    "shared_buffers": "1MB",
    "internal_certificate": "",
    "connect_port": 5432,
    "wal_keep_size": 160
  },
  "gitlab-kas": {
    "api_secret_key": "NG...NmE=",
    "private_api_secret_key": "MmI3N...NzQ=",
    "gitlab_address": "http://gitlab.domain.tld"
  },
  "gitlab-pages": {
    "api_secret_key": "A1+...+rc0c="
  },
  "pgbouncer": {

  },
  "repmgr": {

  },
  "repmgrd": {

  },
  "consul": {

  },
  "spamcheck": {

  },
  "patroni": {

  },
  "gitaly": {
    "storage": [
      {
        "name": "default",
        "path": "/var/opt/gitlab/git-data/repositories"
      }
    ]
  },
  "praefect": {

  },
  "crond": {

  },
  "logrotate": {

  },
  "mattermost": {
    "email_invite_salt": "092e...e2fb",
    "file_public_link_salt": "af...1f3",
    "sql_at_rest_encrypt_key": "461...8ef38",
    "sql_data_source": "user=gitlab_mattermost host=/var/opt/gitlab/postgresql port=5432 dbname=mattermost_production"
  }
}
Converging 0 resources

Running handlers:
Running handlers complete
Chef Infra Client finished, 0/0 resources updated in 10 seconds

Any help is highly apreciated because I’m now trying to make it work since several weeks.
Cheers
fr