GitLab CE Container Registry login error

I’m running GitLab CE 16.1 and it works perfectly.

But now I’m trying to configure Container Registry.

My GitLab’s bundled NGINX is turned off.
My GitLab works with external NGINX server, running on the nearby machine in local net.
It’s ip is 192.168.30.98.
It holds SSL certificates for gitlab.myteam . com and hub.myteam . com which is the address of container registry.

Here is full gitlab.rb configuration (that was changed):
(all http and https link was modified to bypass 10 links limitation for new users.)

## GitLab URL
external_url 'http : // gitlab.myteam . com' 

### Email Settings
gitlab_rails['gitlab_email_from'] = 'service@myteam . com'
gitlab_rails['gitlab_email_reply_to'] = 'service@myteam . com'

### Trusted proxies
gitlab_rails['trusted_proxies'] = ['192.168.30.98']

### Microsoft Graph Mailer
gitlab_rails['microsoft_graph_mailer_enabled'] = true
gitlab_rails['microsoft_graph_mailer_user_id'] = "***"
gitlab_rails['microsoft_graph_mailer_tenant'] = "***"
gitlab_rails['microsoft_graph_mailer_client_id'] = "***"
gitlab_rails['microsoft_graph_mailer_client_secret'] = "***"
gitlab_rails['microsoft_graph_mailer_azure_ad_endpoint'] = "http : // login.microsoftonline . com"
gitlab_rails['microsoft_graph_mailer_graph_endpoint'] = "http : // graph.microsoft . com"

### Reply by email
gitlab_rails['incoming_email_enabled'] = true

#### Incoming Email Address
gitlab_rails['incoming_email_address'] = "gitlab-incoming+%{key}@myteam . com"

#### Inbox options (for Microsoft Graph)
gitlab_rails['incoming_email_inbox_method'] = 'microsoft_graph'
gitlab_rails['incoming_email_inbox_options'] = {
   'tenant_id': '***',
   'client_id': '***',
   'client_secret': '***',
   'poll_interval': 60  # Optional
}

### OmniAuth Settings
###! Docs: http : // docs.gitlab . com/ee/integration/omniauth.html
gitlab_rails['omniauth_enabled'] = true
gitlab_rails['omniauth_allow_single_sign_on'] = ['openid_connect']
gitlab_rails['omniauth_block_auto_created_users'] = false
gitlab_rails['omniauth_auto_link_saml_user'] = true

gitlab_rails['omniauth_providers'] = [
  {
    name: "openid_connect", # do not change this parameter
    label: "MYTEAM Office365", # optional label for login button, defaults to "Openid Connect"
    args: {
      name: "openid_connect",
      scope: ["openid", "profile", "email"],
      response_type: "code",
      issuer:  "http : // login.microsoftonline . com/***/v2.0",
      client_auth_method: "query",
      discovery: true,
      uid_field: "preferred_username",
      pkce: true,
      register_enabled: true,
      external_groups: ["MYTEAM"],
      client_options: {
        identifier: "***",
        secret: "***",
        redirect_uri: "http : // gitlab.myteam . com/users/auth/openid_connect/callback"
      }
    }
  },
  {
    name: "gitlab",
    app_id: "***",
    app_secret: "***"
  }
]

### Extra customization
 gitlab_rails['rack_attack_git_basic_auth'] = {
   'enabled' => true,
   'ip_whitelist' => ["127.0.0.1, 1.1.1.1, 2.2.2.2"],
   'maxretry' => 10,
   'findtime' => 60,
   'bantime' => 3600
 }


################################################################################
## Container Registry settings
##! Docs: http : // docs.gitlab . com/ee/administration/packages/container_registry.html
################################################################################

registry_external_url 'http : // hub.gitlab.myteam . com'

### Settings used by GitLab application
gitlab_rails['registry_enabled'] = true
gitlab_rails['registry_host'] = "hub.gitlab.myteam . com"
gitlab_rails['registry_port'] = "5005" # I tried 5757 port here (same as registry_http_addr) with no success.
# gitlab_rails['registry_path'] = "/var/opt/gitlab/gitlab-rails/shared/registry"

# Notification secret, it's used to authenticate notification requests to GitLab application
# You only need to change this when you use external Registry service, otherwise
# it will be taken directly from notification settings of your Registry
# gitlab_rails['registry_notification_secret'] = nil

###! **Do not change the following 3 settings unless you know what you are
###!   doing**
# gitlab_rails['registry_api_url'] = "http : // 127.0.0.1:5000"
# gitlab_rails['registry_key_path'] = "/var/opt/gitlab/gitlab-rails/certificate.key"
# gitlab_rails['registry_issuer'] = "omnibus-gitlab-issuer"

### Settings used by Registry application
# registry['enable'] = true
# registry['username'] = "registry"
# registry['group'] = "registry"
# registry['uid'] = nil
# registry['gid'] = nil
# registry['dir'] = "/var/opt/gitlab/registry"
registry['registry_http_addr'] = "0.0.0.0:5757"
# registry['debug_addr'] = "localhost:5001"
# registry['log_directory'] = "/var/log/gitlab/registry"
# registry['env_directory'] = "/opt/gitlab/etc/registry/env"
# registry['env'] = {
#   'SSL_CERT_DIR' => "/opt/gitlab/embedded/ssl/certs/"
# }
# registry['log_level'] = "info"
registry['log_level'] = "debug"
# registry['log_formatter'] = "text"
# registry['rootcertbundle'] = "/var/opt/gitlab/registry/certificate.crt"
# registry['health_storagedriver_enabled'] = true
# registry['middleware'] = nil
# registry['storage_delete_enabled'] = true
# registry['validation_enabled'] = false
# registry['autoredirect'] = false
# registry['compatibility_schema1_enabled'] = false

### Registry backend storage
###! Docs: http : // docs.gitlab . com/ee/administration/packages/container_registry.html#configure-storage-for-the-container-registry
# registry['storage'] = {
#   's3' => {
#     'accesskey' => 's3-access-key',
#     'secretkey' => 's3-secret-key-for-access-key',
#     'bucket' => 'your-s3-bucket',
#     'region' => 'your-s3-region',
#     'regionendpoint' => 'your-s3-regionendpoint'
#   },
#   'redirect' => {
#     'disable' => false
#   }
# }

### Registry notifications endpoints
# registry['notifications'] = [
#   {
#     'name' => 'test_endpoint',
#     'url' => 'http : // gitlab.example . com/notify2',
#     'timeout' => '500ms',
#     'threshold' => 5,
#     'backoff' => '1s',
#     'headers' => {
#       "Authorization" => ["AUTHORIZATION_EXAMPLE_TOKEN"]
#     }
#   }
# ]
### Default registry notifications
# registry['default_notifications_timeout'] = "500ms"
# registry['default_notifications_threshold'] = 5
# registry['default_notifications_backoff'] = "1s"
# registry['default_notifications_headers'] = {}


################################################################################
## GitLab Workhorse
##! Docs: http : // gitlab . com/gitlab-org/gitlab/-/blob/master/workhorse/README.md
################################################################################

gitlab_workhorse['listen_network'] = "tcp"
gitlab_workhorse['listen_addr'] = "0.0.0.0:8181"


################################################################################
## GitLab Web server
##! Docs: http : // docs.gitlab . com/omnibus/settings/nginx.html#using-a-non-bundled-web-server
################################################################################

##! When bundled nginx is disabled we need to add the external webserver user to
##! the GitLab webserver group.
 web_server['external_users'] = ['www-data']
# web_server['username'] = 'gitlab-www'
# web_server['group'] = 'gitlab-www'
# web_server['uid'] = nil
# web_server['gid'] = nil
# web_server['shell'] = '/bin/false'
# web_server['home'] = '/var/opt/gitlab/nginx'


################################################################################
## GitLab NGINX
##! Docs: http : // docs.gitlab . com/omnibus/settings/nginx.html
################################################################################

nginx['enable'] = false
nginx['client_max_body_size'] = '250m'
nginx['redirect_http_to_https'] = false
# nginx['redirect_http_to_https_port'] = 80

##! Most root CA's are included by default
# nginx['ssl_client_certificate'] = "/etc/gitlab/ssl/ca.crt"

##! enable/disable 2-way SSL client authentication
# nginx['ssl_verify_client'] = "off"

##! if ssl_verify_client on, verification depth in the client certificates chain
# nginx['ssl_verify_depth'] = "1"

# nginx['ssl_certificate'] = "/etc/gitlab/ssl/#{node['fqdn']}.crt"
# nginx['ssl_certificate_key'] = "/etc/gitlab/ssl/#{node['fqdn']}.key"
# nginx['ssl_ciphers'] = "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY13>
# nginx['ssl_prefer_server_ciphers'] = "off"

##! **Recommended by: http : // raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html
##!                   http : // cipherli.st/**
# nginx['ssl_protocols'] = "TLSv1.2 TLSv1.3"

##! **Recommended in: http : // nginx.org/en/docs/http/ngx_http_ssl_module.html**
# nginx['ssl_session_cache'] = "shared:SSL:10m"

##! **Recommended in: http : // ssl-config.mozilla.org/#server=nginx&version=1.17.7&config=intermediate&openssl=1.1.1d&ocsp=false&guideline=5.6**
# nginx['ssl_session_tickets'] = "off"

##! **Default according to http : // nginx.org/en/docs/http/ngx_http_ssl_module.html**
# nginx['ssl_session_timeout'] = "1d"

# nginx['ssl_dhparam'] = nil # Path to dhparams.pem, eg. /etc/gitlab/ssl/dhparams.pem
# nginx['ssl_password_file'] = nil # Path to file with passphrases for ssl certificate secret keys
# nginx['listen_addresses'] = ['*', '[::]']

##! **Defaults to forcing web browsers to always communicate using only HTTPS**
##! Docs: http : // docs.gitlab . com/omnibus/settings/nginx.html#setting-http-strict-transport-security
# nginx['hsts_max_age'] = 63072000
# nginx['hsts_include_subdomains'] = false

##! Defaults to stripping path information when making cross-origin requests
# nginx['referrer_policy'] = 'strict-origin-when-cross-origin'

##! **Docs: http : // nginx.org/en/docs/http/ngx_http_gzip_module.html**
# nginx['gzip_enabled'] = true

##! **Override only if you use a reverse proxy**
##! Docs: http : // docs.gitlab . com/omnibus/settings/nginx.html#setting-the-nginx-listen-port
# nginx['listen_port'] = nil

##! **Override only if your reverse proxy internally communicates over HTTP**
##! Docs: http : // docs.gitlab . com/omnibus/settings/nginx.html#supporting-proxied-ssl
# nginx['listen_https'] = nil

##! **Override only if you use a reverse proxy with proxy protocol enabled**
##! Docs: http : // docs.gitlab . com/omnibus/settings/nginx.html#configuring-proxy-protocol
# nginx['proxy_protocol'] = false

# nginx['custom_gitlab_server_config'] = "location ^~ /foo-namespace/bar-project/raw/ {\n deny all;\n}\n"
# nginx['custom_nginx_config'] = "include /etc/nginx/conf.d/example.conf;"
# nginx['proxy_read_timeout'] = 3600
# nginx['proxy_connect_timeout'] = 300
# nginx['proxy_set_headers'] = {
#  "Host" => "$http_host_with_default",
#  "X-Real-IP" => "$remote_addr",
#  "X-Forwarded-For" => "$proxy_add_x_forwarded_for",
#  "X-Forwarded-Proto" => "https",
#  "X-Forwarded-Ssl" => "on",
#  "Upgrade" => "$http_upgrade",
#  "Connection" => "$connection_upgrade"
# }
# nginx['proxy_cache_path'] = 'proxy_cache keys_zone=gitlab:10m max_size=1g levels=1:2'
# nginx['proxy_cache'] = 'gitlab'
# nginx['proxy_custom_buffer_size'] = '4k'
# nginx['http2_enabled'] = true
# nginx['real_ip_trusted_addresses'] = []
# nginx['real_ip_header'] = nil
# nginx['real_ip_recursive'] = nil
# nginx['custom_error_pages'] = {
#   '404' => {
#     'title' => 'Example title',
#     'header' => 'Example header',
#     'message' => 'Example message'
#   }
# }

### Advanced settings
# nginx['dir'] = "/var/opt/gitlab/nginx"
# nginx['log_directory'] = "/var/log/gitlab/nginx"
# nginx['log_group'] = nil
# nginx['error_log_level'] = "error"
# nginx['worker_processes'] = 4
# nginx['worker_connections'] = 10240
# nginx['log_format'] = '$remote_addr - $remote_user [$time_local] "$request_method $filtered_request_uri $server_protocol" $status $body_bytes_sent "$filtered_http_refe>
# nginx['sendfile'] = 'on'
# nginx['tcp_nopush'] = 'on'
# nginx['tcp_nodelay'] = 'on'
# nginx['hide_server_tokens'] = 'off'
# nginx['gzip_http_version'] = "1.0"
# nginx['gzip_comp_level'] = "2"
# nginx['gzip_proxied'] = "any"
# nginx['gzip_types'] = [ "text/plain", "text/css", "application/x-javascript", "text/xml", "application/xml", "application/xml+rss", "text/javascript", "application/jso>
# nginx['keepalive_timeout'] = 65
# nginx['keepalive_time'] = '1h'
# nginx['cache_max_size'] = '5000m'
# nginx['server_names_hash_bucket_size'] = 64
##! These paths have proxy_request_buffering disabled
# nginx['request_buffering_off_path_regex'] = "/api/v\\d/jobs/\\d+/artifacts$|/import/gitlab_project$|\\.git/git-receive-pack$|\\.git/gitlab-lfs/objects|\\.git/info/lfs/>

### Nginx status
# nginx['status'] = {
#  "enable" => true,
#  "listen_addresses" => ["127.0.0.1"],
#  "fqdn" => "dev.example . com",
#  "port" => 9999,
#  "vts_enable" => true,
#  "options" => {
#    "server_tokens" => "off", # Don't show the version of NGINX
#    "access_log" => "off", # Disable logs for stats
#    "allow" => "127.0.0.1", # Only allow access from localhost
#    "deny" => "all" # Deny access to anyone else
#  }
# }

##! Service name used to register Nginx as a Consul service
# nginx['consul_service_name'] = 'nginx'
##! Semantic metadata used when registering NGINX as a Consul service
# nginx['consul_service_meta'] = {}

################################################################################
## Registry NGINX
################################################################################

# All the settings defined in the "GitLab Nginx" section are also available in
# this "Registry NGINX" section, using the key `registry_nginx`.  However, those
# settings should be explicitly set. That is, settings given as
# `nginx['some_setting']` WILL NOT be automatically replicated as
# `registry_nginx['some_setting']` and should be set separately.

# Below you can find settings that are exclusive to "Registry NGINX"
# registry_nginx['enable'] = false

# registry_nginx['proxy_set_headers'] = {
#  "Host" => "$http_host",
#  "X-Real-IP" => "$remote_addr",
#  "X-Forwarded-For" => "$proxy_add_x_forwarded_for",
#  "X-Forwarded-Proto" => "https",
#  "X-Forwarded-Ssl" => "on"
# }

# When the registry is automatically enabled using the same domain as `external_url`,
# it listens on this port
# registry_nginx['listen_port'] = 5050

All other settings are in default state.

And here is NGINX (192.168.30.98) config:

# 10.1 - gitlab.myteam . com
upstream gitlab_myteam_com {
        server 192.168.30.54:8181;
}

# 10.2 - hub.gitlab.myteam . com
upstream hub_gitlab_myteam_com {
        server 192.168.30.54:5757;
}

# 10.1 - gitlab.myteam . com

server {
        listen 443 ssl http2;
        server_name gitlab.myteam . com;

        # this file contains paths to certificate and key for *.myteam . com
        include /etc/nginx/myteam . com_certs.conf;

#        ssl_certificate /etc/letsencrypt/live/myteam . com/fullchain.pem;
#        ssl_certificate_key /etc/letsencrypt/live/myteam . com/privkey.pem;

        location / {
                proxy_pass http : // gitlab_myteam_com;
                proxy_set_header Host $host;
                proxy_set_header X-Real-IP $remote_addr;
                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
                proxy_set_header X-Forwarded-Proto $scheme;
        }
}

server {
        listen 80;
        server_name gitlab.myteam . com;
        return 301 http : // $host$request_uri;
}

# 10.2 - hub.gitlab.myteam . com

server {
        listen 443 ssl http2;
        server_name hub.gitlab.myteam . com;

        client_max_body_size 250m;

        ssl_certificate /etc/letsencrypt/live/hub.gitlab.myteam . com/fullchain.pem;
        ssl_certificate_key /etc/letsencrypt/live/hub.gitlab.myteam . com/privkey.pem;


        location / {
                proxy_pass http : // hub_gitlab_myteam_com;
                proxy_set_header Host $host;
                proxy_set_header X-Real-IP $remote_addr;
                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
                proxy_set_header X-Forwarded-Proto $scheme;
                proxy_read_timeout 900;
        }
}

server {
        listen 80;
        server_name hub.gitlab.myteam . com;
        return 301 http : // $host$request_uri;
}

Here is “sudo lsof -i -P -n | grep LISTEN” output on GitLab server:

systemd-r    664   systemd-resolve   14u  IPv4   38434      0t0  TCP 127.0.0.53:53 (LISTEN)
sshd         729              root    3u  IPv4   39647      0t0  TCP *:22 (LISTEN)
sshd         729              root    4u  IPv6   39649      0t0  TCP *:22 (LISTEN)
gitlab-wo    948               git    6u  IPv4   40166      0t0  TCP 127.0.0.1:9229 (LISTEN)
gitlab-wo    948               git    8u  IPv6   40187      0t0  TCP *:8181 (LISTEN)
alertmana    960 gitlab-prometheus    3u  IPv6   39929      0t0  TCP *:9094 (LISTEN)
alertmana    960 gitlab-prometheus    8u  IPv4   38841      0t0  TCP 127.0.0.1:9093 (LISTEN)
gitlab-ka    961               git    7u  IPv4   40172      0t0  TCP 127.0.0.1:8153 (LISTEN)
gitlab-ka    961               git    8u  IPv4   40180      0t0  TCP 127.0.0.1:8154 (LISTEN)
gitlab-ka    961               git    9u  IPv4   40173      0t0  TCP 127.0.0.1:8151 (LISTEN)
gitlab-ka    961               git   10u  IPv4   40174      0t0  TCP 127.0.0.1:8150 (LISTEN)
gitlab-ka    961               git   11u  IPv4   40175      0t0  TCP 127.0.0.1:8155 (LISTEN)
prometheu    966 gitlab-prometheus    7u  IPv4   41201      0t0  TCP 127.0.0.1:9090 (LISTEN)
postgres_    967       gitlab-psql    3u  IPv4   39918      0t0  TCP 127.0.0.1:9187 (LISTEN)
node_expo    973 gitlab-prometheus    3u  IPv4   39920      0t0  TCP 127.0.0.1:9100 (LISTEN)
gitlab-ex    978               git    5u  IPv4   40193      0t0  TCP 127.0.0.1:9168 (LISTEN)
redis_exp    985      gitlab-redis    3u  IPv4   40158      0t0  TCP 127.0.0.1:9121 (LISTEN)
gitaly      1034               git    9u  IPv4   42067      0t0  TCP 127.0.0.1:9236 (LISTEN)
gitaly      1034               git   10u  IPv4   42067      0t0  TCP 127.0.0.1:9236 (LISTEN)
ruby      795234               git   16u  IPv4 5039010      0t0  TCP 127.0.0.1:8092 (LISTEN)
ruby      795236               git   10u  IPv4 5035817      0t0  TCP 127.0.0.1:8082 (LISTEN)
ruby      795249               git   18u  IPv4 5040992      0t0  TCP 127.0.0.1:8080 (LISTEN)
registry  795339          registry    8u  IPv6 5037781      0t0  TCP *:5757 (LISTEN)
ruby      795422               git   18u  IPv4 5040992      0t0  TCP 127.0.0.1:8080 (LISTEN)
ruby      795424               git   18u  IPv4 5040992      0t0  TCP 127.0.0.1:8080 (LISTEN)
ruby      795426               git   18u  IPv4 5040992      0t0  TCP 127.0.0.1:8080 (LISTEN)
ruby      795428               git   18u  IPv4 5040992      0t0  TCP 127.0.0.1:8080 (LISTEN)

As you see, registry is on *.5757 port.

So, here is the issue.
While trying to “docker login” using this commands:

docker login http : // hub.gitlab.myteam . com -u a.user@myteam . com -p ***
docker login http : // hub.gitlab.myteam . com -u a.user -p ***
docker login http : // hub.gitlab.myteam . com -u root -p ***

I got this:

Error response from daemon: Get "http : // hub.gitlab.myteam . com/v2/": denied: access forbidden

No matter what user I use (internal, from Office365 or even root).

Here are logs from “/var/log/gitlab/gitlab-registry/current” logged right when i tried “docker login”:

2023-09-01_16:11:53.07082 time="2023-09-01T19:11:53.070+03:00" level=debug msg="filesystem.Stat(\"/\")" environment=production go_version=go1.19.8 instance_id=e0f2d50e-8ac6-429e-80da-3340b1e59d07 service=registry trace_duration="54.302µs" trace_file=/var/cache/omnibus/src/registry/src/github . com/docker/distribution/registry/storage/driver/base/base.go trace_func="github . com/docker/distribution/registry/storage/driver/base.(*Base).Stat" trace_id=4ca82ed4-0eba-4ed3-92fb-a7cab2dc1ef5 trace_line=155 version=v3.76.0-gitlab

2023-09-01_16:12:03.07104 time="2023-09-01T19:12:03.070+03:00" level=debug msg="filesystem.Stat(\"/\")" environment=production go_version=go1.19.8 instance_id=e0f2d50e-8ac6-429e-80da-3340b1e59d07 service=registry trace_duration="50.402µs" trace_file=/var/cache/omnibus/src/registry/src/github . com/docker/distribution/registry/storage/driver/base/base.go trace_func="github . com/docker/distribution/registry/storage/driver/base.(*Base).Stat" trace_id=248dc62b-8ef0-4637-992c-6a10a00f576b trace_line=155 version=v3.76.0-gitlab

2023-09-01_16:12:11.29728 time="2023-09-01T19:12:11.297+03:00" level=info msg="router info" config_http_addr="0.0.0.0:5757" config_http_host= config_http_net= config_http_prefix= config_http_relative_urls=false correlation_id=01H98R8B5162MP3PTMNC3DMSWW go_version=go1.19.8 method=GET path=/v2/ root_repo= router=gorilla/mux version=v3.76.0-gitlab

2023-09-01_16:12:11.29733 time="2023-09-01T19:12:11.297+03:00" level=debug msg="authorizing request" correlation_id=01H98R8B5162MP3PTMNC3DMSWW go_version=go1.19.8 root_repo= version=v3.76.0-gitlab

2023-09-01_16:12:11.29751 {"content_type":"application/json","correlation_id":"01H98R8B5162MP3PTMNC3DMSWW","duration_ms":0,"host":"hub.gitlab.myteam . com","level":"info","method":"GET","msg":"access","proto":"HTTP/1.0","referrer":"","remote_addr":"192.168.30.98:58974","remote_ip":"192.168.30.98","status":401,"system":"http","time":"2023-09-01T19:12:11.297+03:00","ttfb_ms":0,"uri":"/v2/","user_agent":"docker/20.10.18 go/go1.18.6 git-commit/e42327a kernel/5.4.0-156-generic os/linux arch/amd64 UpstreamClient(Docker-Client/20.10.18 \\(linux\\))","written_bytes":87}

2023-09-01_16:12:13.07124 time="2023-09-01T19:12:13.071+03:00" level=debug msg="filesystem.Stat(\"/\")" environment=production go_version=go1.19.8 instance_id=e0f2d50e-8ac6-429e-80da-3340b1e59d07 service=registry trace_duration="65.003µs" trace_file=/var/cache/omnibus/src/registry/src/github . com/docker/distribution/registry/storage/driver/base/base.go trace_func="github . com/docker/distribution/registry/storage/driver/base.(*Base).Stat" trace_id=cf1ba3e5-cb1f-44e9-8788-4338e4b1af85 trace_line=155 version=v3.76.0-gitlab

Here are related NGINX access log (there are no error logs):

192.168.30.140 - - [01/Sep/2023:21:04:21 +0300] "GET /v2/ HTTP/1.1" 401 87 "-" "docker/20.10.18 go/go1.18.6 git-commit/e42327a kernel/5.4.0-156-generic os/linux arch/amd64 UpstreamClient(Docker-Client/20.10.18 \x5C(linux\x5C))" "-"
192.168.30.140 - root [01/Sep/2023:21:04:21 +0300] "GET /jwt/auth?account=root&client_id=docker&offline_token=true&service=container_registry HTTP/1.1" 301 169 "-" "docker/20.10.18 go/go1.18.6 git-commit/e42327a kernel/5.4.0-156-generic os/linux arch/amd64 UpstreamClient(Docker-Client/20.10.18 \x5C(linux\x5C))" "-"
192.168.30.140 - - [01/Sep/2023:21:04:21 +0300] "GET /jwt/auth?account=root&client_id=docker&offline_token=true&service=container_registry HTTP/1.1" 403 77 "http : // gitlab.myteam . com/jwt/auth?account=root&client_id=docker&offline_token=true&service=container_registry" "docker/20.10.18 go/go1.18.6 git-commit/e42327a kernel/5.4.0-156-generic os/linux arch/amd64 UpstreamClient(Docker-Client/20.10.18 \x5C(linux\x5C))" "-"

Here is curl from client:

curl http : // hub.gitlab.myteam . com
(no answer)

curl http : // hub.gitlab.myteam . com/v2
<a href="/v2/">Moved Permanently</a>.

curl http : // hub.gitlab.myteam . com/v2/
{"errors":[{"code":"UNAUTHORIZED","message":"authentication required","detail":null}]}

The most strange part.

If I do this:

curl -u "a.user:users_token" "http : // gitlab.myteam . com/jwt/auth?service=container_registry"

I recieve token!

And after that, if I use that token in stright request:

curl -I -H "Authorization: Bearer token_here" http : // 192.168.30.54:5757/v2/

I got this:

HTTP/1.1 200 OK
Content-Length: 2
Content-Type: application/json
Docker-Distribution-Api-Version: registry/2.0
Gitlab-Container-Registry-Features: tag_delete
Gitlab-Container-Registry-Version: 3.76.0-gitlab
X-Content-Type-Options: nosniff
Date: Fri, 01 Sep 2023 18:31:28 GMT

Also, I tried to configure container registry on the same domain name as gitlab server:

registry_external_url 'http : // gitlab.myteam . com'
gitlab_rails['registry_enabled'] = true
gitlab_rails['registry_host'] = "gitlab.myteam . com"
gitlab_rails['registry_port'] = "5757" # Default was 5005, nothing changed. 
# gitlab_rails['registry_path'] = "/var/opt/gitlab/gitlab-rails/shared/registry"

###! **Do not change the following 3 settings unless you know what you are
###!   doing**
# gitlab_rails['registry_api_url'] = "http : // 127.0.0.1:5000"
# gitlab_rails['registry_key_path'] = "/var/opt/gitlab/gitlab-rails/certificate.key"
# gitlab_rails['registry_issuer'] = "omnibus-gitlab-issuer"

registry['registry_http_addr'] = "0.0.0.0:5757"

And with this NGINX setup:

# 10.1 - gitlab.myteam . com
upstream gitlab_myteam_dev {
        server 192.168.30.54:8181;
}

# 10.2 - hub.gitlab.myteam . com
upstream gitlab_registry {
        server 192.168.30.54:5757;
}

# 10.1 - gitlab.myteam . com

server {
        listen 443 ssl http2;
        server_name gitlab.myteam . com;

        # this file contains paths to certificate and key for myteam . com
        include /etc/nginx/myteam . com_certs.conf;

#        ssl_certificate /etc/letsencrypt/live/myteam . com/fullchain.pem;
#        ssl_certificate_key /etc/letsencrypt/live/myteam . com/privkey.pem;

        # Location block for gitlab.myteam . com
        location / {
                proxy_pass http : // gitlab_myteam_dev;
                proxy_set_header Host $host;
                proxy_set_header X-Real-IP $remote_addr;
                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
                proxy_set_header X-Forwarded-Proto $scheme;
        }

        # Location block for the GitLab Container Registry
        location ~ ^/v2/ {
                proxy_pass http : // gitlab_registry;
                proxy_set_header Host $host;
                proxy_set_header X-Real-IP $remote_addr;
                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
                proxy_set_header X-Forwarded-Proto $scheme;
                proxy_set_header Docker-Distribution-Api-Version registry/2.0;
                proxy_read_timeout 900;
        }
}

server {
        listen 80;
        server_name gitlab.myteam . com;
        return 301 http : // $host$request_uri;
}

That does not helps. Result was the same.

Which token type is used for authorization?

GitLab 16 changed some authorization methods, and forbids extern auth by default (see red banner in the docs).

Well, that was said, “external authorization prevents personal access tokens and deploy tokens from accessing container and package registries”.

That’s clear.

But I tried to use Group Token (GAT) and Project Token (PAT) with same result:

docker login gitlab.myteam.com -u a.user
Password: *GAT*
Error response from daemon: Get "https://gitlab.myteam.com/v2/": denied: access forbidden

docker login gitlab.myteam.com -u a.user@myteam.team
Password: *GAT*
Error response from daemon: Get "https://gitlab.myteam.com/v2/": denied: access forbidden

docker login https://gitlab.myteam.com/groups/myteam -u a.user@myteam.team
Password: *GAT*
Error response from daemon: Get "https://gitlab.myteam.com/v2/": denied: access forbidden

docker login https://gitlab.myteam.com/a.user/cert_api -u a.user@myteam.team
Password: *PAT*
Error response from daemon: Get "https://gitlab.myteam.com/v2/": denied: access forbidden

docker login https://gitlab.myteam.com -u a.user@myteam.team
Password: *PAT*
Error response from daemon: Get "https://gitlab.myteam.com/v2/": denied: access forbidden

Also, I tried to disable External Authorization (by clearing “Allow deploy tokens and deploy keys to be used with external authorization” checkbox).

It didn’t help.

Maybe there is other way to disable External Authorization?
There is nothing in Docs about it.

Any suggestions?
I still can’t find solution.
Please, help.

Hello @akanil !

I’m facing exactly the same issue.

I have a GitLab instance and a GitLab container registry under my GitLab domain, but when I execute docker login it gives me access forbidden. Have you solved this issue?

just pinging @dnsmichi to help us

I don’t know what exactly I did, but it works now.
I think the issue was in external url, but I can’t check it right now - my server have just broken.
Tomorrow I will bring it up and check my config.

Nice. Thank u for your quickly response. If you can tell me what you did to work it will help me a lot.

My instance started to break when my teammates and me implemented HTTPS in GitLab Container Registry.

first check youк externa (base) URL.
as i remember, it should also have httpS.
for example - httpS://gitlab.myteam.com
and registry URL should ALSO have httpS.
https is good practice, but needs a bit more complex configuration.

Maybe it can be the problem so, my external url base is not https, because I implementend it just in my nginx. I will try it as soon as possible

Some other things, if you are not using https for the registry, or https with self-signed certificates, docker needs to be configured to use insecure registries. The docker documentation shows how you can configure this in /etc/docker/daemon.json.

@iwalker I want to use https on my registry, so your suggest is not userful for me, but really thanks. It can be helpful for another people.

@akanil Can you share the content of your gitlab.rb? then can i see what is wrong on my configuration.

Another question, do you have your htttps certificate under the nginx or under the gitlab instance?

@gustavosimon here is it.

## GitLab URL
##! URL on which GitLab will be reachable.
##! For more details on configuring external_url see:
##! https : / / docs.gitlab.com/omnibus/settings/configuration.html#configuring-the-external-url-for-gitlab
##!
##! Note: During installation/upgrades, the value of the environment variable
##! EXTERNAL_URL will be used to populate/replace this value.
##! On AWS EC2 instances, we also attempt to fetch the public hostname/IP
##! address from AWS. For more details, see:
##! https : / / docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html
external_url 'https : / / gitlab.myteam.com'



################################################################################
################################################################################
##                Configuration Settings for GitLab CE and EE                 ##
################################################################################
################################################################################

################################################################################
## gitlab.yml configuration
##! Docs: https : / / gitlab.com/gitlab-org/omnibus-gitlab/blob/master/doc/settings/gitlab.yml.md
################################################################################
# gitlab_rails['gitlab_ssh_host'] = 'ssh.host_example.com'
# gitlab_rails['gitlab_ssh_user'] = ''
# gitlab_rails['time_zone'] = 'UTC'

### Email Settings

# gitlab_rails['gitlab_email_enabled'] = true

##! If your SMTP server does not like the default 'From: gitlab@gitlab.example.com'
##! can change the 'From' with this setting.
gitlab_rails['gitlab_email_from'] = 'service@myteam.com'
# gitlab_rails['gitlab_email_display_name'] = 'Example'
gitlab_rails['gitlab_email_reply_to'] = 'service@myteam.com'
# gitlab_rails['gitlab_email_subject_suffix'] = ''
# gitlab_rails['gitlab_email_smime_enabled'] = false
# gitlab_rails['gitlab_email_smime_key_file'] = '/etc/gitlab/ssl/gitlab_smime.key'
# gitlab_rails['gitlab_email_smime_cert_file'] = '/etc/gitlab/ssl/gitlab_smime.crt'
# gitlab_rails['gitlab_email_smime_ca_certs_file'] = '/etc/gitlab/ssl/gitlab_smime_cas.crt'


### Trusted proxies
###! Customize if you have GitLab behind a reverse proxy which is running on a
###! different machine.
###! **Add the IP address for your reverse proxy to the list, otherwise users
###!   will appear signed in from that address.**
gitlab_rails['trusted_proxies'] = ['192.168.30.98']


### Microsoft Graph Mailer
###! Allows delivery of emails using Microsoft Graph API with OAuth 2.0 client
###! credentials flow.
###! Docs: https : / / docs.gitlab.com/omnibus/settings/microsoft_graph_mailer.html
gitlab_rails['microsoft_graph_mailer_enabled'] = true
gitlab_rails['microsoft_graph_mailer_user_id'] = "***"
gitlab_rails['microsoft_graph_mailer_tenant'] = "****"
gitlab_rails['microsoft_graph_mailer_client_id'] = "***"
gitlab_rails['microsoft_graph_mailer_client_secret'] = "***"
gitlab_rails['microsoft_graph_mailer_azure_ad_endpoint'] = "https : / / login.microsoftonline.com"
gitlab_rails['microsoft_graph_mailer_graph_endpoint'] = "https : / / graph.microsoft.com"

### Reply by email
###! Allow users to comment on issues and merge requests by replying to
###! notification emails.
###! Docs: https : / / docs.gitlab.com/ee/administration/reply_by_email.html
gitlab_rails['incoming_email_enabled'] = true

#### Incoming Email Address
####! The email address including the `%{key}` placeholder that will be replaced
####! to reference the item being replied to.
####! **The placeholder can be omitted but if present, it must appear in the
####!   "user" part of the address (before the `@`).**
# gitlab_rails['incoming_email_address'] = "gitlab-incoming+%{key}@gmail.com"
gitlab_rails['incoming_email_address'] = "gitlab-incoming+%{key}@myteam.com"


#### Inbox options (for Microsoft Graph)
gitlab_rails['incoming_email_inbox_method'] = 'microsoft_graph'
gitlab_rails['incoming_email_inbox_options'] = {
   'tenant_id': '***',
   'client_id': '***',
   'client_secret': '***',
   'poll_interval': 60  # Optional
}


### OmniAuth Settings
###! Docs: https : / / docs.gitlab.com/ee/integration/omniauth.html
gitlab_rails['omniauth_enabled'] = true
# gitlab_rails['omniauth_allow_single_sign_on'] = ['azure_oauth2']
gitlab_rails['omniauth_allow_single_sign_on'] = ['openid_connect']
# gitlab_rails['omniauth_sync_email_from_provider'] = 'saml'
# gitlab_rails['omniauth_sync_profile_from_provider'] = ['saml']
# gitlab_rails['omniauth_sync_profile_attributes'] = ['email']
# gitlab_rails['omniauth_auto_sign_in_with_provider'] = 'saml'
gitlab_rails['omniauth_block_auto_created_users'] = false
# gitlab_rails['omniauth_auto_link_ldap_user'] = false
gitlab_rails['omniauth_auto_link_saml_user'] = true
# gitlab_rails['omniauth_auto_link_user'] = ['twitter']
# gitlab_rails['omniauth_external_providers'] = ['twitter', 'google_oauth2']
# gitlab_rails['omniauth_allow_bypass_two_factor'] = ['google_oauth2']
#maybe this line is needed# gitlab_rails['omniauth_auto_link_azure_oauth2_user'] = true

gitlab_rails['omniauth_providers'] = [
  {
    name: "openid_connect", # do not change this parameter
    label: "MYTEAM Office365", # optional label for login button, defaults to "Openid Connect"
    args: {
      name: "openid_connect",
      scope: ["openid", "profile", "email"],
      response_type: "code",
      issuer:  "https : / / login.microsoftonline.com/***/v2.0",
      client_auth_method: "query",
      discovery: true,
      uid_field: "preferred_username",
      pkce: true,
      register_enabled: true,
      external_groups: ["MYTEAM"],
      client_options: {
        identifier: "***",
        secret: "***",
        redirect_uri: "https : / / gitlab.myteam.com/users/auth/openid_connect/callback"
      }
    }
  },
  {
    name: "gitlab",
    app_id: "***",
    app_secret: "***"
  }
]


##! Docs: https : / / docs.gitlab.com/omnibus/settings/environment-variables.html
# gitlab_rails['env'] = {
#   'BUNDLE_GEMFILE' => "/opt/gitlab/embedded/service/gitlab-rails/Gemfile",
#   'PATH' => "/opt/gitlab/bin:/opt/gitlab/embedded/bin:/bin:/usr/bin"
# }

 gitlab_rails['rack_attack_git_basic_auth'] = {
   'enabled' => true,
   'ip_whitelist' => ["127.0.0.1, *.*.*.*"],
   'maxretry' => 10,
   'findtime' => 60,
   'bantime' => 3600
 }



################################################################################
## Container Registry settings
##! Docs: https : / / docs.gitlab.com/ee/administration/packages/container_registry.html
################################################################################

registry_external_url 'https : / / registry.gitlab.myteam.com'

### Settings used by GitLab application
gitlab_rails['registry_enabled'] = true
gitlab_rails['registry_host'] = "registry.gitlab.myteam.com"
gitlab_rails['registry_port'] = "5005"
# gitlab_rails['registry_path'] = "/var/opt/gitlab/gitlab-rails/shared/registry"

# Notification secret, it's used to authenticate notification requests to GitLab application
# You only need to change this when you use external Registry service, otherwise
# it will be taken directly from notification settings of your Registry
# gitlab_rails['registry_notification_secret'] = nil

###! **Do not change the following 3 settings unless you know what you are
###!   doing**
# gitlab_rails['registry_api_url'] = "http : / / 127.0.0.1:5000"
# gitlab_rails['registry_key_path'] = "/var/opt/gitlab/gitlab-rails/certificate.key"
# gitlab_rails['registry_issuer'] = "omnibus-gitlab-issuer"

### Settings used by Registry application
registry['enable'] = true
# registry['username'] = "registry"
# registry['group'] = "registry"
# registry['uid'] = nil
# registry['gid'] = nil
# registry['dir'] = "/var/opt/gitlab/registry"
registry['registry_http_addr'] = "0.0.0.0:5000"
# registry['debug_addr'] = "localhost:5001"
# registry['log_directory'] = "/var/log/gitlab/registry"
# registry['env_directory'] = "/opt/gitlab/etc/registry/env"
# registry['env'] = {
#   'SSL_CERT_DIR' => "/opt/gitlab/embedded/ssl/certs/"
# }
# registry['log_level'] = "info"
registry['log_level'] = "debug"
# registry['log_formatter'] = "text"
# registry['rootcertbundle'] = "/var/opt/gitlab/registry/certificate.crt"
# registry['health_storagedriver_enabled'] = true
# registry['middleware'] = nil
# registry['storage_delete_enabled'] = true
# registry['validation_enabled'] = false
# registry['autoredirect'] = false
# registry['compatibility_schema1_enabled'] = false


################################################################################
## GitLab Workhorse
##! Docs: https : / / gitlab.com/gitlab-org/gitlab/-/blob/master/workhorse/README.md
################################################################################

# gitlab_workhorse['enable'] = true
# gitlab_workhorse['ha'] = false
# gitlab_workhorse['alt_document_root'] = nil

##! Duration to wait for all requests to finish (e.g. "10s" for 10
##! seconds). By default this is disabled to preserve the existing
##! behavior of fast shutdown. This should not be set higher than 30
##! seconds, since gitlab-ctl will wait up to 30 seconds (as defined by
##! the SVWAIT variable) and report a timeout error if the process has
##! not shut down.
# gitlab_workhorse['shutdown_timeout'] = nil
# gitlab_workhorse['listen_network'] = "unix"
gitlab_workhorse['listen_network'] = "tcp"
# gitlab_workhorse['listen_umask'] = 000
# gitlab_workhorse['listen_addr'] = "/var/opt/gitlab/gitlab-workhorse/sockets/socket"
gitlab_workhorse['listen_addr'] = "0.0.0.0:8181"
# gitlab_workhorse['auth_backend'] = "http : / / localhost:8080"



################################################################################
## GitLab Web server
##! Docs: https : / / docs.gitlab.com/omnibus/settings/nginx.html#using-a-non-bundled-web-server
################################################################################

##! When bundled nginx is disabled we need to add the external webserver user to
##! the GitLab webserver group.
 web_server['external_users'] = ['www-data']
# web_server['username'] = 'gitlab-www'
# web_server['group'] = 'gitlab-www'
# web_server['uid'] = nil
# web_server['gid'] = nil
# web_server['shell'] = '/bin/false'
# web_server['home'] = '/var/opt/gitlab/nginx'

################################################################################
## GitLab NGINX
##! Docs: https : / / docs.gitlab.com/omnibus/settings/nginx.html
################################################################################

nginx['enable'] = false
nginx['client_max_body_size'] = '250m'
nginx['redirect_http_to_https'] = false
# nginx['redirect_http_to_https_port'] = 80

All other settings are in default state.

Keep in mind, that embedded Nginx is turned off.
I’m using external Nginx to publish Gitlab and Registry.
It holds certificates for all domains.

Thank you so much! I think that with it I can solve my problem.

@akanil it would be really nice, if you share your nginx config file too, if you can.

I can’t investigate the problem yet, but it will be super helpful too

Sure. Here it is:

# gitlab.myteam.com
upstream gitlab_myteam_com {
        server 192.168.30.54:8181;
}

# registry.gitlab.myteam.com
upstream registry_gitlab_myteam_com {
        server 192.168.30.54:5000;
}


# gitlab.myteam.com

server {
        listen 443 ssl http2;
        server_name gitlab.myteam.com;

        # this file contains paths to certificate and key for myteam.com
        include /etc/nginx/myteam.com_certs.conf;

        # Location block for gitlab.myteam.com
        location / {
                proxy_pass http://gitlab_myteam_com;
                proxy_set_header Host $host;
                proxy_set_header X-Real-IP $remote_addr;
                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
                proxy_set_header X-Forwarded-Proto $scheme;
        }


server {
        listen 80;
        server_name gitlab.myteam.com;
        return 301 https://$host$request_uri;
}


# registry.gitlab.myteam.com

server {
        listen 443 ssl http2;
        server_name registry.gitlab.myteam.com;

        client_max_body_size 500m;

        ssl_certificate /etc/letsencrypt/live/registry.gitlab.myteam.com/fullchain.pem;
        ssl_certificate_key /etc/letsencrypt/live/registry.gitlab.myteam.com/privkey.pem;

        location / {
                proxy_pass http://registry_gitlab_myteam_com;
                proxy_set_header Host $host;
                proxy_set_header X-Real-IP $remote_addr;
                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
                proxy_set_header X-Forwarded-Proto $scheme;
#               proxy_set_header Docker-Distribution-Api-Version registry/2.0; # test setting
                proxy_read_timeout 900;
        }
}

server {
        listen 80;
        server_name registry.gitlab.myteam.com;
        return 301 https://$host$request_uri;
}

Hello @akanil . Thanks for sending your nginx.conf. I cannot fix my issue yet. Could you share your docker-compose.yml file? If you have it, of course.