GitLab Pipeline Push to GitLab Registry ERROR: error parsing HTTP 404 response body: invalid character '<' looking for beginning of value:

asdasdasd · April 21, 2022, 4:29am

GitLab Pipeline Push to GitLab Registry ERROR: error parsing HTTP 404 response body: invalid character ‘<’ looking for beginning of value:

Hi, I deployed GitLab and GitLab Runner on VM inside an Intranet environment behind HTTP Proxy and Custom CA Cert.

Everything works well but the CI pipeline sometimes able to push to GitLab Registry, mostly cannot with error:

#9 writing image sha256:c6e9dd2b86c6b96b5febce2e07ff90eb600e4534422111d22982445c4ef5c4ab done
#9 naming to registry.intranet.domain-name/gitlab-ci-example/test-ssl-registry/master:441e5771392f91334220e23bf657632ea13c1f12 done
#9 naming to registry.intranet.domain-name/gitlab-ci-example/test-ssl-registry/master:latest done
#9 DONE 0.0s
#10 pushing registry.intranet.domain-name/gitlab-ci-example/test-ssl-registry/master:441e5771392f91334220e23bf657632ea13c1f12 with docker
#10 pushing layer 9d830bd22c50
#10 pushing layer 208e467e5f77
#10 pushing layer 36ffdceb4c77
#10 pushing layer 9d830bd22c50 0.2s done
#10 pushing layer 208e467e5f77 0.2s done
#10 pushing layer 36ffdceb4c77 0.2s done
#10 ERROR: error parsing HTTP 404 response body: invalid character '<' looking for beginning of value: "<!DOCTYPE html>\n<html>\n<head>\n  <meta content=\"width=device-width, initial-scale=1, maximum-scale=1\" name=\"viewport\">\n  <title>The page you're looking for could not be found (404)</title>\n  <style>\n    body {\n      color: #666;\n      text-align: center;\n      font-family: \"Helvetica Neue\", Helvetica, Arial, sans-serif;\n      margin: auto;\n      font-size: 14px;\n    }\n\n    h1 {\n      font-size: 56px;\n      line-height: 100px;\n      font-weight: 400;\n

What are you seeing, and how does that differ from what you expect to see?

It should be run normally without error

Consider including screenshots, error messages, and/or other helpful visuals

What version are you on? Are you using self-managed or GitLab.com?
- GitLab (Hint: /help): 14.9.3-ee
- Runner (Hint: /admin/runners): 14.10.0
Add the CI configuration from .gitlab-ci.yml and other configuration if relevant (e.g. docker-compose.yml)

config.toml

concurrent = 1
check_interval = 0

[session_server]
  session_timeout = 1800

[[runners]]
  name = "runner-1"
  url = "https://git.intranet.domain-name"
  token = "redacted"
  tls-cert-file = "/etc/gitlab-runner/certs/sancertall.crt"
  tls-key-file = "/etc/gitlab-runner/certs/sancertall.key"
  executor = "docker"
  [runners.custom_build_dir]
  [runners.cache]
    [runners.cache.s3]
    [runners.cache.gcs]
    [runners.cache.azure]
  [runners.docker]
    tls_verify = false
    image = "ubuntu:latest"
    privileged = true
    disable_entrypoint_overwrite = false
    oom_kill_disable = false
    disable_cache = false
    volumes = ["/cache"]
    shm_size = 0

[[runners]]
  name = "runner-2"
  url = "https://git.intranet.domain-name"
  token = "redacted"
  tls-cert-file = "/etc/gitlab-runner/certs/sancertall.crt"
  tls-key-file = "/etc/gitlab-runner/certs/sancertall.key"
  executor = "docker"
  [runners.custom_build_dir]
  [runners.cache]
    [runners.cache.s3]
    [runners.cache.gcs]
    [runners.cache.azure]
  [runners.docker]
    tls_verify = false
    image = "ubuntu:latest"
    privileged = false
    disable_entrypoint_overwrite = false
    oom_kill_disable = false
    disable_cache = false
    volumes = ["/certs/client", "/cache", "/var/run/docker.sock:/var/run/docker.sock"]
    shm_size = 0

Files in /etc/gitlab-runner/ directory

root@server-runner:~# ls -la /etc/gitlab-runner/
total 16
drwx------.  3 root root   38 Apr 20 03:53 .
drwxr-xr-x. 91 root root 8192 Apr 20 03:49 ..
drwxr-xr-x.  2 root root  127 Apr 20 22:29 certs
-rw-------.  1 root root 1340 Apr 20 22:07 config.toml
root@server-runner:~# ls -la /etc/gitlab-runner/certs/
total 24
drwxr-xr-x. 2 root root  127 Apr 20 22:29 .
drwx------. 3 root root   38 Apr 20 03:53 ..
-rw-r--r--. 1 root root 3938 Apr 20 04:16 git.intranet.domain-name.crt
-rw-r--r--. 1 root root 3938 Apr 20 22:29 registry.intranet.domain-name.crt
-rw-r--r--. 1 root root 8560 Apr 20 04:28 sancertall.crt
-rw-r--r--. 1 root root 1704 Apr 20 04:28 sancertall.key

.gitlab-ci.yml

# You can override the included template(s) by including variable overrides
# SAST customization: https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings
# Secret Detection customization: https://docs.gitlab.com/ee/user/application_security/secret_detection/#customizing-settings
# Note that environment variables can be set in several places
# See https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence
image: php:8.1

stages:
  - build

variables:
  POSTGRES_USER: db_user
  POSTGRES_PASSWORD: db_pass
  POSTGRES_DB: db_name
  DOCKER_DRIVER: overlay2
  DOCKER_TLS_CERTDIR: ""
  HTTP_PROXY: "http://proxy-dc:8080/"
  HTTPS_PROXY: "http://proxy-dc:8080/"
  http_proxy: "http://proxy-dc:8080/"
  https_proxy: "http://proxy-dc:8080/"
  NO_PROXY: "docker"
  no_proxy: "docker"

test_build:
  image: docker:dind
  stage: build
  variables:
    IMAGE_TAG_BACKEND: $CI_REGISTRY_IMAGE/backend:$CI_COMMIT_REF_SLUG
    IMAGE_TAG_PHP: $CI_REGISTRY_IMAGE/php:$CI_COMMIT_REF_SLUG
    DOCKER_TLS_CERTDIR: ''
    HTTP_PROXY: "http://proxy-dc:8080/"
    HTTPS_PROXY: "http://proxy-dc:8080/"
    http_proxy: "http://proxy-dc:8080/"
    https_proxy: "http://proxy-dc:8080/"
    NO_PROXY: "docker"
    no_proxy: "docker"
  services:
    - name: 'docker:20.10.6-dind'
      command: [ '--tls=false', '--host=tcp://0.0.0.0:2375' ]
      alias: docker
  script:
    - export http_proxy="http://proxy-dc:8080/"
    - export https_proxy="http://proxy-dc:8080/"
    - export ftp_proxy="http://proxy-dc:8080/"
    - export HTTP_PROXY="http://proxy-dc:8080/"
    - export HTTPS_PROXY="http://proxy-dc:8080/"
    - export FTP_PROXY="http://proxy-dc:8080/"
    - export no_proxy="127.0.0.1,localhost,10.245.192.52,docker"
    - export NO_PROXY="127.0.0.1,localhost,10.245.192.52,docker"
    - echo $CI_REGISTRY_USER
    - echo $CI_REGISTRY
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY

include:
  - template: Jobs/Build.gitlab-ci.yml

GitLab.rb file

external_url 'https://git.intranet.domain-name'
nginx['ssl_certificate'] = "/data/gitlab/cert/sancertall.crt"
nginx['ssl_certificate_key'] = "/data/gitlab/cert/sancertall.key"
# registry_external_url 'https://registry.example.com'
# registry_external_url 'https://git.intranet.domain-name:5050'
registry_external_url 'https://registry.intranet.domain-name'

### Settings used by GitLab application
gitlab_rails['registry_enabled'] = true
# gitlab_rails['registry_host'] = "registry.gitlab.example.com"
# gitlab_rails['registry_port'] = "5050"
# gitlab_rails['registry_path'] = "/var/opt/gitlab/gitlab-rails/shared/registry"
gitlab_rails['registry_path'] = "/data/gitlab-storage/registry"
registry['env'] = {
  'http_proxy' => "http://proxy-dc:8080",
  'https_proxy' => "http://proxy-dc:8080",
  'no_proxy' => ".intranet.domain-name,127.0.0.1,localhost,10.245.192.52,proxy-dc",
  'HTTP_PROXY' => "http://proxy-dc:8080",
  'HTTPS_PROXY' => "http://proxy-dc:8080",
  'NO_PROXY' => ".intranet.domain-name,127.0.0.1,localhost,10.245.192.52,proxy-dc"
}

registry_nginx['ssl_certificate'] = "/data/gitlab/cert/sancertall.crt"
registry_nginx['ssl_certificate_key'] = "/data/gitlab/cert/sancertall.key"

What troubleshooting steps have you already taken? Can you link to any docs or other resources so we know where you have been?

I have already tried to import the SSL cert from Self-signed certificates or custom Certification Authorities | GitLab but still doesn’t work

Update:
From the previous configuration, I changed the gitlab.rb file in the following line (make registry on the same domain with the git):

registry_external_url 'https://git.intranet.domain-name:5050'
# registry_external_url 'https://registry.intranet.domain-name'

and it can push successfully.

My question, what should I change so that the registry can run with the external url at https://registry.intranet.domain-name (separate domain with the git)?

Thanks for taking the time to be thorough in your request, it really helps!

Nassor · July 28, 2022, 1:25pm

I’m having the same problem, but couldn’t be able to fix with above suggested solutions. Has anyone any idea on how to fix this? i have tried to use

registry['env'] = {
       "REGISTRY_HTTP_RELATIVEURLS" => true
}

in gitlab.rb conf file but doesn’t seem to have solved it.

asdasdasd · December 9, 2022, 4:03am

For anyone coming to this thread, I was able to fix the problem with configuration below.

Gitlab.rb file:


external_url 'https://git.domain-name'
gitlab_rails['time_zone'] = 'Asia/Jakarta'
gitlab_rails['smtp_enable'] = true
gitlab_rails['smtp_address'] = "smtp-server"
gitlab_rails['smtp_domain'] = "domain-name"
gitlab_rails['smtp_enable_starttls_auto'] = true
gitlab_rails['smtp_tls'] = false
gitlab_rails['smtp_ssl'] = false
gitlab_rails['smtp_pool'] = true
gitlab_rails['smtp_openssl_verify_mode'] = 'none'
gitlab_rails['gitlab_email_enabled'] = true
gitlab_rails['gitlab_email_from'] = 'no-reply@git.domain-name'
gitlab_rails['gitlab_email_display_name'] = 'Gitlab'
gitlab_rails['gitlab_email_reply_to'] = 'noreply@git.domain-name'
gitlab_rails['gitlab_default_theme'] = 2
gitlab_rails['gitlab_repository_downloads_path'] = '/data/gitlab-tmp/repositories'
gitlab_rails['monitoring_whitelist'] = ['127.0.0.0/8', '::1/128']
gitlab_rails['artifacts_enabled'] = true
gitlab_rails['artifacts_path'] = "/data/gitlab-storage/artifacts"
gitlab_rails['lfs_enabled'] = true
gitlab_rails['lfs_storage_path'] = "/data/gitlab-storage/lfs-objects"
gitlab_rails['uploads_directory'] = "/data/gitlab-storage/gitlab-uploads"
gitlab_rails['terraform_state_enabled'] = true
gitlab_rails['terraform_state_storage_path'] = "/data/gitlab-storage/terraform_state"
gitlab_rails['pages_local_store_enabled'] = true
gitlab_rails['pages_local_store_path'] = "/data/gitlab-storage/pages"
gitlab_rails['impersonation_enabled'] = true
gitlab_rails['usage_ping_enabled'] = true
gitlab_rails['ldap_enabled'] = true
gitlab_rails['prevent_ldap_sign_in'] = false
gitlab_rails['ldap_servers'] = YAML.load <<-'EOS'
   main: # 'main' is the GitLab 'provider ID' of this LDAP server
     label: 'LDAP'
     host: 'ldap-host-server'
     port: 389
     uid: 'sAMAccountName'
     bind_dn: 'bind_dn'
     password: 'bind_pass'
     encryption: 'plain' # "start_tls" or "simple_tls" or "plain"
     active_directory: true
     allow_username_or_email_login: true
     lowercase_usernames: true
     block_auto_created_users: false
     base: 'base_ldap'
     user_filter: ''
EOS
gitlab_rails['omniauth_enabled'] = true
gitlab_rails['omniauth_allow_single_sign_on'] = ['gitlab']
gitlab_rails['omniauth_auto_link_ldap_user'] = true
gitlab_rails['omniauth_block_auto_created_users'] = true
gitlab_rails['omniauth_providers'] = [
  {
    name: "gitlab",
    # label: "Provider name", # optional label for login button, defaults to "GitLab.com"
    app_id: "redacted",
    app_secret: "redacted",
    args: { scope: "read_api" } # optional: defaults to the scopes of the application
  },
  {
    name: "github",
    # label: "Provider name", # optional label for login button, defaults to "GitHub"
    app_id: "redacted",
    app_secret: "redacted",
    args: { scope: "user:email" }
  }
]

gitlab_rails['manage_backup_path'] = true
gitlab_rails['backup_path'] = "/data/gitlab-backup-storage/backups"
gitlab_rails['backup_keep_time'] = 604800
gitlab_rails['shared_path'] = '/data/gitlab-shared-storage'
gitlab_rails['encrypted_settings_path'] = '/data/gitlab-storage/encrypted_settings'
high_availability['mountpoint'] = ["/data"]
gitlab_rails['env'] = {
    "BUNDLE_GEMFILE" => "/opt/gitlab/embedded/service/gitlab-rails/Gemfile",
    "PATH" => "/opt/gitlab/bin:/opt/gitlab/embedded/bin:/bin:/usr/bin",
    "http_proxy" => "http://proxy-dc:8080",
    "https_proxy" => "http://proxy-dc:8080",
    "no_proxy" => ".domain-name,127.0.0.1,localhost,10.245.192.52,proxy-dc,10.245.0.1",
    "HTTP_PROXY" => "http://proxy-dc:8080",
    "HTTPS_PROXY" => "http://proxy-dc:8080",
    "NO_PROXY" => ".domain-name,127.0.0.1,localhost,10.245.192.52,proxy-dc,10.245.0.1"
}

registry_external_url 'https://registry.domain-name'
gitlab_rails['registry_path'] = "/data/gitlab-storage/registry"
registry['env'] = {
  'http_proxy' => "http://proxy-dc:8080",
  'https_proxy' => "http://proxy-dc:8080",
  'no_proxy' => ".domain-name,127.0.0.1,localhost,10.245.192.52,proxy-dc",
  'HTTP_PROXY' => "http://proxy-dc:8080",
  'HTTPS_PROXY' => "http://proxy-dc:8080",
  'NO_PROXY' => ".domain-name,127.0.0.1,localhost,10.245.192.52,proxy-dc"
}

gitlab_workhorse['env'] = {
  #'PATH' => "/opt/gitlab/bin:/opt/gitlab/embedded/bin:/bin:/usr/bin",
  #'SSL_CERT_DIR' => "/opt/gitlab/embedded/ssl/certs/"
  'http_proxy' => "http://proxy-dc:8080",
  'https_proxy' => "http://proxy-dc:8080",
  'no_proxy' => ".domain-name,127.0.0.1,localhost,10.245.192.52,proxy-dc",
  'HTTP_PROXY' => "http://proxy-dc:8080",
  'HTTPS_PROXY' => "http://proxy-dc:8080",
  'NO_PROXY' => ".domain-name,127.0.0.1,localhost,10.245.192.52,proxy-dc"
}

nginx['ssl_certificate'] = "/data/gitlab/cert/sancertall.crt"
nginx['ssl_certificate_key'] = "/data/gitlab/cert/sancertall.key"

nginx['worker_processes'] = 4
pages_external_url "http://gitpages.domain-name/"
gitlab_pages['enable'] = true
gitlab_pages['status_uri'] = "/@status"
gitlab_pages['max_connections'] = 0
gitlab_pages['max_uri_length'] = 1024
gitlab_pages['log_format'] = "json"
gitlab_pages['dir'] = "/data/gitlab-pages"
gitlab_pages['artifacts_server'] = true
gitlab_pages['env'] = {
  'http_proxy' => "http://proxy-dc:8080",
  'https_proxy' => "http://proxy-dc:8080",
  'no_proxy' => ".domain-name,127.0.0.1,localhost,10.245.192.52,proxy-dc",
  'HTTP_PROXY' => "http://proxy-dc:8080",
  'HTTPS_PROXY' => "http://proxy-dc:8080",
  'NO_PROXY' => ".domain-name,127.0.0.1,localhost,10.245.192.52,proxy-dc"
}
gitlab_rails['pages_path'] = "/data/gitlab-storage/pages"
gitlab_kas['enable'] = true
gitlab_kas['listen_address'] = '0.0.0.0:8150'
gitlab_kas['dir'] = '/var/opt/gitlab/gitlab-kas'
gitaly['env'] = {
  'http_proxy' => "http://proxy-dc:8080",
  'https_proxy' => "http://proxy-dc:8080",
  'no_proxy' => ".domain-name,127.0.0.1,localhost,10.245.192.52,proxy-dc",
  'HTTP_PROXY' => "http://proxy-dc:8080",
  'HTTPS_PROXY' => "http://proxy-dc:8080",
  'NO_PROXY' => ".domain-name,127.0.0.1,localhost,10.245.192.52,proxy-dc"
}

gitlab_rails['dependency_proxy_enabled'] = true
gitlab_rails['dependency_proxy_storage_path'] = "/data/gitlab-shared-storage/dependency_proxy"

config.toml


concurrent = 1
check_interval = 0

[session_server]
  listen_address = "[::]:8093"
  session_timeout = 1800

[[runners]]
  name = "runner-1"
  url = "https://git.domain-name"
  token = "redacted"
  tls-cert-file = "/etc/gitlab-runner/certs/sancertall.crt"
  tls-key-file = "/etc/gitlab-runner/certs/sancertall.key"
  executor = "docker"
  output_limit = 1024000
  [runners.custom_build_dir]
  [runners.cache]
    [runners.cache.s3]
    [runners.cache.gcs]
    [runners.cache.azure]
  [runners.docker]
    tls_verify = false
    image = "ubuntu:latest"
    privileged = true
    disable_entrypoint_overwrite = false
    oom_kill_disable = false
    disable_cache = false
    volumes = ["/certs/client", "/cache", "/var/run/docker.sock:/var/run/docker.sock"]
    shm_size = 0

[[runners]]
  name = "runner-2"
  url = "https://git.domain-name"
  token = "redacted"
  tls-cert-file = "/etc/gitlab-runner/certs/sancertall.crt"
  tls-key-file = "/etc/gitlab-runner/certs/sancertall.key"
  executor = "docker"
  output_limit = 1024000
  [runners.custom_build_dir]
  [runners.cache]
    [runners.cache.s3]
    [runners.cache.gcs]
    [runners.cache.azure]
  [runners.docker]
    tls_verify = false
    image = "ubuntu:latest"
    privileged = true
    disable_entrypoint_overwrite = false
    oom_kill_disable = false
    disable_cache = false
    volumes = ["/certs/client", "/cache", "/var/run/docker.sock:/var/run/docker.sock"]
    shm_size = 0

[[runners]]
  name = "runner-browser-performance-1"
  url = "https://git.domain-name"
  token = "redacted"
  executor = "docker"
  output_limit = 1024000
  [runners.custom_build_dir]
  [runners.cache]
    [runners.cache.s3]
    [runners.cache.gcs]
    [runners.cache.azure]
  [runners.docker]
    tls_verify = false
    image = "ubuntu:latest"
    privileged = true
    disable_entrypoint_overwrite = false
    oom_kill_disable = false
    disable_cache = false
    volumes = ["/cache"]
    shm_size = 0

with same .gitlab-ci files, this configuration can run well.