I have installed GitLab with the following command:
helm upgrade --install gitlab gitlab \
--timeout 600s \
--set global.hosts.domain=***masked*** \
--set global.hosts.externalIP=***masked*** \
--set global.ingress.annotations."cert-manager\.io/issuer"=letsencrypt \
--set global.ingress.class=nginx \
--set certmanager.install=false \
--set certmanager.installCRDs=false \
--set certmanager.rbac.create=false \
--set nginx-ingress.enabled=false \
--set prometheus.install=false \
--set postgresql.image.tag=13.6.0 \
--set global.edition=ce \
--set global.ingress.configureCertmanager=false \
--set global.smtp.address=smtp.office365.com \
--set global.smtp.authentication=login \
--set global.smtp.user_name="***masked***" \
--set global.smtp.password.secret=ms365 \
--set global.smtp.password.key=password \
--set global.email.from="***masked***" \
--set global.email.reply_to="***masked***" \
--set global.smtp.domain="***masked***" \
--set global.smtp.enabled=true \
--set global.smtp.port=587 \
--set global.smtp.starttls_auto=true \
--set gitlab-runner.install=false
as you may see, I used my own ingress controller, cert manager and storage controller(deployed with nfs-subdir-external-provisioner).
It was successfully installed but when it comes to make a new repository, the following errors I get:
Failed to create repository
Failed to create repository
A similar issue reported here, but the detailed logs are bit different as follows:
{
"component": "gitaly",
"subcomponent": "gitaly",
"level": "info",
"correlation_id": "01GZPBC1JBS1X1ZT3PVVQ9EC5Q",
"grpc.code": "OK",
"grpc.meta.auth_version": "v2",
"grpc.meta.client_name": "gitlab-web",
"grpc.meta.deadline_type": "regular",
"grpc.meta.method_type": "unary",
"grpc.method": "RepositoryExists",
"grpc.request.deadline": "2023-05-05T16:19:34.009",
"grpc.request.fullMethod": "/gitaly.RepositoryService/RepositoryExists",
"grpc.request.glProjectPath": "",
"grpc.request.glRepository": "",
"grpc.request.payload_bytes": 95,
"grpc.request.repoPath": "@hashed/e6/29/e629fa6598d732768f7c726b4b621285f9c3b85303900aa912017db7617d8bdb.git",
"grpc.request.repoStorage": "default",
"grpc.response.payload_bytes": 0,
"grpc.service": "gitaly.RepositoryService",
"grpc.start_time": "2023-05-05T16:19:24.989",
"grpc.time_ms": 0.497,
"level": "info",
"limit.concurrency_queue_ms": 0,
"msg": "finished unary call with code OK",
"peer.address": "**masked**",
"pid": 1,
"remote_ip": "**masked**",
"span.kind": "server",
"system": "grpc",
"time": "2023-05-05T16:19:24.989Z",
"user_id": "1",
"username": "root"
}
{
"component": "gitaly",
"subcomponent": "gitaly",
"level": "info",
"correlation_id": "01GZPBC1JBS1X1ZT3PVVQ9EC5Q",
"diskcache": "7769bbad-bbea-46eb-bd7e-4c81be65b09c",
"grpc.meta.auth_version": "v2",
"grpc.meta.client_name": "gitlab-web",
"grpc.meta.deadline_type": "regular",
"grpc.meta.method_type": "unary",
"grpc.method": "CreateRepository",
"grpc.request.deadline": "2023-05-05T16:19:35.008",
"grpc.request.fullMethod": "/gitaly.RepositoryService/CreateRepository",
"grpc.request.glProjectPath": "gitlab-instance-8fcc9114/test",
"grpc.request.glRepository": "project-15",
"grpc.request.repoPath": "@hashed/e6/29/e629fa6598d732768f7c726b4b621285f9c3b85303900aa912017db7617d8bdb.git",
"grpc.request.repoStorage": "default",
"grpc.service": "gitaly.RepositoryService",
"grpc.start_time": "2023-05-05T16:19:25.018",
"level": "info",
"msg": "diskcache state change",
"peer.address": "**masked**",
"pid": 1,
"remote_ip": "**masked**",
"span.kind": "server",
"system": "grpc",
"time": "2023-05-05T16:19:25.488Z",
"user_id": "1",
"username": "root"
}
{
"component": "gitaly",
"subcomponent": "gitaly",
"level": "error",
"command.count": 1,
"command.cpu_time_ms": 3,
"command.inblock": 16,
"command.majflt": 0,
"command.maxrss": 310756,
"command.minflt": 144,
"command.oublock": 32,
"command.real_time_ms": 276,
"command.spawn_token_wait_ms": 0,
"command.system_time_ms": 0,
"command.user_time_ms": 3,
"correlation_id": "01GZPBC1JBS1X1ZT3PVVQ9EC5Q",
"error": "creating repository: locking repository: creating lock file: open /home/git/repositories/@hashed/e6/29/e629fa6598d732768f7c726b4b621285f9c3b85303900aa912017db7617d8bdb.git.lock: permission denied",
"grpc.code": "Internal",
"grpc.meta.auth_version": "v2",
"grpc.meta.client_name": "gitlab-web",
"grpc.meta.deadline_type": "regular",
"grpc.meta.method_type": "unary",
"grpc.method": "CreateRepository",
"grpc.request.deadline": "2023-05-05T16:19:35.008",
"grpc.request.fullMethod": "/gitaly.RepositoryService/CreateRepository",
"grpc.request.glProjectPath": "gitlab-instance-8fcc9114/test",
"grpc.request.glRepository": "project-15",
"grpc.request.payload_bytes": 139,
"grpc.request.repoPath": "@hashed/e6/29/e629fa6598d732768f7c726b4b621285f9c3b85303900aa912017db7617d8bdb.git",
"grpc.request.repoStorage": "default",
"grpc.response.payload_bytes": 0,
"grpc.service": "gitaly.RepositoryService",
"grpc.start_time": "2023-05-05T16:19:25.018",
"grpc.time_ms": 470.343,
"level": "error",
"limit.concurrency_queue_ms": 0,
"msg": "finished unary call with code Internal",
"peer.address": "**masked**",
"pid": 1,
"remote_ip": "**masked**",
"span.kind": "server",
"system": "grpc",
"time": "2023-05-05T16:19:25.489Z",
"user_id": "1",
"username": "root"
}
{
"component": "gitlab",
"subcomponent": "exceptions_json",
"level": "error",
"severity": "ERROR",
"time": "2023-05-05T16:19:25.494Z",
"correlation_id": "01GZPBC1JBS1X1ZT3PVVQ9EC5Q",
"exception.class": "Gitlab::Git::CommandError",
"exception.message": "13:creating repository: locking repository: creating lock file: open /home/git/repositories/@hashed/e6/29/e629fa6598d732768f7c726b4b621285f9c3b85303900aa912017db7617d8bdb.git.lock: permission denied.",
"exception.backtrace": [
"lib/gitlab/git/wraps_gitaly_errors.rb:15:in `rescue in wrapped_gitaly_errors'",
"lib/gitlab/git/wraps_gitaly_errors.rb:6:in `wrapped_gitaly_errors'",
"lib/gitlab/git/repository.rb:107:in `create_repository'",
"app/models/project.rb:1881:in `create_repository'",
"app/services/projects/create_service.rb:231:in `block in save_project_and_import_data'",
"app/models/concerns/cross_database_modification.rb:92:in `block in transaction'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:121:in `public_send'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:121:in `block in write_using_load_balancer'",
"lib/gitlab/database/load_balancing/load_balancer.rb:127:in `block in read_write'",
"lib/gitlab/database/load_balancing/load_balancer.rb:198:in `retry_with_backoff'",
"lib/gitlab/database/load_balancing/load_balancer.rb:116:in `read_write'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:120:in `write_using_load_balancer'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:72:in `transaction'",
"lib/gitlab/database.rb:369:in `block in transaction'",
"lib/gitlab/database.rb:368:in `transaction'",
"app/models/concerns/cross_database_modification.rb:83:in `transaction'",
"app/services/projects/create_service.rb:222:in `save_project_and_import_data'",
"app/services/projects/create_service.rb:74:in `execute'",
"app/controllers/projects_controller.rb:99:in `create'",
"app/controllers/application_controller.rb:524:in `set_current_admin'",
"lib/gitlab/session.rb:11:in `with_session'",
"app/controllers/application_controller.rb:515:in `set_session_storage'",
"lib/gitlab/i18n.rb:107:in `with_locale'",
"lib/gitlab/i18n.rb:113:in `with_user_locale'",
"app/controllers/application_controller.rb:506:in `set_locale'",
"app/controllers/application_controller.rb:499:in `set_current_context'",
"lib/gitlab/metrics/elasticsearch_rack_middleware.rb:16:in `call'",
"lib/gitlab/middleware/memory_report.rb:13:in `call'",
"lib/gitlab/middleware/speedscope.rb:13:in `call'",
"lib/gitlab/database/load_balancing/rack_middleware.rb:23:in `call'",
"lib/gitlab/middleware/rails_queue_duration.rb:33:in `call'",
"lib/gitlab/metrics/rack_middleware.rb:16:in `block in call'",
"lib/gitlab/metrics/web_transaction.rb:46:in `run'",
"lib/gitlab/metrics/rack_middleware.rb:16:in `call'",
"lib/gitlab/jira/middleware.rb:19:in `call'",
"lib/gitlab/middleware/go.rb:20:in `call'",
"lib/gitlab/etag_caching/middleware.rb:21:in `call'",
"lib/gitlab/middleware/query_analyzer.rb:11:in `block in call'",
"lib/gitlab/database/query_analyzer.rb:37:in `within'",
"lib/gitlab/middleware/query_analyzer.rb:11:in `call'",
"lib/gitlab/middleware/multipart.rb:173:in `call'",
"lib/gitlab/middleware/read_only/controller.rb:50:in `call'",
"lib/gitlab/middleware/read_only.rb:18:in `call'",
"lib/gitlab/middleware/same_site_cookies.rb:27:in `call'",
"lib/gitlab/middleware/basic_health_check.rb:25:in `call'",
"lib/gitlab/middleware/handle_malformed_strings.rb:21:in `call'",
"lib/gitlab/middleware/handle_ip_spoof_attack_error.rb:25:in `call'",
"lib/gitlab/middleware/request_context.rb:21:in `call'",
"lib/gitlab/middleware/webhook_recursion_detection.rb:15:in `call'",
"config/initializers/fix_local_cache_middleware.rb:11:in `call'",
"lib/gitlab/middleware/compressed_json.rb:37:in `call'",
"lib/gitlab/middleware/rack_multipart_tempfile_factory.rb:19:in `call'",
"lib/gitlab/middleware/sidekiq_web_static.rb:20:in `call'",
"lib/gitlab/metrics/requests_rack_middleware.rb:79:in `call'",
"lib/gitlab/middleware/release_env.rb:13:in `call'"
],
"exception.cause_class": "GRPC::Internal",
"user.username": "root",
"tags.program": "web",
"tags.locale": "en",
"tags.feature_category": "projects",
"tags.correlation_id": "01GZPBC1JBS1X1ZT3PVVQ9EC5Q",
"extra.project": {
"id": 15,
"full_path": "gitlab-instance-8fcc9114/test",
"disk_path": "@hashed/e6/29/e629fa6598d732768f7c726b4b621285f9c3b85303900aa912017db7617d8bdb"
}
}
{
"component": "gitlab",
"subcomponent": "application_json",
"level": "error",
"severity": "ERROR",
"time": "2023-05-05T16:19:25.497Z",
"correlation_id": "01GZPBC1JBS1X1ZT3PVVQ9EC5Q",
"message": "Unable to save project. Error: Failed to create repository"
}
When I check the nfs directory, I found that the permission of the lock file was 400(directory was 770):
# find @hashed -type f | xargs -i ls -ln {} | tail -4
-r--------+ 1 1000 1000 0 May 6 01:18 @hashed/85/27/8527a891e224136950ff32ca212b45bc93f69fbb801c3b1ebedac52775f99e61.git.lock
-r--------+ 1 1000 1000 0 May 6 01:19 @hashed/e6/29/e629fa6598d732768f7c726b4b621285f9c3b85303900aa912017db7617d8bdb.git.lock
-r--------+ 1 1000 1000 0 May 6 01:35 @hashed/b1/7e/b17ef6d19c7a5b1ee83b907c595526dcb1eb06db8227d650d5dda0a9f4ce8cd9.git.lock
-r--------+ 1 1000 1000 0 May 6 01:36 @hashed/45/23/4523540f1504cd17100c4835e85b7eefd49911580f8efff0599a8f283be6b9e3.git.lock
I don’t know why gitaly cannot obtain a lock while it actually creates the file. Does anyone have similar issues? Is it a problem with the nfs configuration?
Any information is welcome…