I run into the same issue with the same version of Gitlab and a very similar .gitlab-ci.yml.
I try to trigger 15 child pipelines and roughly 9 or 8 get created properly and the remaining lead to Cannot obtain an exclusive lease for ci/pipeline_processing/atomic_processing_service::pipeline_id:154719. There must be another instance already in execution.
Exception entry is:
{
"severity": "ERROR",
"time": "2021-05-17T14:51:15.841Z",
"correlation_id": "01F5XCYEE73QSY3PHQ9JJAG58Q",
"exception.class": "ObjectStorage::ExclusiveLeaseTaken",
"exception.message": "Exclusive lease for object_storage_migrate:Ci::JobArtifact is already taken.",
"exception.backtrace": [
"app/uploaders/object_storage.rb:455:in `with_exclusive_lease'",
"app/uploaders/object_storage.rb:274:in `use_file'",
"lib/gitlab/ci/artifact_file_reader.rb:73:in `read_with_legacy_artifact_file_reader'",
"lib/gitlab/ci/artifact_file_reader.rb:51:in `read_zip_file!'",
"lib/gitlab/ci/artifact_file_reader.rb:30:in `read'",
"lib/gitlab/ci/config/external/file/artifact.rb:25:in `block in content'",
"lib/gitlab/utils/strong_memoize.rb:30:in `strong_memoize'",
"lib/gitlab/ci/config/external/file/artifact.rb:22:in `content'",
"lib/gitlab/ci/config/external/file/artifact.rb:45:in `validate_content!'",
"lib/gitlab/ci/config/external/file/base.rb:72:in `validate!'",
"lib/gitlab/ci/config/external/file/base.rb:20:in `initialize'",
"lib/gitlab/ci/config/external/file/artifact.rb:18:in `initialize'",
"lib/gitlab/ci/config/external/mapper.rb:105:in `new'",
"lib/gitlab/ci/config/external/mapper.rb:105:in `block in select_first_matching'",
"lib/gitlab/ci/config/external/mapper.rb:104:in `map'",
"lib/gitlab/ci/config/external/mapper.rb:104:in `select_first_matching'",
"lib/gitlab/ci/config/external/mapper.rb:40:in `map'",
"lib/gitlab/ci/config/external/mapper.rb:40:in `process'",
"lib/gitlab/ci/config/external/processor.rb:12:in `initialize'",
"lib/gitlab/ci/config.rb:96:in `new'",
"lib/gitlab/ci/config.rb:96:in `build_config'",
"ee/lib/ee/gitlab/ci/config_ee.rb:18:in `build_config'",
"lib/gitlab/ci/config.rb:83:in `expand_config'",
"lib/gitlab/ci/config.rb:28:in `initialize'",
"lib/gitlab/ci/yaml_processor.rb:23:in `new'",
"lib/gitlab/ci/yaml_processor.rb:23:in `execute'",
"lib/gitlab/ci/pipeline/chain/config/process.rb:22:in `perform!'",
"lib/gitlab/ci/pipeline/chain/sequence.rb:19:in `block in build!'",
"lib/gitlab/ci/pipeline/chain/sequence.rb:16:in `each'",
"lib/gitlab/ci/pipeline/chain/sequence.rb:16:in `build!'",
"app/services/ci/create_pipeline_service.rb:83:in `execute'",
"app/services/ci/create_downstream_pipeline_service.rb:35:in `execute'",
"app/workers/ci/create_cross_project_pipeline_worker.rb:14:in `block in perform'",
"app/workers/ci/create_cross_project_pipeline_worker.rb:11:in `perform'",
"lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb:16:in `perform'",
"lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb:40:in `perform'",
"lib/gitlab/sidekiq_middleware/duplicate_jobs/server.rb:8:in `call'",
"lib/gitlab/sidekiq_middleware/worker_context.rb:9:in `wrap_in_optional_context'",
"lib/gitlab/sidekiq_middleware/worker_context/server.rb:17:in `block in call'",
"lib/gitlab/application_context.rb:70:in `block in use'",
"lib/gitlab/application_context.rb:70:in `use'",
"lib/gitlab/application_context.rb:27:in `with_context'",
"lib/gitlab/sidekiq_middleware/worker_context/server.rb:15:in `call'",
"lib/gitlab/sidekiq_status/server_middleware.rb:7:in `call'",
"lib/gitlab/sidekiq_versioning/middleware.rb:9:in `call'",
"lib/gitlab/sidekiq_middleware/admin_mode/server.rb:14:in `call'",
"lib/gitlab/sidekiq_middleware/instrumentation_logger.rb:27:in `call'",
"lib/gitlab/sidekiq_middleware/batch_loader.rb:7:in `call'",
"lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb:7:in `call'",
"lib/gitlab/sidekiq_middleware/request_store_middleware.rb:10:in `block in call'",
"lib/gitlab/with_request_store.rb:17:in `enabling_request_store'",
"lib/gitlab/with_request_store.rb:10:in `with_request_store'",
"lib/gitlab/sidekiq_middleware/request_store_middleware.rb:9:in `call'",
"lib/gitlab/sidekiq_middleware/server_metrics.rb:25:in `block in call'",
"lib/gitlab/sidekiq_middleware/server_metrics.rb:48:in `block in instrument'",
"lib/gitlab/metrics/background_transaction.rb:30:in `run'",
"lib/gitlab/sidekiq_middleware/server_metrics.rb:48:in `instrument'",
"ee/lib/ee/gitlab/sidekiq_middleware/server_metrics.rb:17:in `instrument'",
"lib/gitlab/sidekiq_middleware/server_metrics.rb:24:in `call'",
"lib/gitlab/sidekiq_middleware/monitor.rb:8:in `block in call'",
"lib/gitlab/sidekiq_daemon/monitor.rb:49:in `within_job'",
"lib/gitlab/sidekiq_middleware/monitor.rb:7:in `call'",
"lib/gitlab/sidekiq_logging/structured_logger.rb:19:in `call'"
],
"user.username": "xxx",
"tags.program": "sidekiq",
"tags.locale": "en",
"tags.feature_category": "continuous_integration",
"tags.correlation_id": "01F5XCYEE73QSY3PHQ9JJAG58Q",
"extra.sidekiq": {
"class": "Ci::CreateCrossProjectPipelineWorker",
"args": [
"363516"
],
"retry": 3,
"queue": "pipeline_default:ci_create_cross_project_pipeline",
"version": 0,
"queue_namespace": "pipeline_default",
"jid": "7c79f6249f2a8e41ae2207d1",
"created_at": 1621263072.0381656,
"meta.user": "xxx",
"meta.project": "meta/gitlab-ci/containers/docker-tomee",
"meta.root_namespace": "meta",
"meta.caller_id": "PipelineProcessWorker",
"meta.remote_ip": "172.17.0.6",
"meta.feature_category": "continuous_integration",
"meta.client_id": "user/3",
"correlation_id": "01F5XCYEE73QSY3PHQ9JJAG58Q",
"enqueued_at": 1621263072.0553265
},
"extra.project_id": 102,
"extra.sha": "f2fe456a8ed206977451d6f171287b690107cde7"
}