Autoscaling AWS gitlab runner fails to upload artifacts
I’m adding my own gitlab runners to my projects for building large projects (yocto). I’m creating these runners in AWS as per the instructions from gitlab.com:
Autoscaling GitLab Runner on AWS EC2 | GitLab
The pipeline I am using to test this builds fine, but fails to upload the build artifacts. These are the error messages from the build log.
Uploading artifacts...
u-boot/u-boot: found 1 matching files and directories
u-boot/u-boot.bin: found 1 matching files and directories
WARNING: Uploading artifacts as "archive" to coordinator... 520 id=3081205700 responseStatus=520 status=520 token=3Mnk7241
WARNING: Retrying... context=artifacts-uploader error=invalid argument
WARNING: Uploading artifacts as "archive" to coordinator... 520 id=3081205700 responseStatus=520 status=520 token=3Mnk7241
WARNING: Retrying... context=artifacts-uploader error=invalid argument
WARNING: Uploading artifacts as "archive" to coordinator... 520 id=3081205700 responseStatus=520 status=520 token=3Mnk7241
FATAL: invalid argument
The AWS environment is set up using the following CDK script I’ve written. Bear with me I’m not a cloud/devops expert…:
from aws_cdk import (
Stack,
aws_ec2 as ec2,
aws_iam as iam,
aws_s3 as s3,
aws_secretsmanager as secretsmanager
)
from constructs import Construct
class GitlabCiStack(Stack):
def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
# ubuntu 22.04 in eu-west-1
ami = "ami-0d75513e7706cf2d9"
# gitlab runner role
mgr_role = iam.Role(self, "gitlab-runner-manager-role",
assumed_by=iam.ServicePrincipal("ec2.amazonaws.com"))
mgr_role.add_managed_policy(iam.ManagedPolicy.from_aws_managed_policy_name("AdministratorAccess"))
# vpc
vpc = ec2.Vpc(
self, id='gitlab-ci-vpc',
max_azs=1,
subnet_configuration=[
{
"cidrMask": 24,
"name": "Public",
"subnetType": ec2.SubnetType.PUBLIC
},
])
subnet_id = vpc.select_subnets(subnet_type=ec2.SubnetType.PUBLIC).subnet_ids[0]
subnet_selection = ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC)
cache_bucket = s3.Bucket(self, 'gitlab-ci-cache')
mgr_user_data = ec2.UserData.for_linux()
# install gitlab runner
mgr_user_data.add_commands(
'curl -L "https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh" | bash',
'apt-get install -y gitlab-runner')
# install docker
mgr_user_data.add_commands(
'apt-get install -y ca-certificates curl gnupg lsb-release',
'mkdir -p /etc/apt/keyrings',
'curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg',
'echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null',
'apt-get update',
'apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin',
'service docker start')
# install docker machine
mgr_user_data.add_commands(
'curl -O "https://gitlab-docker-machine-downloads.s3.amazonaws.com/v0.16.2-gitlab.11/docker-machine-Linux-x86_64"',
'cp docker-machine-Linux-x86_64 /usr/local/bin/docker-machine',
'chmod +x /usr/local/bin/docker-machine')
# configure gitlab runner
mgr_user_data.add_commands(
'export CI_SERVER_URL=https://www.gitlab.com/ ',
'export REGISTRATION_TOKEN=XXXXXXXXXXXXXXXXXXXXXXXXXXXXX',
'export RUNNER_EXECUTOR=docker+machine',
'export DOCKER_IMAGE=ubuntu:22.04',
'export RUNNER_LIMIT=1',
'export DOCKER_DISABLE_CACHE=1',
'export MACHINE_IDLE_COUNT_MIN=0',
'export MACHINE_IDLE_TIME=300',
'export MACHINE_DRIVER=amazonec2',
'export CACHE_TYPE=s3',
'export CACHE_SHARED=1',
'export CACHE_S3_SERVER_ADDRESS=s3.amazonaws.com',
'export CACHE_S3_BUCKET_NAME={cache_bucket.bucket_name}',
'export CACHE_S3_BUCKET_LOCATION=eu-west-1',
'export MACHINE_NAME="gitlab-docker-machine-%s"',
"gitlab-runner register --non-interactive"
" --machine-machine-options amazonec2-region=eu-west-1"
f" --machine-machine-options amazonec2-vpc-id={vpc.vpc_id}"
f" --machine-machine-options amazonec2-ami={ami}"
f" --machine-machine-options amazonec2-subnet-id={subnet_id}"
" --machine-machine-options amazonec2-use-private-address=true"
" --machine-machine-options amazonec2-tags=runner-manager-name,gitlab-aws-autoscaler,gitlab,true,gitlab-runner-autoscale,true"
" --machine-machine-options amazonec2-instance-type=m4.2xlarge"
" --machine-machine-options amazonec2-iam-instance-profile=GitlabCiStack-gitlabrunnermanagerInstanceProfile99EE852E-nZRjAjV0A4Ci"
)
mgr_user_data.add_commands('cat /etc/gitlab-runner/config.toml')
manager = ec2.Instance(
self,
id="gitlab-runner-manager",
vpc=vpc,
key_name="gitlab-ci",
vpc_subnets=subnet_selection,
instance_type=ec2.InstanceType("t2.micro"),
machine_image=ec2.MachineImage.generic_linux({
"eu-west-1": ami}),
user_data=mgr_user_data,
user_data_causes_replacement=True,
role=mgr_role)
I also have a hard time debugging this because the docker image doing the file upload is cleaned up immediately after it finishes. Any tips on this are also very welcome.