Consume artifacts from child pipelines

GitLab CI/CD
,
1

I am using parent/child pipelines in a single project and attempting to pass artifacts between child pipelines. The documentation says this is possible, but doesn’t state how to do it.

My parent pipeline is called “Base” and it kicks off two child pipelines in successive stages:

  1. Node - builds a nodejs application and artifacts “node_modules” directory
  2. Docker - builds a docker image, copying in the “node_modules” artifact generated from the Node pipeline

I think all I have to do is somehow get the $CI_PIPELINE_ID from the Node pipeline up into the parent, then pass that pipeline into the Docker pipeline so it can access the artifacts:

Build Docker:
  stage: Build
  needs:
    - pipeline: $NODE_PIPELINE_ID
      job: Build Node

Any ideas? I’d prefer not to have the Node pipeline trigger the docker one, but I can do that if I need to, it will just be less straight forward for developers to view the pipeline results in the web UI.

2

Hello, answer to your question is in documentation: `.gitlab-ci.yml` keyword reference | GitLab

3

You could do something like this.

My GL_TOKEN is overwritten in the CICD with more access to get access to the project, I create a access token and I create the GL_TOKEN for the cicd.


# Expose artifacts from the child pipeline
npm-expose-artifacts:
  stage: test
  image: '$CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX/debian:latest'
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      when: always
    - when: never  # If no conditions above are met, don't run
  script:
    - echo "Running npm expose artifacts job"
    - apt-get update && apt-get install -y curl jq unzip fastjar gawk file
    - curl --version
    - jq --version
    - echo "Publishing artifacts"
    - echo "GitLab User Token [REDACTED]"
    - echo "$CI_API_V4_URL"
    - echo "$GITLAB_USER_TOKEN"

    # Fetch the child pipeline ID with added logging
    - echo "Fetching child pipeline ID..."
    - echo "$CI_API_V4_URL/projects/$CI_PROJECT_ID/pipelines/$CI_PIPELINE_ID/bridges"
    - >
      export BRIDGES_RESPONSE=$(curl --silent --header "PRIVATE-TOKEN: $GL_TOKEN"
      "$CI_API_V4_URL/projects/$CI_PROJECT_ID/pipelines/$CI_PIPELINE_ID/bridges")
    - echo "Bridges Response $BRIDGES_RESPONSE"
    - >
      export CI_CHILD_PIPELINE_ID=$(echo "$BRIDGES_RESPONSE" | jq -r ".[0].downstream_pipeline.id" 
      2>/dev/null || echo "null")
    - >
      if [ "$CI_CHILD_PIPELINE_ID" = "null" ] || [ -z "$CI_CHILD_PIPELINE_ID" ]; then
        echo "Error: Child pipeline ID not found or failed to parse response.";
        exit 1;
      fi
    - echo "Child Pipeline ID  $CI_CHILD_PIPELINE_ID"

    # Fetch the job ID from the child pipeline with added logging
    - echo "Fetching child job ID... $CI_API_V4_URL/projects/$CI_PROJECT_ID/pipelines/$CI_CHILD_PIPELINE_ID/jobs"
    - >
      export JOBS_RESPONSE=$(curl --silent --header "PRIVATE-TOKEN: $GL_TOKEN" 
      "$CI_API_V4_URL/projects/$CI_PROJECT_ID/pipelines/$CI_CHILD_PIPELINE_ID/jobs")
    - echo "Jobs Response  $JOBS_RESPONSE"
    - >
      export CI_CHILD_JOB_ID=$(echo "$JOBS_RESPONSE" | jq -r '.[0].id' 
      2>/dev/null || echo "null")
    - >
      if [ "$CI_CHILD_JOB_ID" = "null" ] || [ -z "$CI_CHILD_JOB_ID" ]; then
        echo "Error: Child job ID not found or failed to parse response.";
        exit 1;
      fi
    - echo "Child Job ID  $CI_CHILD_JOB_ID"

    - >

    - >
      export URLS_ARTIFACTS=$(echo "$JOBS_RESPONSE" | jq -r --arg ci_api_v4_url "$CI_API_V4_URL" '
          .[] |
          .project.id as $project_id |
          .name as $name |
          .id as $job_id |
          "\($ci_api_v4_url)/projects/27363/jobs/\($job_id)/artifacts/ \($name)"
      ')
    
    - echo "$URLS_ARTIFACTS"

    - DOWNLOAD_DIR=".artifacts-download"
    - mkdir -p "$DOWNLOAD_DIR"

    - echo "List of URLs to download:"
    - echo "$URLS_ARTIFACTS"
    - cd "$DOWNLOAD_DIR"
    - >
      echo "$URLS_ARTIFACTS" | while IFS= read -r line; do
        ls -la "."
        # Split the line into URL and filename
        # url=$(echo "$line" | awk '{print $1}')
        # filename=$(echo "$line" | awk '{print $2}')
        echo "start loop"
        echo "line is $line"
        line=$(echo "$line" | xargs)
        echo "line is $line"
        read -r url job_name <<< "$line"

        echo "url is $url"
        echo "job_name is $job_name"

          STATUS_CODE=$(curl -H "JOB-TOKEN: $CI_JOB_TOKEN" -o /dev/null -s -w "%{http_code}" "$url")
          if [ "$STATUS_CODE" -eq 404 ]; then
            echo "404 ignoring $job_name"
          else
            mkdir "$job_name"
            cd "$job_name"
            # Download the artifact using curl
            echo "Downloading $filename from $url..."
            # curl -v -L -H "PRIVATE-TOKEN: $GL_TOKEN" "$url&job_token=$CI_JOB_TOKEN"
            echo "$url"
            #curl -L -H "PRIVATE-TOKEN: $GL_TOKEN" "$url" -o "artifacts.zip"
            # curl -L -H "PRIVATE-TOKEN: $CI_JOB_TOKEN" "$url" -o "artifacts.zip"
            curl -L -H "JOB-TOKEN: $CI_JOB_TOKEN" "$url" -o "artifacts.zip"

            ls -la .
            echo "Extracting ZIP  artifacts.zip"
            file artifacts.zip
            jar tf artifacts.zip
            jar xvf artifacts.zip
            ls -la .

            cd ..
          fi

        echo "-----------------------"
      done
    - cd ..
    - ls -la "$DOWNLOAD_DIR"
    - find "$DOWNLOAD_DIR" | sed -e "s/[^-][^\/]*\//  |/g" -e "s/|\([^ ]\)/|-\1/"

    - echo "Listing files..."
    - ls -la
    - find ./.artifacts-download | sed -e "s/[^-][^\/]*\//  |/g" -e "s/|\([^ ]\)/|-\1/"

    - find ./.artifacts-download | sed -e "s/[^-][^\/]*\//  |/g" -e "s/|\([^ ]\)/|-\1/"
  needs:
    - npm-create-jobs
  artifacts:
    when: on_success
    paths:
      - .artifacts-download
    reports:
      junit:
        - .artifacts-download/update-this.test.xml
      coverage_report:
        coverage_format: cobertura
        path: .artifacts-download/changethis.xml
      codequality: .analysis/codequality-combine/gl-codequality.json

Hope it helps