Possible bug: webhooks containing jobs with `null` stage name

:wave: Hi! From Datadog, we process GitLab webhooks from customers, and we’ve been noticing a transient issue with GitLab pipeline webhooks where sometimes jobs have no corresponding stage name (stage: null). This seems to only happen sometimes on canceled pipelines.

We’ve been noticing this issue for a few months, but we haven’t been able to debug/reproduce this as these webhooks come from our customers, which send them to us. We think there might be some sort of race condition here.

  1. Has anyone ever encountered this behaviour before? Should we file an issue? (there is no known reproduction).
  2. Is there a configuration that would lead to this behaviour? We’ve encountered this “consistently” for a set of customers, but the only common condition is that their pipelines were canceled.

Thanks a lot! :pray:

Example webhook GitLab version: 16.11.0-pre
Example webhook (redacted):

{
  "object_kind": "pipeline",
  "object_attributes": {
    "id": 1248227494,
    "iid": 142467,
    "name": null,
    "ref": "${SOME_REF}",
    "tag": false,
    "sha": "${SOME_SHA}",
    "before_sha": "${ANOTHER_SHA}",
    "source": "push",
    "status": "canceled",
    "detailed_status": "canceled",
    "stages": [
      ".pre",
      "utils",
      "check",
      "build",
      "deploy",
      "tests"
    ],
    "created_at": "2024-04-03 12:54:31 UTC",
    "finished_at": "2024-04-03 12:54:49 UTC",
    "duration": 9,
    "queued_duration": 8,
    "variables": [],
    "url": "${SOMEURL}"
  },
  "merge_request": null,
  "user": {
    "id": 6006040,
    "name": "${SOME_USERNAME}",
    "username": "${SOME_USERNAME}",
    "avatar_url": "${SOME_AVATAR_URL}",
    "email": "${SOME_EMAIL}"
  },
  "project": {
    "id": 17884450,
    "name": "${SOME_NAME}",
    "description": "${SOME_DESCRIPTION}",
    "web_url": "${SOME_WEB_URL}",
    "avatar_url": "${SOME_AVATAR_URL}",
    "git_ssh_url": "${SOME_GIT_URL}",
    "git_http_url": "{SOME_GIT_URL}",
    "namespace": "${SOME_NS}",
    "visibility_level": 0,
    "path_with_namespace": "${SOME_PATH}",
    "default_branch": "master",
    "ci_config_path": ""
  },
  "commit": {
    "id": "${SOME_ID}",
    "message": "${SOME_MSG}",
    "title": "${SOME_TITLE}",
    "timestamp": "${SOME_TS}",
    "url": "${SOME_URL}",
    "author": {
      "name": "${SOME_AUTHOR}",
      "email": "${SOME_EMAIL}"
    }
  },
  "builds": [
    {
      "id": 6546363204,
      "stage": null,
      "name": "${SOME_NAME}",
      "status": "canceled",
      "created_at": "2024-04-03 12:54:31 UTC",
      "started_at": null,
      "finished_at": "2024-04-03 12:54:47 UTC",
      "duration": null,
      "queued_duration": null,
      "failure_reason": null,
      "when": "manual",
      "manual": true,
      "allow_failure": true,
      "user": {
        "id": 6006040,
        "name": "${SOME_NAME}",
        "username": "${SOME_USERNAME}",
        "avatar_url": "${SOME_URL}",
        "email": "${SOME_EMAIL}"
      },
      "runner": null,
      "artifacts_file": {
        "filename": null,
        "size": null
      },
      "environment": null
    },
    {
      "id": 6546363207,
      "stage": null,
      "name": "${ANOTHER_NAME}",
      "status": "canceled",
      "created_at": "2024-04-03 12:54:32 UTC",
      "started_at": null,
      "finished_at": "2024-04-03 12:54:48 UTC",
      "duration": null,
      "queued_duration": null,
      "failure_reason": null,
      "when": "on_success",
      "manual": false,
      "allow_failure": false,
      "user": {
        "id": 6006040,
        "name": "${SOME_NAME}",
        "username": "${SOME_USERNAME}",
        "avatar_url": "${SOME_URL}",
        "email": "${SOME_EMAIL}"
      },
      "runner": null,
      "artifacts_file": {
        "filename": null,
        "size": null
      },
      "environment": {
        "name": "${SOME_ENV}",
        "action": "start",
        "deployment_tier": "other"
      }
    }
  ]
}