# L10 TH error-code registry (TH Error Code Specification v0.3 §7, §11.1).
# Source of truth for codegen once CI lands; l10_error_codes.py must stay in
# sync (l10_error_codes_test enforces it).
#
# Identity segments (e.g. TH-CPU-0001) are immutable: never renumber, never
# reuse. A change of meaning requires a new code.
#
# TH-CPU-0002 (CPU_STRESS_PERF_BELOW_THRESHOLD) is reserved but NOT allocated:
# CpuStress has no performance-threshold exit today. Allocate it only when
# such a check is added.
#
# Catalog: L10_error_codes.csv. Packed form is Domain|Category|Severity|
# QuickAction|sequence (spec §2.2). L10 may reuse a 1X identity only when
# the ErrorCodeDef is identical; a different name or disposition needs a
# new code. l10_error_codes_test enforces that across catalogs.
#
# ETCH-39238
- error_code: TH-BIOS-0001
  name: BIOS_EXPECTED_CONFIG_UNAVAILABLE
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 5  # ERROR_CONFIG
  quick_action: 9  # QA_ESCALATE
  component: L10.BIOS.BIOS
  test_cases: [CheckBiosVersion, CheckBiosBootOrder]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    Expected BIOS version or boot-order configuration could not be loaded; no
    BIOS verdict is available.
  probable_causes:
    - "Test-system, station, or configuration problem covering BIOS"
    - "Missing, malformed, or unreachable input required by BIOS_EXPECTED_CONFIG_UNAVAILABLE"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): CheckBiosVersion, CheckBiosBootOrder"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [BIOS]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-BIOS-0002
  name: BIOS_VERSION_QUERY_FAILED
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.BIOS.BIOS
  test_cases: [CheckBiosVersion]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The running BIOS version could not be queried.
  probable_causes:
    - "Test-system, station, or configuration problem covering BIOS"
    - "Missing, malformed, or unreachable input required by BIOS_VERSION_QUERY_FAILED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): CheckBiosVersion"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [BIOS]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-BIOS-0003
  name: BIOS_VERSION_OUTPUT_INVALID
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.BIOS.BIOS
  test_cases: [CheckBiosVersion]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    BIOS version output was empty, incomplete, or could not be parsed.
  probable_causes:
    - "Test-system, station, or configuration problem covering BIOS"
    - "Missing, malformed, or unreachable input required by BIOS_VERSION_OUTPUT_INVALID"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): CheckBiosVersion"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [BIOS]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-BIOS-0004
  name: BIOS_VERSION_MISMATCH
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 5  # ERROR_CONFIG
  quick_action: 9  # QA_ESCALATE
  component: L10.BIOS.BIOS
  test_cases: [CheckBiosVersion]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The running BIOS version does not match the expected L10 release.
  probable_causes:
    - Defective or out-of-spec BIOS on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckBiosVersion"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [BIOS]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-BIOS-0005
  name: BIOS_BOOT_ORDER_QUERY_FAILED
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.BIOS.BIOS
  test_cases: [CheckBiosBootOrder]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The configured BIOS boot order could not be queried.
  probable_causes:
    - "Test-system, station, or configuration problem covering BIOS"
    - "Missing, malformed, or unreachable input required by BIOS_BOOT_ORDER_QUERY_FAILED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): CheckBiosBootOrder"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [BIOS]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-BIOS-0006
  name: BIOS_BOOT_ORDER_MISMATCH
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 5  # ERROR_CONFIG
  quick_action: 9  # QA_ESCALATE
  component: L10.BIOS.BIOS
  test_cases: [CheckBiosBootOrder]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The BIOS boot order does not match the required L10 configuration.
  probable_causes:
    - Defective or out-of-spec BIOS on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckBiosBootOrder"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [BIOS]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-BIOS-0007
  name: BIOS_UPDATE_FAILED
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.BIOS.BIOS
  test_cases: [SohuServerBiosUpdate]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The server BIOS update command failed or did not complete successfully.
  probable_causes:
    - Defective or out-of-spec BIOS on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): SohuServerBiosUpdate"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [BIOS]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-BIOS-0008
  name: BIOS_POST_UPDATE_VERIFY_FAILED
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.BIOS.BIOS
  test_cases: [SohuServerBiosUpdate]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The BIOS did not report the requested version after update and reboot.
  probable_causes:
    - Defective or out-of-spec BIOS on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): SohuServerBiosUpdate"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [BIOS]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-BIOS-0009
  name: BIOS_BOOT_ORDER_UPDATE_FAILED
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.BIOS.BIOS
  test_cases: [CheckBiosBootOrder]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    Staging the required BIOS boot order via the BMC failed.
  probable_causes:
    - Defective or out-of-spec BIOS on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckBiosBootOrder"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [BIOS]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-BMC-0001
  name: BMC_EXPECTED_CONFIG_UNAVAILABLE
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 5  # ERROR_CONFIG
  quick_action: 9  # QA_ESCALATE
  component: L10.BMC.BMC
  test_cases: [BmcCheck, BmcCheckFanLocation, BmcCheckFanControl, BmcRecovery]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    Expected BMC/product configuration could not be loaded; no BMC comparison
    verdict is available.
  probable_causes:
    - "Test-system, station, or configuration problem covering BMC"
    - "Missing, malformed, or unreachable input required by BMC_EXPECTED_CONFIG_UNAVAILABLE"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): BmcCheck, BmcCheckFanLocation, BmcCheckFanControl"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [BMC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-BMC-0002
  name: BMC_INVENTORY_QUERY_FAILED
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.BMC.BMC
  test_cases: [BmcCheck]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The BMC inventory or management query command failed.
  probable_causes:
    - "Test-system, station, or configuration problem covering BMC"
    - "Missing, malformed, or unreachable input required by BMC_INVENTORY_QUERY_FAILED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): BmcCheck"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [BMC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-BMC-0003
  name: BMC_INVENTORY_OUTPUT_INVALID
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.BMC.BMC
  test_cases: [BmcCheck]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    BMC inventory output was incomplete or could not be parsed.
  probable_causes:
    - "Test-system, station, or configuration problem covering BMC"
    - "Missing, malformed, or unreachable input required by BMC_INVENTORY_OUTPUT_INVALID"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): BmcCheck"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [BMC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-BMC-0004
  name: BMC_IDENTITY_MISMATCH
  version: 1
  category: 11  # CATEGORY_DATA
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.BMC.BMC
  test_cases: [BmcCheck]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    BMC-reported platform vendor, product, or model identity does not match
    expected configuration.
  probable_causes:
    - Defective or out-of-spec BMC on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): BmcCheck"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [BMC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-BMC-0005
  name: BMC_MANAGEMENT_ADDRESS_MISMATCH
  version: 1
  category: 11  # CATEGORY_DATA
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.BMC.BMC
  test_cases: [BmcCheck]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The BMC management IP address or MAC address does not match the expected
    server configuration.
  probable_causes:
    - Defective or out-of-spec BMC on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): BmcCheck"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [BMC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-BMC-0006
  name: BMC_FIRMWARE_VERSION_MISMATCH
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 5  # ERROR_CONFIG
  quick_action: 9  # QA_ESCALATE
  component: L10.BMC.BMC
  test_cases: [BmcCheck]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The running BMC firmware version does not match the expected L10 release.
  probable_causes:
    - Defective or out-of-spec BMC on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): BmcCheck"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [BMC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-BMC-0007
  name: BMC_FAN_LOCATION_PROGRAM_FAILED
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.BMC.BMC
  test_cases: [BmcCheckFanLocation]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The BMC command to program fan locations failed.
  probable_causes:
    - Defective or out-of-spec BMC on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): BmcCheckFanLocation"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [BMC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-BMC-0008
  name: BMC_FAN_LOCATION_READBACK_FAILED
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.BMC.BMC
  test_cases: [BmcCheckFanLocation]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    Programmed fan-location data could not be read back from the BMC.
  probable_causes:
    - "Test-system, station, or configuration problem covering BMC"
    - "Missing, malformed, or unreachable input required by BMC_FAN_LOCATION_READBACK_FAILED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): BmcCheckFanLocation"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [BMC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-BMC-0009
  name: BMC_FAN_CONFIG_UNAVAILABLE
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 5  # ERROR_CONFIG
  quick_action: 9  # QA_ESCALATE
  component: L10.BMC.BMC
  test_cases: [BmcCheckFanControl]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    BMC fan-zone, PWM, or sensor configuration needed by the fan-control test is
    missing or invalid.
  probable_causes:
    - "Test-system, station, or configuration problem covering BMC"
    - "Missing, malformed, or unreachable input required by BMC_FAN_CONFIG_UNAVAILABLE"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): BmcCheckFanControl"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [BMC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-BMC-0010
  name: BMC_SENSOR_QUERY_FAILED
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.BMC.BMC
  test_cases: [BmcCheckFanControl]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The BMC sensor query failed or returned no usable fan/thermal readings.
  probable_causes:
    - "Test-system, station, or configuration problem covering BMC"
    - "Missing, malformed, or unreachable input required by BMC_SENSOR_QUERY_FAILED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): BmcCheckFanControl"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [BMC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-BMC-0011
  name: BMC_CONNECTIVITY_FAILED
  version: 1
  category: 1  # CATEGORY_NETWORK
  severity: 4  # ERROR_TRIAGE
  quick_action: 6  # QA_RETRY
  component: L10.BMC.BMC
  test_cases: [BmcConnectivityCheck, BmcRecovery]
  disposition: FAILED
  retryable: true
  max_auto_retry: 1
  description: >
    BMC connectivity validation observed packet loss or could not complete the
    configured reachability check.
  probable_causes:
    - Defective or out-of-spec BMC on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): BmcConnectivityCheck, BmcRecovery"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Retry the affected L10 subset once and capture logs from the failing run.", retest: "affected_subset", est_time_min: 10}
    - {step: 2, who: "escalate", action: "If persistent, isolate the unit; attach logs; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [BMC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-BMC-0012
  name: BMC_SSH_UNAVAILABLE
  version: 1
  category: 1  # CATEGORY_NETWORK
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.BMC.BMC
  test_cases: [SohuServerBmcFirmwareUpdate, SohuServerBiosUpdate, PsuFirmwareUpdate, CpldFirmwareUpdate, L10VBBUpdate, BmcPowerCycle]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    An SSH session to the BMC could not be established for the requested
    operation.
  probable_causes:
    - "Test-system, station, or configuration problem covering BMC"
    - "Missing, malformed, or unreachable input required by BMC_SSH_UNAVAILABLE"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): SohuServerBmcFirmwareUpdate, SohuServerBiosUpdate, PsuFirmwareUpdate, CpldFirmwareUpdate, L10VBBUpdate, BmcPowerCycle"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [BMC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-BMC-0013
  name: BMC_UPDATE_IMAGE_STAGING_FAILED
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.BMC.BMC
  test_cases: [SohuServerBmcFirmwareUpdate, SohuServerBiosUpdate, PsuFirmwareUpdate, CpldFirmwareUpdate, L10VBBUpdate]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The firmware image or update configuration could not be staged to the BMC.
  probable_causes:
    - "Test-system, station, or configuration problem covering BMC"
    - "Missing, malformed, or unreachable input required by BMC_UPDATE_IMAGE_STAGING_FAILED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): SohuServerBmcFirmwareUpdate, SohuServerBiosUpdate, PsuFirmwareUpdate, CpldFirmwareUpdate, L10VBBUpdate"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [BMC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-BMC-0014
  name: BMC_FIRMWARE_UPDATE_COMMAND_FAILED
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.BMC.BMC
  test_cases: [SohuServerBmcFirmwareUpdate]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The BMC firmware update command failed or did not complete successfully.
  probable_causes:
    - Defective or out-of-spec BMC on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): SohuServerBmcFirmwareUpdate"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [BMC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-BMC-0015
  name: BMC_POST_UPDATE_VERIFY_FAILED
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.BMC.BMC
  test_cases: [SohuServerBmcFirmwareUpdate]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The BMC did not report the requested firmware version after update.
  probable_causes:
    - Defective or out-of-spec BMC on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): SohuServerBmcFirmwareUpdate"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [BMC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-BMC-0016
  name: BMC_POWER_CYCLE_INTERVAL_SET_FAILED
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.BMC.BMC
  test_cases: [SetPowerCycleInterval]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The requested BMC power-cycle interval could not be configured.
  probable_causes:
    - "Test-system, station, or configuration problem covering BMC"
    - "Missing, malformed, or unreachable input required by BMC_POWER_CYCLE_INTERVAL_SET_FAILED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): SetPowerCycleInterval"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [BMC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-BMC-0017
  name: BMC_SYSTEM_EVENT_LOG_QUERY_FAILED
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.BMC.BMC
  test_cases: [StoreSystemLogStates, CheckSystemLogStates]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The BMC system-event log could not be read or parsed.
  probable_causes:
    - "Test-system, station, or configuration problem covering BMC"
    - "Missing, malformed, or unreachable input required by BMC_SYSTEM_EVENT_LOG_QUERY_FAILED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): StoreSystemLogStates, CheckSystemLogStates"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [BMC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-BMC-0018
  name: BMC_CRITICAL_EVENT_DETECTED
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.BMC.BMC
  test_cases: [CheckSystemLogStates]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    A new critical BMC/SEL event was detected between the stored baseline and
    the post-test log state.
  probable_causes:
    - Defective or out-of-spec BMC on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckSystemLogStates"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [BMC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-BMC-0019
  name: BMC_RAW_COMMAND_FAILED
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.BMC.BMC
  test_cases: [BmcCheck, BmcCheckFanLocation, BmcCheckFanControl]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    A required low-level BMC raw command failed or returned no response.
  probable_causes:
    - "Test-system, station, or configuration problem covering BMC"
    - "Missing, malformed, or unreachable input required by BMC_RAW_COMMAND_FAILED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): BmcCheck, BmcCheckFanLocation, BmcCheckFanControl"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [BMC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-BMC-0020
  name: BMC_UPDATE_MODEL_CONFIG_UNAVAILABLE
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 5  # ERROR_CONFIG
  quick_action: 9  # QA_ESCALATE
  component: L10.BMC.BMC
  test_cases: [SohuServerBmcFirmwareUpdate, SohuServerBiosUpdate, PsuFirmwareUpdate, CpldFirmwareUpdate, L10VBBUpdate]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The BMC update flow could not resolve the server model or target-specific
    update configuration.
  probable_causes:
    - "Test-system, station, or configuration problem covering BMC"
    - "Missing, malformed, or unreachable input required by BMC_UPDATE_MODEL_CONFIG_UNAVAILABLE"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): SohuServerBmcFirmwareUpdate, SohuServerBiosUpdate, PsuFirmwareUpdate, CpldFirmwareUpdate, L10VBBUpdate"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [BMC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-C2C-0001
  name: C2C_LINKUP_FAILED
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.C2C.SOHU_SERDES
  test_cases: [C2cLinkupMultiChipTestCase, C2cThroughputMultiChipTestCase]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    C2C lanes failed to train or link up within the timeout: RX equalization did
    not converge after its configured attempts, or the link-up test reported one
    or more datalinks/lanes down.
  probable_causes:
    - Marginal SERDES TX/RX lane on the module
    - FIR taps from the preceding sweep not valid for this rate/FEC mode
    - Loopback fixture or cable not seated on the station
    - PHY configuration mismatch for the requested FEC mode
  capture:
    - suite_run_url
    - link-up test output directory (per-lane PoR summary)
    - affected datalinks/lanes in affected_instances
    - "fec_mode, phy_loopback_mode and mac_loopback flag for the run"
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Retry the C2C subset up to three times in loopback mode with the affected lanes.", retest: "C2C_subset", est_time_min: 10}
    - {step: 2, who: "technician", action: "Reseat the loopback fixture/cable per station SOP; confirm the FIR sweep ran before link-up.", retest: "C2C_subset", est_time_min: 15}
    - {step: 3, who: "escalate", action: "Isolate the module; attach the link-up PoR summary and per-lane data; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [SOHU_SERDES, C2C_LOOPBACK_FIXTURE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-C2C-0007
  name: C2C_INTEGRITY_MISMATCH
  version: 1
  category: 11  # CATEGORY_DATA
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.C2C.SOHU_SERDES
  test_cases: [C2cIntegrityMultiChipTestCase]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    Bit-by-bit C2C loopback comparison found a data mismatch on at least one
    datalink/lane.
  probable_causes:
    - Marginal SERDES lane corrupting data under load
    - FEC not correcting at the configured rate for this lane
    - Loopback lane map does not match the fixture wiring
    - CSRAM/WDMA data path defect on the module
  capture:
    - suite_run_url
    - integrity test output directory (per-lane error plots)
    - per-lane mismatch data captured BEFORE any retry
    - affected datalinks/lanes in affected_instances
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture the per-lane error plots BEFORE retrying, then retry the C2C subset up to three times.", retest: "C2C_subset", est_time_min: 10}
    - {step: 2, who: "technician", action: "Verify the loopback lane map matches the fixture wiring; reseat the fixture per station SOP.", retest: "C2C_subset", est_time_min: 15}
    - {step: 3, who: "escalate", action: "Isolate the module; attach the per-lane mismatch data; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [SOHU_SERDES, C2C_LOOPBACK_FIXTURE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-C2C-0008
  name: C2C_THROUGHPUT_BELOW_THRESHOLD
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.C2C.SOHU_SERDES
  test_cases: [C2cThroughputMultiChipTestCase]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The large C2C transfer did not meet its acceptance criteria: measured
    throughput below the configured percentage of line rate (90% in the 1X
    suite), or the post-FEC uncorrected error fraction above its limit.
  probable_causes:
    - Marginal SERDES lane forcing FEC retransmission/correction
    - CSRAM PLL not held at the test frequency for the transfer
    - DMA read arbiter FIFO depth misconfigured for the rate
    - Loopback fixture insertion loss out of spec
  capture:
    - suite_run_url
    - large-transfer output directory (throughput and post-FEC error counters)
    - min_throughput_percent and max_post_fec_err_frac used for the run
    - affected datalinks/lanes in affected_instances
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Retry the C2C subset up to three times with the affected mode and lanes.", retest: "C2C_subset", est_time_min: 15}
    - {step: 2, who: "technician", action: "Reseat the loopback fixture; confirm the CSRAM PLL boost applied during the transfer.", retest: "C2C_subset", est_time_min: 15}
    - {step: 3, who: "escalate", action: "Isolate the module; attach throughput and post-FEC counters; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [SOHU_SERDES, C2C_LOOPBACK_FIXTURE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-C2C-0004
  name: C2C_LANE_MARGIN_BELOW_THRESHOLD
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.C2C.SOHU_SERDES
  test_cases: [C2cFirCdrSweepMultiChipTestCase]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    Worst per-lane minimum BER from the eye-diagram sweep exceeded the
    configured ber_threshold; the SERDES margin spec is not final.
  probable_causes:
    - Marginal SERDES lane (closed eye) on the module
    - Eye-diagram sweep threshold tighter than the (unfinalized) SERDES spec
    - PRBS pattern or rate not matched to the station fixture
    - Loopback fixture insertion loss out of spec
  capture:
    - suite_run_url
    - eye-diagram sweep output directory (per-lane BER matrices)
    - lane_margin_worst_min_ber measurement and per-lane breakdown
    - affected lanes in affected_instances
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Retry the lane-margin test up to three times; record the per-lane BER breakdown for each run.", retest: "C2C_subset", est_time_min: 15}
    - {step: 2, who: "escalate", action: "Route to SIVAL/platform triage with the per-lane BER matrices; do not isolate the module on this code alone until the SERDES margin spec is final.", retest: "n/a", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [SOHU_SERDES, C2C_LOOPBACK_FIXTURE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-C2C-0005
  name: C2C_PRBS_ERROR_THRESHOLD_EXCEEDED
  version: 1
  category: 11  # CATEGORY_DATA
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.C2C.SOHU_SERDES
  test_cases: [C2cPrbsMultiChipTestCase]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The C2C PRBS run completed but the observed bit-error count or rate exceeded
    the configured acceptance limit.
  probable_causes:
    - Defective or out-of-spec C2C on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): C2cPrbsMultiChipTestCase"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [C2C]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-C2C-0006
  name: C2C_SWEEP_OUTPUT_INVALID
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.C2C.SOHU_SERDES
  test_cases: [C2cFirCdrSweepMultiChipTestCase, C2cPrbsMultiChipTestCase]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The FIR/CDR or PRBS test did not produce complete, parseable lane results,
    so the C2C verdict is inconclusive.
  probable_causes:
    - "Test-system, station, or configuration problem covering C2C"
    - "Missing, malformed, or unreachable input required by C2C_SWEEP_OUTPUT_INVALID"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): C2cFirCdrSweepMultiChipTestCase, C2cPrbsMultiChipTestCase"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [C2C]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-CPU-0001
  name: CPU_STRESS_FAIL
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 2  # ERROR_ISOLATE
  quick_action: 6  # QA_RETRY
  component: L10.CPU.HOST_CPU
  test_cases: [CpuStress]
  disposition: FAILED
  retryable: true
  max_auto_retry: 1
  description: >
    stress-ng CPU stress failed because a stressor/verification error occurred
    or the tool returned its workload-failure status.
  probable_causes:
    - Marginal/defective CPU core producing compute errors under load
    - Inadequate cooling or heatsink mounting (thermal events during stress)
    - VRM / power delivery instability under full-core load
    - BIOS power/performance settings out of spec
  capture:
    - suite_run_url
    - full stress-ng stdout/stderr (incl. bogo-ops metrics)
    - dmesg / MCE excerpt for the run window
    - CPU frequency + thermal telemetry during the run
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Verify station ambient/airflow nominal, rerun CPU subset.", retest: "CPU_subset", est_time_min: 5}
    - {step: 2, who: "technician", action: "Inspect heatsink mounting and TIM per assembly SOP; remount if out of spec.", retest: "CPU_subset", est_time_min: 15}
    - {step: 3, who: "escalate", action: "Preserve unit; attach stress-ng logs + MCE excerpt; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [HOST_CPU, CPU_COOLER, VRM]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-CPU-0003
  name: CPU_STRESS_TOOL_UNAVAILABLE
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 5  # ERROR_CONFIG
  quick_action: 9  # QA_ESCALATE
  component: L10.CPU.HOST_CPU
  test_cases: [CpuStress]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    stress-ng is not installed or cannot be executed on the target; no CPU
    verdict is available (ERROR disposition; excluded from DUT yield).
  probable_causes:
    - DUT image missing the stress-ng package (provisioning gap)
    - InstallLinuxPackagesTestCase not run or failed earlier in the suite
    - Broken PATH or corrupted stress-ng binary on the target
  capture:
    - suite_run_url
    - stderr of the stress-ng version probe
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Install stress-ng on the target (or rerun the package-install step), rerun CPU subset.", retest: "CPU_subset", est_time_min: 5}
    - {step: 2, who: "escalate", action: "If provisioning image is missing the package, route to test-infra; open Jira. Do NOT route the unit to rework or FA.", retest: "n/a", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: []
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-CPU-0004
  name: CPU_STRESS_EXECUTION_ERROR
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 6  # QA_RETRY
  component: L10.CPU.HOST_CPU
  test_cases: [CpuStress]
  disposition: ERROR
  retryable: true
  max_auto_retry: 1
  description: >
    stress-ng returned an execution, initialization, option, resource,
    unsupported-stressor, or signal error rather than a CPU workload verdict.
  probable_causes:
    - stress-ng version on the DUT does not support a requested option/method
    - Insufficient memory/resources on the DUT to launch stressors
    - "stress-ng killed by an external signal (OOM killer, session teardown)"
  capture:
    - suite_run_url
    - full stress-ng stdout/stderr and exit code
    - dmesg excerpt for the run window (OOM / signal evidence)
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Confirm no other workload active on the DUT, rerun CPU subset.", retest: "CPU_subset", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Route logs to engineering triage with the stress-ng exit code and dmesg excerpt.", retest: "n/a", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: []
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-CPU-0005
  name: CPU_INVENTORY_QUERY_FAILED
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.CPU.HOST_CPU
  test_cases: [CheckCpu]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The CPU inventory command failed; no CPU BOM verdict is available.
  probable_causes:
    - "Test-system, station, or configuration problem covering CPU"
    - "Missing, malformed, or unreachable input required by CPU_INVENTORY_QUERY_FAILED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): CheckCpu"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [CPU]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-CPU-0006
  name: CPU_INVENTORY_OUTPUT_INVALID
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.CPU.HOST_CPU
  test_cases: [CheckCpu]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The CPU inventory output was malformed or could not be parsed into processor
    records.
  probable_causes:
    - "Test-system, station, or configuration problem covering CPU"
    - "Missing, malformed, or unreachable input required by CPU_INVENTORY_OUTPUT_INVALID"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): CheckCpu"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [CPU]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-CPU-0007
  name: CPU_COUNT_MISMATCH
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.CPU.HOST_CPU
  test_cases: [CheckCpu]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The discovered processor count does not match the expected L10 BOM.
  probable_causes:
    - Defective or out-of-spec CPU on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckCpu"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [CPU]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-CPU-0008
  name: CPU_IDENTITY_MISMATCH
  version: 1
  category: 11  # CATEGORY_DATA
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.CPU.HOST_CPU
  test_cases: [CheckCpu]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    A processor part number or manufacturer does not match the expected L10 BOM.
  probable_causes:
    - Defective or out-of-spec CPU on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckCpu"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [CPU]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-CPU-0009
  name: CPU_TOPOLOGY_MISMATCH
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.CPU.HOST_CPU
  test_cases: [CheckCpu]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    A processor core count or thread count does not match the expected L10 BOM.
  probable_causes:
    - Defective or out-of-spec CPU on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckCpu"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [CPU]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-CPU-0010
  name: CPU_CLOCK_MISMATCH
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.CPU.HOST_CPU
  test_cases: [CheckCpu]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    A processor base or maximum clock reported by inventory does not match the
    expected L10 BOM.
  probable_causes:
    - Defective or out-of-spec CPU on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckCpu"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [CPU]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-DISK-0001
  name: DISK_INVENTORY_QUERY_FAILED
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.DISK.STORAGE
  test_cases: [CheckDisk, DiskFormat, DiskSmart, DiskStress]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The block-device or NVMe inventory command failed; no storage verdict is
    available.
  probable_causes:
    - "Test-system, station, or configuration problem covering Disk"
    - "Missing, malformed, or unreachable input required by DISK_INVENTORY_QUERY_FAILED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): CheckDisk, DiskFormat, DiskSmart, DiskStress"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [DISK]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-DISK-0002
  name: DISK_INVENTORY_OUTPUT_INVALID
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.DISK.STORAGE
  test_cases: [CheckDisk, DiskFormat, DiskStress]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    Block-device or NVMe inventory output was malformed or could not be parsed.
  probable_causes:
    - "Test-system, station, or configuration problem covering Disk"
    - "Missing, malformed, or unreachable input required by DISK_INVENTORY_OUTPUT_INVALID"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): CheckDisk, DiskFormat, DiskStress"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [DISK]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-DISK-0003
  name: NO_DISKS_DETECTED
  version: 1
  category: 2  # CATEGORY_STORAGE
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.DISK.STORAGE
  test_cases: [CheckDisk, DiskSmart]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    No storage devices were detected although the L10 BOM requires disks.
  probable_causes:
    - Defective or out-of-spec Disk on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckDisk, DiskSmart"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [DISK]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-DISK-0004
  name: DISK_COUNT_MISMATCH
  version: 1
  category: 2  # CATEGORY_STORAGE
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.DISK.STORAGE
  test_cases: [CheckDisk]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The discovered disk count does not match the expected L10 BOM.
  probable_causes:
    - Defective or out-of-spec Disk on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckDisk"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [DISK]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-DISK-0005
  name: DISK_IDENTITY_OR_CAPACITY_MISMATCH
  version: 1
  category: 11  # CATEGORY_DATA
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.DISK.STORAGE
  test_cases: [CheckDisk]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    A disk model, serial number, namespace, or capacity does not match the
    expected L10 BOM.
  probable_causes:
    - Defective or out-of-spec Disk on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckDisk"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [DISK]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-DISK-0006
  name: MIXED_DRIVE_CONFIGURATION
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.DISK.STORAGE
  test_cases: [CheckDisk]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    Installed drives do not satisfy the required homogeneous model/capacity
    configuration.
  probable_causes:
    - Defective or out-of-spec Disk on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckDisk"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [DISK]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-DISK-0007
  name: BOOT_DEVICE_UNDETERMINED
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 5  # ERROR_CONFIG
  quick_action: 9  # QA_ESCALATE
  component: L10.DISK.STORAGE
  test_cases: [DiskFormat, DiskStress]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The test could not reliably identify the host boot device and therefore
    could not safely select data drives.
  probable_causes:
    - "Test-system, station, or configuration problem covering Disk"
    - "Missing, malformed, or unreachable input required by BOOT_DEVICE_UNDETERMINED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): DiskFormat, DiskStress"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [DISK]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-DISK-0008
  name: NO_TESTABLE_NVME_DEVICES
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 5  # ERROR_CONFIG
  quick_action: 9  # QA_ESCALATE
  component: L10.DISK.STORAGE
  test_cases: [DiskFormat, DiskSmart, DiskStress]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    No eligible non-boot NVMe device remained after discovery and configured
    exclusions.
  probable_causes:
    - "Test-system, station, or configuration problem covering Disk"
    - "Missing, malformed, or unreachable input required by NO_TESTABLE_NVME_DEVICES"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): DiskFormat, DiskSmart, DiskStress"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [DISK]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-DISK-0009
  name: NVME_NAMESPACE_MISSING
  version: 1
  category: 2  # CATEGORY_STORAGE
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.DISK.NVME
  test_cases: [DiskFormat]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    An expected NVMe controller has no usable namespace and automatic formatting
    is disabled.
  probable_causes:
    - Defective or out-of-spec NVMe SSD on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): DiskFormat"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NVME_SSD]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-DISK-0010
  name: NVME_FORMAT_FAILED
  version: 1
  category: 2  # CATEGORY_STORAGE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.DISK.NVME
  test_cases: [DiskFormat]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The NVMe format or namespace creation command failed.
  probable_causes:
    - Defective or out-of-spec NVMe SSD on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): DiskFormat"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NVME_SSD]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-DISK-0011
  name: NVME_NAMESPACE_NOT_CREATED
  version: 1
  category: 2  # CATEGORY_STORAGE
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.DISK.NVME
  test_cases: [DiskFormat]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The expected NVMe namespace was still absent after the format/namespace
    operation completed.
  probable_causes:
    - Defective or out-of-spec NVMe SSD on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): DiskFormat"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NVME_SSD]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-DISK-0012
  name: NVME_SMART_SELF_TEST_FAILED
  version: 1
  category: 2  # CATEGORY_STORAGE
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.DISK.NVME
  test_cases: [DiskSmart]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    An NVMe device self-test completed with a failure or did not complete
    successfully.
  probable_causes:
    - Defective or out-of-spec NVMe SSD on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): DiskSmart"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NVME_SSD]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-DISK-0013
  name: NVME_SMART_DATA_UNAVAILABLE
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.DISK.NVME
  test_cases: [DiskSmart]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    SMART/health data could not be read for an eligible NVMe device.
  probable_causes:
    - "Test-system, station, or configuration problem covering NVMe SSD"
    - "Missing, malformed, or unreachable input required by NVME_SMART_DATA_UNAVAILABLE"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): DiskSmart"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NVME_SSD]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-DISK-0014
  name: NVME_SMART_CRITICAL_WARNING
  version: 1
  category: 2  # CATEGORY_STORAGE
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.DISK.NVME
  test_cases: [DiskSmart]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The NVMe SMART critical-warning field is non-zero.
  probable_causes:
    - Defective or out-of-spec NVMe SSD on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): DiskSmart"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NVME_SSD]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-DISK-0015
  name: NVME_AVAILABLE_SPARE_BELOW_THRESHOLD
  version: 1
  category: 2  # CATEGORY_STORAGE
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.DISK.NVME
  test_cases: [DiskSmart]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    NVMe available spare is below the device or configured threshold.
  probable_causes:
    - Defective or out-of-spec NVMe SSD on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): DiskSmart"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NVME_SSD]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-DISK-0016
  name: NVME_MEDIA_ERRORS_DETECTED
  version: 1
  category: 2  # CATEGORY_STORAGE
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.DISK.NVME
  test_cases: [DiskSmart]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    NVMe SMART reports one or more media/data-integrity errors.
  probable_causes:
    - Defective or out-of-spec NVMe SSD on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): DiskSmart"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NVME_SSD]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-DISK-0017
  name: NVME_TEMPERATURE_OUT_OF_RANGE
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.DISK.NVME
  test_cases: [DiskSmart]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    NVMe temperature is outside the configured safe operating range.
  probable_causes:
    - Defective or out-of-spec NVMe SSD on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): DiskSmart"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NVME_SSD]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-DISK-0018
  name: NVME_WEAR_LIMIT_EXCEEDED
  version: 1
  category: 2  # CATEGORY_STORAGE
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.DISK.NVME
  test_cases: [DiskSmart]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    NVMe percentage-used or wear indicator exceeds the configured acceptance
    limit.
  probable_causes:
    - Defective or out-of-spec NVMe SSD on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): DiskSmart"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NVME_SSD]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-DISK-0019
  name: DISK_STRESS_FIO_FAILED
  version: 1
  category: 2  # CATEGORY_STORAGE
  severity: 2  # ERROR_ISOLATE
  quick_action: 6  # QA_RETRY
  component: L10.DISK.STORAGE
  test_cases: [DiskStress]
  disposition: FAILED
  retryable: true
  max_auto_retry: 1
  description: >
    The fio disk stress workload returned a workload-failure status for an
    eligible drive.
  probable_causes:
    - Defective or out-of-spec Disk on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): DiskStress"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Retry the affected L10 subset once and capture logs from the failing run.", retest: "affected_subset", est_time_min: 10}
    - {step: 2, who: "escalate", action: "If persistent, isolate the unit; attach logs; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [DISK]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-DISK-0020
  name: DISK_STRESS_OUTPUT_INVALID
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.DISK.STORAGE
  test_cases: [DiskStress]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    fio output was missing or could not be parsed into valid per-drive results.
  probable_causes:
    - "Test-system, station, or configuration problem covering Disk"
    - "Missing, malformed, or unreachable input required by DISK_STRESS_OUTPUT_INVALID"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): DiskStress"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [DISK]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-DMA-0002
  name: DMA_DATA_PATH_TEST_FAILED
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 2  # ERROR_ISOLATE
  quick_action: 6  # QA_RETRY
  component: L10.DMA.HOST_CHIP_PATH
  test_cases: [SohuDmaMultiChipTestCase]
  disposition: FAILED
  retryable: true
  max_auto_retry: 1
  description: >
    Host-chip DMA data-path test failure
  probable_causes:
    - Marginal PCIe or on-chip data path corrupting DMA transfers
    - Stuck or transition-faulty bits in the target SRAM region
    - DMA engine / descriptor handling fault
  capture:
    - suite_run_url
    - "full dma_test_app stdout/stderr, exit code, and RNG seed"
    - "unit_sn, fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Rerun the DMA subset up to 3 times; reseat the module if failures persist.", retest: "DMA_subset", est_time_min: 10}
    - {step: 2, who: "escalate", action: "Preserve unit; attach dma_test_app output and seed; isolate the module and route to triage.", retest: "n/a", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [SOHU_ASIC, MEZZ_MODULE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-ENV-0009
  name: HOST_OR_BMC_UNREACHABLE
  version: 1
  category: 1  # CATEGORY_NETWORK
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.ENV.STATION
  test_cases: [SohuHostRebootTestCase, BmcPowerCycle, BmcConnectivityCheck]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The station host or its BMC did not answer ping or SSH after boot, so no
    module test could start.
  probable_causes:
    - Station host powered off or stuck before the OS came up
    - BMC network or management cabling fault
    - sshd not running or host key changed on the station host
    - Station host IP/DHCP reservation changed
  capture:
    - suite_run_url
    - ping and ssh transcript for the station host and its BMC
    - "station_id, harness_version, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Check station power, BMC network and management cabling; confirm the host answers ping and ssh.", retest: "station_check", est_time_min: 10}
    - {step: 2, who: "escalate", action: "Route to test-infra with the ping/ssh transcript. Do NOT route any module to rework or FA on this code.", retest: "n/a", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: []
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-ENV-0010
  name: PLX_SWITCH_DISCOVERY_FAILED
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.ENV.STATION
  test_cases: [CheckPlxDaemonFirmwareVersion, UpdatePlxDaemonFirmwareVersion]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    PLX daemon reported no PEX switches
  probable_causes:
    - VBB MCU firmware is not updated to the required version
    - PEX switches are unavailable to the PLX daemon
  capture:
    - suite_run_url
    - per-test log and transcript
    - VBB MCU firmware version
    - "harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Check and update the VBB MCU firmware, then rerun the affected test.", retest: "affected_subset"}
    - {step: 2, who: "escalate", action: "Escalate to Etched if no PEX switches are reported after the firmware check.", retest: "n/a"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: []
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-ENV-0011
  name: HOST_DRIVER_SETUP_FAILED
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.ENV.STATION
  test_cases: [SoftwareProvisioning, BroadcomUpdateFwAndConfigure]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    A host driver/setup utility step failed: kernel module reload, PCIe teardown
    or bus rescan, sysfs parameter write, or log-reader activation returned
    non-zero.
  probable_causes:
    - "sohu_mdev kernel module not loaded, or loaded at a version without the expected parameter"
    - Missing sudo/permissions for the setup command on the station host
    - Earlier setup step in the suite failed and left the host in an unexpected state
    - "Host tooling (driver package, log reader) not provisioned on this station"
  capture:
    - suite_run_url
    - "exact command, exit code, stdout and stderr of the failing setup step"
    - lsmod / modinfo output for sohu_mdev
    - dmesg excerpt for the run window
    - "harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Fix station host environment (reload the driver, restore permissions) and rerun the setup subset.", retest: "setup_subset", est_time_min: 10}
    - {step: 2, who: "escalate", action: "Route to test-infra with the command output and lsmod/dmesg excerpt. Do NOT route the unit to rework or FA.", retest: "n/a", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: []
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-ENV-0005
  name: MODEL_REGISTRY_MOUNT_FAILED
  version: 1
  category: 1  # CATEGORY_NETWORK
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.ENV.MODEL_REGISTRY
  test_cases: [ModelRegistryNfsMountTestCase]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The model-registry NFS share could not be mounted or validated on the L10
    host.
  probable_causes:
    - "Test-system, station, or configuration problem covering Model Registry"
    - "Missing, malformed, or unreachable input required by MODEL_REGISTRY_MOUNT_FAILED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): ModelRegistryNfsMountTestCase"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [MODEL_REGISTRY]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-ENV-0006
  name: MODEL_ARTIFACT_UNAVAILABLE
  version: 1
  category: 11  # CATEGORY_DATA
  severity: 5  # ERROR_CONFIG
  quick_action: 9  # QA_ESCALATE
  component: L10.ENV.MODEL_REGISTRY
  test_cases: [StageModelRegistryToShmTestCase, SohuMlpTp8TestCase, SohuLlama70bForwardIteratedTestCase, SohuLlama70bHp8ForwardIteratedTestCase, SohuLlamaHp2Up4CachedInferenceMaxTestCase, SohuLlamaHp2Up4NospecInferenceMaxTestCase, SohuLlamaHp8CachedInferenceMaxTestCase, Llama8bFp8Tp1SmokeTestCase]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    A required model artifact, Hugging Face cache, or model path is missing or unreadable.
  probable_causes:
    - "Test-system, station, or configuration problem covering Model Registry"
    - "Missing, malformed, or unreachable input required by MODEL_ARTIFACT_UNAVAILABLE"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): StageModelRegistryToShmTestCase, SohuMlpTp8TestCase, SohuLlama70bForwardIteratedTestCase, SohuLlama70bHp8ForwardIteratedTestCase, SohuLlamaHp2Up4CachedInferenceMaxTestCase, SohuLlamaHp2Up4NospecInferenceMaxTestCase, SohuLlamaHp8CachedInferenceMaxTestCase, Llama8bFp8Tp1SmokeTestCase"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [MODEL_REGISTRY]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-ENV-0007
  name: MODEL_STAGE_TO_SHM_FAILED
  version: 1
  category: 9  # CATEGORY_RESOURCE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.ENV.SHM
  test_cases: [StageModelRegistryToShmTestCase]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The required model artifact could not be copied or staged into shared
    memory.
  probable_causes:
    - "Test-system, station, or configuration problem covering Shared Memory"
    - "Missing, malformed, or unreachable input required by MODEL_STAGE_TO_SHM_FAILED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): StageModelRegistryToShmTestCase"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [SHARED_MEMORY]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-ENV-0008
  name: MODEL_SHM_CLEANUP_FAILED
  version: 1
  category: 9  # CATEGORY_RESOURCE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.ENV.SHM
  test_cases: [FreeModelRegistryFromShmTestCase]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The staged model could not be removed from shared memory during cleanup.
  probable_causes:
    - "Test-system, station, or configuration problem covering Shared Memory"
    - "Missing, malformed, or unreachable input required by MODEL_SHM_CLEANUP_FAILED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): FreeModelRegistryFromShmTestCase"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [SHARED_MEMORY]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-ENV-0012
  name: WEIGHTS_STAGE_TO_DISK_FAILED
  version: 1
  category: 9  # CATEGORY_RESOURCE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.ENV.WEIGHTS_CACHE
  test_cases: [StageWeightsTestCase]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The requested model weights could not be mirrored onto the server's local
    disk, or the destination lacked the free space to hold them.
  probable_causes:
    - "Test-system, station, or configuration problem covering the local weights cache"
    - "Missing, malformed, or unreachable input required by WEIGHTS_STAGE_TO_DISK_FAILED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): StageWeightsTestCase"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.253
  suspected_components: [DISK]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-ENV-0013
  name: WEIGHTS_DISK_CLEANUP_FAILED
  version: 1
  category: 9  # CATEGORY_RESOURCE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.ENV.WEIGHTS_CACHE
  test_cases: [FreeWeightsTestCase]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The staged model weights could not be removed from the server's local disk
    during cleanup.
  probable_causes:
    - "Test-system, station, or configuration problem covering the local weights cache"
    - "Missing, malformed, or unreachable input required by WEIGHTS_DISK_CLEANUP_FAILED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): FreeWeightsTestCase"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.253
  suspected_components: [DISK]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-FW-0001
  name: SOHU_FIRMWARE_UPDATE_FAILED
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.FW.FIRMWARE
  test_cases: [MultiChipSohuHotReloadFirmwareUpdateTestCase]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    Sohu app firmware update failed or ASIC not pingable after update
  probable_causes:
    - Image package rejected by the bootloader (bad signature or layout)
    - ASIC hung during the jump to application firmware
    - Mezzanine/VBB comms lost mid-update
    - Power sequencing fault during the update
  capture:
    - suite_run_url
    - mezz and VBB UART logs across the update window
    - version info before the update
    - image package path and release id
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture the UART log for the update window; verify the staged image package matches the release.", retest: "n/a", est_time_min: 10}
    - {step: 2, who: "escalate", action: "Escalate to firmware with the UART log and the image package release id; do not retry blind.", retest: "n/a", stop_condition: "No further update attempts before the UART log is reviewed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [SOHU_APP_FIRMWARE, SOHU_BOOTLOADER, MEZZANINE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-FW-0032
  name: SOHU_FIRMWARE_HASH_MISMATCH
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.FW.FIRMWARE
  test_cases: [MultiChipSohuHotReloadFirmwareUpdateTestCase]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The ASIC booted application firmware after the update, but the running image
    hash does not match the hash in the image package that was just staged.
  probable_causes:
    - Station staged an image package from a different release
    - Update silently fell back to a previously flashed image
    - Image package metadata (version info) does not match its payload
    - QSPI/bootloader served a stale image
  capture:
    - suite_run_url
    - expected hash (from the image package) and the running hash
    - image package path and release id
    - version info before and after the update
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Compare the staged image package against the intended release package; confirm the station is serving the right build.", retest: "n/a", est_time_min: 10}
    - {step: 2, who: "escalate", action: "Route to firmware/test-infra triage with both hashes and the package release id.", retest: "n/a", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [SOHU_APP_FIRMWARE, STATION_IMAGE_STAGING]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-FW-0005
  name: FIXTURE_FIRMWARE_UPDATE_FAILED
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 5  # ERROR_CONFIG
  quick_action: 2  # QA_WARM_REBOOT
  component: L10.FW.FIRMWARE
  test_cases: [L10VBBUpdate]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    Fixture firmware update failed (golden VBB / retimer): OpenOCD flash,
    ST-Link adapter, post-flash UART/RPC ready check, or application
    firmware-hash verification did not succeed.
  probable_causes:
    - Fixture firmware update or verification failed
  capture:
    - suite_run_url
    - per-test log and transcript
    - firmware update transcript
    - "harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Halt testing and escalate to Etched.", retest: "affected_subset"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: []
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-FW-0033
  name: DVFS_PROFILE_APPLY_FAILED
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 5  # ERROR_CONFIG
  quick_action: 9  # QA_ESCALATE
  component: L10.FW.FIRMWARE
  test_cases: [SohuSetClockProfileMultiChipTestCase]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    Board DVFS profile detection or application failed
  probable_causes:
    - Profile missing for this board revision
    - Board ID readback fault
  capture:
    - suite_run_url
    - per-test log and transcript
    - "unit_sn, fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Verify the board ID and profile tables.", retest: "affected_subset", est_time_min: 10}
    - {step: 2, who: "escalate", action: "Escalate if the profile is missing for this board revision.", retest: "n/a", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: []
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-FW-0034
  name: SOHU_HOT_RELOAD_INELIGIBLE
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 5  # ERROR_CONFIG
  quick_action: 9  # QA_ESCALATE
  component: L10.FW.FIRMWARE
  test_cases: [MultiChipSohuHotReloadFirmwareUpdateTestCase]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    Hot reload could not start, so no firmware was written
  probable_causes:
    - "Test-system, station, or configuration problem covering FW"
    - "Missing, malformed, or unreachable input required by SOHU_HOT_RELOAD_INELIGIBLE"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): MultiChipSohuHotReloadFirmwareUpdateTestCase"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [FW]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-FW-0035
  name: SOHU_SECURITY_STATE_QUERY_FAILED
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.FW.SOHU
  test_cases: [CheckSohuSecurityState]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The running Sohu firmware did not report its security mode or TPM state,
    so the required provisioning could not be judged.
  probable_causes:
    - Older firmware not supporting this feature
    - Transient hardware failure
  capture:
    - suite_run_url
    - "failing test case(s): CheckSohuSecurityState"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.260
  suspected_components: [SOHU_FIRMWARE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-FW-0036
  name: SOHU_SECURITY_MODE_MISMATCH
  version: 1
  category: 10  # CATEGORY_SECURITY
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.FW.SOHU
  test_cases: [CheckSohuSecurityState]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The running Sohu firmware does not enforce the required security mode, so
    the unit is not running production-locked firmware.
  probable_causes:
    - "Debug-signed or development firmware left on the unit after provisioning"
    - "Firmware update to the pinned production release did not take effect"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckSohuSecurityState"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.260
  suspected_components: [SOHU_FIRMWARE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-FW-0037
  name: SOHU_TPM_PROVISIONING_MISMATCH
  version: 1
  category: 10  # CATEGORY_SECURITY
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.FW.SOHU
  test_cases: [CheckSohuSecurityState]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The Sohu TPM does not carry the required provisioning: its
    anti-rollback state is not permanent, or debug signing keys are not revoked.
  probable_causes:
    - "TPM provisioned with a reclaimable (prod-test) profile instead of the permanent one"
    - "TPM provisioning skipped, interrupted, or not activated by a power cycle"
    - "Debug signing-key revocation not applied at provisioning"
  capture:
    - suite_run_url
    - "failing test case(s): CheckSohuSecurityState"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.260
  suspected_components: [SOHU_FIRMWARE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-FW-0038
  name: SOHU_SPDM_ATTESTATION_FAILED
  version: 1
  category: 10  # CATEGORY_SECURITY
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.FW.SOHU
  test_cases: [SpdmAttestMultiChipTestCase]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    A Sohu did not attest over SPDM: its certificate chain does not validate
    against the required trust anchor, its measurements do not match the
    policy, it did not answer, or its attestation report is unusable.
  probable_causes:
    - "TPM identity issued under the debug PKI on a production-signed image"
    - "Firmware image or measured-boot state not matching the attestation policy"
    - "SPDM responder not answering over the PCIe MCTP tunnel"
  capture:
    - suite_run_url
    - "failing test case(s): SpdmAttestMultiChipTestCase"
    - per-test log, per-chip verifier transcript and attestation report
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-chip transcript, report and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.260
  suspected_components: [SOHU_FIRMWARE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-FW-0039
  name: SOHU_SPDM_ATTESTATION_UNAVAILABLE
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.HAR.HARNESS
  test_cases: [SpdmAttestMultiChipTestCase]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The station could not run SPDM attestation for one or more chips: the PCIe
    MCTP tunnel could not be served or reached, or the verifier could not be
    launched or its output written. The affected chips were not judged.
  probable_causes:
    - "Another MCTP proxy holding the tunnel port"
    - "RPC proxy unreachable from the station"
    - "spdm_verifier binary not runnable on the station, or its artifact directory not writable"
  capture:
    - suite_run_url
    - "failing test case(s): SpdmAttestMultiChipTestCase"
    - per-test log and MCTP bridge frame trace
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, proxy deployment, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.260
  suspected_components: [TEST_INFRASTRUCTURE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-FW-0040
  name: SOHU_DEBUG_KEY_REVOCATION_FAILED
  version: 1
  category: 10  # CATEGORY_SECURITY
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.FW.SOHU
  test_cases: [RevokeSohuDebugKeys]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The root of trust refused or did not apply the debug signing-key
    revocation, or the revoked mask read back after reset does not cover
    every debug slot.
  probable_causes:
    - "Running firmware is not production signed, so the root of trust refused the request"
    - "Root-of-trust write to the revocation mask failed"
    - "Revocation mask read back after reset does not match the request"
  capture:
    - suite_run_url
    - "failing test case(s): RevokeSohuDebugKeys"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.265
  suspected_components: [SOHU_FIRMWARE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-FW-0010
  name: EXPECTED_VERSION_CONFIG_UNAVAILABLE
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 5  # ERROR_CONFIG
  quick_action: 9  # QA_ESCALATE
  component: L10.FW.FIRMWARE
  test_cases: [CheckOs, CheckKernelVersion, CheckSohuFwVersion, CheckPsuFirmwareVersion, CheckCpldBpFirmwareVersion]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    Expected host or component version data could not be loaded; no version
    verdict is available.
  probable_causes:
    - "Test-system, station, or configuration problem covering Firmware"
    - "Missing, malformed, or unreachable input required by EXPECTED_VERSION_CONFIG_UNAVAILABLE"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): CheckOs, CheckKernelVersion, CheckSohuFwVersion, CheckPsuFirmwareVersion, CheckCpldBpFirmwareVersion"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [FIRMWARE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-FW-0011
  name: HOST_OS_VERSION_QUERY_FAILED
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.FW.HOST_OS
  test_cases: [CheckOs]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The host operating-system release could not be queried.
  probable_causes:
    - "Test-system, station, or configuration problem covering Host OS"
    - "Missing, malformed, or unreachable input required by HOST_OS_VERSION_QUERY_FAILED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): CheckOs"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [HOST_OS]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-FW-0012
  name: HOST_OS_VERSION_OUTPUT_INVALID
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.FW.HOST_OS
  test_cases: [CheckOs]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    Operating-system release output was incomplete or could not be parsed.
  probable_causes:
    - "Test-system, station, or configuration problem covering Host OS"
    - "Missing, malformed, or unreachable input required by HOST_OS_VERSION_OUTPUT_INVALID"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): CheckOs"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [HOST_OS]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-FW-0013
  name: HOST_OS_VERSION_MISMATCH
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 5  # ERROR_CONFIG
  quick_action: 9  # QA_ESCALATE
  component: L10.FW.HOST_OS
  test_cases: [CheckOs]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The host operating-system release does not match the required L10 image.
  probable_causes:
    - Defective or out-of-spec Host OS on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckOs"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [HOST_OS]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-FW-0014
  name: HOST_KERNEL_VERSION_QUERY_FAILED
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.FW.HOST_KERNEL
  test_cases: [CheckKernelVersion]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The running host kernel version could not be queried.
  probable_causes:
    - "Test-system, station, or configuration problem covering Host Kernel"
    - "Missing, malformed, or unreachable input required by HOST_KERNEL_VERSION_QUERY_FAILED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): CheckKernelVersion"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [HOST_KERNEL]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-FW-0015
  name: HOST_KERNEL_VERSION_OUTPUT_INVALID
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.FW.HOST_KERNEL
  test_cases: [CheckKernelVersion]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    Kernel version output was empty or could not be parsed.
  probable_causes:
    - "Test-system, station, or configuration problem covering Host Kernel"
    - "Missing, malformed, or unreachable input required by HOST_KERNEL_VERSION_OUTPUT_INVALID"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): CheckKernelVersion"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [HOST_KERNEL]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-FW-0016
  name: HOST_KERNEL_VERSION_MISMATCH
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 5  # ERROR_CONFIG
  quick_action: 9  # QA_ESCALATE
  component: L10.FW.HOST_KERNEL
  test_cases: [CheckKernelVersion]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The running host kernel does not match the required L10 release.
  probable_causes:
    - Defective or out-of-spec Host Kernel on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckKernelVersion"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [HOST_KERNEL]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-FW-0017
  name: SOHU_FIRMWARE_VERSION_QUERY_FAILED
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.FW.SOHU
  test_cases: [CheckSohuFwVersion]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The running Sohu firmware version could not be queried.
  probable_causes:
    - "Test-system, station, or configuration problem covering Sohu Firmware"
    - "Missing, malformed, or unreachable input required by SOHU_FIRMWARE_VERSION_QUERY_FAILED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): CheckSohuFwVersion"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [SOHU_FIRMWARE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-FW-0018
  name: SOHU_FIRMWARE_VERSION_MISMATCH
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 5  # ERROR_CONFIG
  quick_action: 9  # QA_ESCALATE
  component: L10.FW.SOHU
  test_cases: [CheckSohuFwVersion]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The running Sohu firmware version does not match the required L10 release.
  probable_causes:
    - Defective or out-of-spec Sohu Firmware on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckSohuFwVersion"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [SOHU_FIRMWARE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-FW-0019
  name: PSU_FIRMWARE_VERSION_QUERY_FAILED
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.FW.PSU
  test_cases: [CheckPsuFirmwareVersion]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    One or more PSU firmware versions could not be queried.
  probable_causes:
    - "Test-system, station, or configuration problem covering PSU Firmware"
    - "Missing, malformed, or unreachable input required by PSU_FIRMWARE_VERSION_QUERY_FAILED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): CheckPsuFirmwareVersion"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [PSU_FIRMWARE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-FW-0020
  name: PSU_FIRMWARE_VERSION_MISMATCH
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 5  # ERROR_CONFIG
  quick_action: 9  # QA_ESCALATE
  component: L10.FW.PSU
  test_cases: [CheckPsuFirmwareVersion]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    A PSU firmware version does not match the required L10 release.
  probable_causes:
    - Defective or out-of-spec PSU Firmware on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckPsuFirmwareVersion"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [PSU_FIRMWARE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-FW-0021
  name: CPLD_BP_FIRMWARE_VERSION_QUERY_FAILED
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.FW.CPLD
  test_cases: [CheckCpldBpFirmwareVersion]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The backplane/CPLD firmware version could not be queried.
  probable_causes:
    - "Test-system, station, or configuration problem covering CPLD Firmware"
    - "Missing, malformed, or unreachable input required by CPLD_BP_FIRMWARE_VERSION_QUERY_FAILED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): CheckCpldBpFirmwareVersion"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [CPLD_FIRMWARE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-FW-0022
  name: CPLD_BP_FIRMWARE_VERSION_MISMATCH
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 5  # ERROR_CONFIG
  quick_action: 9  # QA_ESCALATE
  component: L10.FW.CPLD
  test_cases: [CheckCpldBpFirmwareVersion]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The backplane/CPLD firmware version does not match the required L10 release.
  probable_causes:
    - Defective or out-of-spec CPLD Firmware on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckCpldBpFirmwareVersion"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [CPLD_FIRMWARE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-FW-0023
  name: PSU_FIRMWARE_UPDATE_FAILED
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.FW.PSU
  test_cases: [PsuFirmwareUpdate]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The PSU firmware update command failed or did not complete successfully.
  probable_causes:
    - Defective or out-of-spec PSU Firmware on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): PsuFirmwareUpdate"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [PSU_FIRMWARE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-FW-0024
  name: PSU_POST_UPDATE_VERIFY_FAILED
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.FW.PSU
  test_cases: [PsuFirmwareUpdate]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    A PSU did not report the requested firmware version after update.
  probable_causes:
    - Defective or out-of-spec PSU Firmware on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): PsuFirmwareUpdate"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [PSU_FIRMWARE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-FW-0025
  name: CPLD_FIRMWARE_UPDATE_FAILED
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.FW.CPLD
  test_cases: [CpldFirmwareUpdate]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The CPLD/backplane firmware update command failed or did not complete
    successfully.
  probable_causes:
    - Defective or out-of-spec CPLD Firmware on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CpldFirmwareUpdate"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [CPLD_FIRMWARE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-FW-0026
  name: CPLD_POST_UPDATE_VERIFY_FAILED
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.FW.CPLD
  test_cases: [CpldFirmwareUpdate]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The CPLD/backplane did not report the requested firmware version after
    update.
  probable_causes:
    - Defective or out-of-spec CPLD Firmware on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CpldFirmwareUpdate"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [CPLD_FIRMWARE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-FW-0027
  name: SOFTWARE_PROVISIONING_FAILED
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.FW.HOST_SW
  test_cases: [SoftwareProvisioning]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The L10 host software provisioning workflow failed to apply the required
    software state.
  probable_causes:
    - Defective or out-of-spec Host Software on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): SoftwareProvisioning"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [HOST_SOFTWARE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-FW-0028
  name: LINUX_PACKAGE_INSTALL_FAILED
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.FW.HOST_SW
  test_cases: [InstallLinuxPackagesTestCase]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    A required Linux package could not be installed or configured.
  probable_causes:
    - Defective or out-of-spec Host Software on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): InstallLinuxPackagesTestCase"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [HOST_SOFTWARE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-FW-0029
  name: SOHU_BOOTLOADER_UPDATE_FAILED
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.FW.SOHU_BOOTLOADER
  test_cases: [UpdateSohuBootloader]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The Sohu bootloader update command failed or the device was not usable after
    programming.
  probable_causes:
    - Defective or out-of-spec Sohu Bootloader on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): UpdateSohuBootloader"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [SOHU_BOOTLOADER]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-FW-0030
  name: SOHU_BOOTLOADER_POST_UPDATE_VERIFY_FAILED
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.FW.SOHU_BOOTLOADER
  test_cases: [UpdateSohuBootloader]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The running Sohu bootloader does not match the requested image after update.
  probable_causes:
    - Defective or out-of-spec Sohu Bootloader on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): UpdateSohuBootloader"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [SOHU_BOOTLOADER]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-FW-0031
  name: HOST_SYSTEM_LOG_CRITICAL_EVENT_DETECTED
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.FW.HOST_SW
  test_cases: [CheckSystemLogStates]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    A new critical host kernel, service, or system event was detected after the
    L10 workload.
  probable_causes:
    - Defective or out-of-spec Host Software on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckSystemLogStates"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [HOST_SOFTWARE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-HAR-0001
  name: TEST_TIMEOUT
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 0  # QA_NO_ACT
  component: L10.HAR.HARNESS
  test_cases: []  # harness-level; applies to any L10 test case
  disposition: ABORTED
  retryable: false
  max_auto_retry: 0
  description: >
    The test exceeded its configured timeout and was terminated before producing
    a verdict (ABORTED disposition).
  probable_causes:
    - DUT hang or severe slowdown under the test workload
    - Test timeout configured too low for this platform
    - Harness/executor stall (e.g. remote connection loss)
  capture:
    - suite_run_url
    - per-test log and transcript
    - configured timeout and measured duration
    - "harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Rerun the affected subset once to check reproducibility.", retest: "affected_subset", est_time_min: 10}
    - {step: 2, who: "escalate", action: "Route to engineering triage with the per-test log and timeout configuration.", retest: "n/a", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: []
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-HAR-0002
  name: TEST_ARGS_INVALID
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 5  # ERROR_CONFIG
  quick_action: 9  # QA_ESCALATE
  component: L10.HAR.HARNESS
  test_cases: [CpuStress, MemoryStressNg, MemoryStress, IoStress, SohuSnapTestCase, ExecBinaryTestCase]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    A test case was invoked with invalid or unsupported arguments and rejected
    the run before touching the DUT.
  probable_causes:
    - Suite YAML passes arguments the test does not accept
    - Suite YAML requests a mode/path the test does not support
  capture:
    - suite_run_url
    - the offending args from the per-test log
    - "harness_version, station_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the suite configuration for this test; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 10}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: []
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-HAR-0003
  name: TEST_DEPENDENCY_UNAVAILABLE
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 5  # ERROR_CONFIG
  quick_action: 9  # QA_ESCALATE
  component: L10.HAR.HARNESS
  test_cases:
    - CpuStress
    - MemoryStressNg
    - MemoryStress
    - IoStress
    - LoopbackTopologyTest
    - NetworkBwRdmaTest
    - ModelRegistryNfsMountTestCase
    - StageModelRegistryToShmTestCase
    - SohuServerBmcFirmwareUpdate
    - SohuServerBiosUpdate
    - PsuFirmwareUpdate
    - CpldFirmwareUpdate
    - L10VBBUpdate
    - InstallLinuxPackagesTestCase
    - UpdatePlxDaemonFirmwareVersion
    - BroadcomUpdateFwAndConfigure
    - UpdateSohuBootloader
    - MultiChipSohuHotReloadFirmwareUpdateTestCase
    - ExecBinaryTestCase
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    Tool/binary/data file the test depends on is missing on the test system
    (test-system ERROR — excluded from module yield)
  probable_causes:
    - Test package/runfiles deployment gap on the station
    - Harness release missing a data dependency
  capture:
    - suite_run_url
    - the missing path from the per-test log
    - "harness_version, station_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station deployment / harness package; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: []
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-HAR-0004
  name: NESTED_SUITE_ORCHESTRATION_FAILED
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.HAR.HARNESS
  test_cases: [RunInStress]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    Nested suite failed to launch or complete orchestration (child test failures
    carry their own codes)
  probable_causes:
    - Suite configuration or resource wiring error in the nested wrapper
    - Child suite could not be launched at all
  capture:
    - suite_run_url
    - per-test log and transcript
    - "unit_sn, fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Check the suite config and resources for the nested wrapper.", retest: "affected_subset", est_time_min: 10}
    - {step: 2, who: "escalate", action: "Route to test-infra; do NOT route the unit to rework or FA.", retest: "n/a", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: []
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-HAR-0005
  name: TEST_TOOL_EXECUTION_FAILED
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 6  # QA_RETRY
  component: L10.HAR.HARNESS
  test_cases: [IoStress, ExecBinaryTestCase, CpldDiagnosticsRemoteTestCase]
  disposition: ERROR
  retryable: true
  max_auto_retry: 1
  description: >
    A test-owned command or binary failed to execute and did not produce a
    domain-specific DUT verdict.
  probable_causes:
    - "Test-system, station, or configuration problem covering Harness"
    - "Missing, malformed, or unreachable input required by TEST_TOOL_EXECUTION_FAILED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): IoStress, ExecBinaryTestCase, CpldDiagnosticsRemoteTestCase"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [HARNESS]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-HAR-0006
  name: TEST_METRICS_PARSE_FAILED
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.HAR.HARNESS
  test_cases: [IoStress, ExecBinaryTestCase]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    A test tool completed without complete, parseable metrics needed to
    determine the result.
  probable_causes:
    - "Test-system, station, or configuration problem covering Harness"
    - "Missing, malformed, or unreachable input required by TEST_METRICS_PARSE_FAILED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): IoStress, ExecBinaryTestCase"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [HARNESS]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-HAR-0007
  name: TEST_OUTPUT_UNAVAILABLE
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.HAR.HARNESS
  test_cases: [StoreSystemLogStates, CheckSystemLogStates, ExecBinaryTestCase]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    Expected test output, state, or result artifacts were not produced.
  probable_causes:
    - "Test-system, station, or configuration problem covering Harness"
    - "Missing, malformed, or unreachable input required by TEST_OUTPUT_UNAVAILABLE"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): StoreSystemLogStates, CheckSystemLogStates, ExecBinaryTestCase"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [HARNESS]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-HAR-0008
  name: TEST_PRECONDITION_NOT_MET
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 5  # ERROR_CONFIG
  quick_action: 9  # QA_ESCALATE
  component: L10.HAR.HARNESS
  test_cases: [RunInStress, IoStress, NetworkBwRdmaTest]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    A required test precondition was not satisfied, so the test did not exercise
    the DUT as intended.
  probable_causes:
    - "Test-system, station, or configuration problem covering Harness"
    - "Missing, malformed, or unreachable input required by TEST_PRECONDITION_NOT_MET"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): RunInStress, IoStress, NetworkBwRdmaTest"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [HARNESS]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-HAR-0009
  name: TEST_CLEANUP_FAILED
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.HAR.HARNESS
  test_cases: [LoopbackTopologyTest, NetworkBwRdmaTest, FreeModelRegistryFromShmTestCase, FreeWeightsTestCase]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    Test cleanup failed and may have left processes, mounts, namespaces, files,
    or device state behind.
  probable_causes:
    - "Test-system, station, or configuration problem covering Harness"
    - "Missing, malformed, or unreachable input required by TEST_CLEANUP_FAILED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): LoopbackTopologyTest, NetworkBwRdmaTest, FreeModelRegistryFromShmTestCase, FreeWeightsTestCase"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [HARNESS]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-HAR-0010
  name: SYSTEM_LOG_SNAPSHOT_FAILED
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.HAR.HARNESS
  test_cases: [StoreSystemLogStates]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The pre-test system-log baseline could not be captured.
  probable_causes:
    - "Test-system, station, or configuration problem covering Harness"
    - "Missing, malformed, or unreachable input required by SYSTEM_LOG_SNAPSHOT_FAILED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): StoreSystemLogStates"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [HARNESS]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-HAR-0011
  name: SYSTEM_LOG_COMPARE_FAILED
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.HAR.HARNESS
  test_cases: [CheckSystemLogStates]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    System-log state could not be queried, parsed, or compared with the stored
    baseline.
  probable_causes:
    - "Test-system, station, or configuration problem covering Harness"
    - "Missing, malformed, or unreachable input required by SYSTEM_LOG_COMPARE_FAILED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): CheckSystemLogStates"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [HARNESS]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-HAR-9998
  name: TEST_FAILED_UNCLASSIFIED
  version: 1
  category: 0  # CATEGORY_NONE
  severity: 3  # ERROR_UNKNOWN
  quick_action: 0  # QA_NO_ACT
  component: L10.HAR.HARNESS
  test_cases: []  # harness-level; applies to any L10 test case
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    A test returned TestFailedError without a specific TH identity; this
    catch-all is a taxonomy-health failure and must be replaced by a specific
    code.
  probable_causes:
    - Test case not yet migrated to specific TH error codes
  capture:
    - suite_run_url
    - per-test log and transcript
    - "harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Follow the failing test case's own documentation/SOP.", retest: "affected_subset", est_time_min: 10}
    - {step: 2, who: "escalate", action: "Open a taxonomy-gap ticket to allocate a specific code for this failure mode.", retest: "n/a", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: []
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-HAR-9999
  name: UNHANDLED_EXCEPTION
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 3  # ERROR_UNKNOWN
  quick_action: 9  # QA_ESCALATE
  component: L10.HAR.HARNESS
  test_cases: []  # harness-level; applies to any L10 test case
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    An unhandled exception escaped the test and was wrapped by the harness; the
    traceback is retained in the per-test log (ERROR disposition).
  probable_causes:
    - Bug in the test case or harness code
    - Unexpected environment state the test does not handle
  capture:
    - suite_run_url
    - per-test log with full traceback
    - "harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "escalate", action: "Route to the owning test/SW engineer with the traceback; do NOT route the unit to rework or FA.", retest: "n/a", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: []
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-HBM-0009
  name: HBM_PMBIST_FAILED
  version: 1
  category: 3  # CATEGORY_MEMORY
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.HBM.HBM
  test_cases: [SohuPmbistMultiChipTestCase]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    HBM PMBIST failure
  probable_causes:
    - HBM stack cell or repair-map fault
    - Marginal HBM interface at speed
  capture:
    - suite_run_url
    - per-test log and transcript
    - "unit_sn, fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Rerun up to 3 times (stack in affected_instances); isolate.", retest: "affected_subset", est_time_min: 10}
    - {step: 2, who: "escalate", action: "Isolate the module after retry exhaustion; escalate to Etched with the captured evidence.", retest: "n/a", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: []
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-HBM-0010
  name: HBM_MTC_STRESS_FAILED
  version: 1
  category: 3  # CATEGORY_MEMORY
  severity: 2  # ERROR_ISOLATE
  quick_action: 6  # QA_RETRY
  component: L10.HBM.HBM
  test_cases: [SohuMtcStressMultiChipTestCase]
  disposition: FAILED
  retryable: true
  max_auto_retry: 1
  description: >
    HBM MTC stress test failure
  probable_causes:
    - HBM stack fault under stress
    - Marginal pseudo-channel at speed
  capture:
    - suite_run_url
    - per-test log and transcript
    - "unit_sn, fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Rerun up to 3 times (stack+pseudo-channel in affected_instances); isolate.", retest: "affected_subset", est_time_min: 10}
    - {step: 2, who: "escalate", action: "Isolate the module after retry exhaustion; escalate to Etched with the captured evidence.", retest: "n/a", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: []
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-HBM-0011
  name: HBM_DMA_BANDWIDTH_BELOW_THRESHOLD
  version: 1
  category: 3  # CATEGORY_MEMORY
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.HBM.HBM
  test_cases: [SohuDmaMultiChipTestCase]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    PCIe-to-HBM read or write throughput on the side under test did not reach
    the configured min_bandwidth_gb_s floor.
  probable_causes:
    - HBM channel running below rate after training
    - Degraded or repaired HBM lanes reducing effective width
    - Sohu HBM controller or DMA engine underperforming
  capture:
    - suite_run_url
    - "measured bandwidth vs configured min_bandwidth_gb_s, and the side tested"
    - per-transfer DMA timestamps recorded by the test
    - "unit_sn, fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "escalate", action: "Escalate to Etched with the measured bandwidth, the side tested, and the configured threshold.", retest: "n/a", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [HBM_STACK, ASIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-HBM-0006
  name: HBM_LANE_REPAIR_FAILED
  version: 1
  category: 3  # CATEGORY_MEMORY
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.HBM.HBM
  test_cases: [SohuLaneRepairMultiChipTestCase]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    HBM lane repair failed - no passing repair found
  probable_causes:
    - More failing lanes on the pseudo-channel than there are spares
    - HBM stack or interposer defect beyond lane-repair coverage
    - Sohu HBM PHY fault on the affected channel
  capture:
    - suite_run_url
    - hbm_lane_repairs JSON artifact (per-target repair map)
    - Lane Failures measurement (HBM/channel/pseudo-channel of each failure)
    - "unit_sn, fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "escalate", action: "Escalate to Etched with the lane-repair map; the stack is unrepairable, so screen the module out rather than reworking it.", retest: "n/a", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [HBM_STACK, INTERPOSER, ASIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-HBM-0012
  name: HBM_DEVICE_ID_MISMATCH
  version: 1
  category: 11  # CATEGORY_DATA
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.HBM.HBM
  test_cases: [SohuHbmDeviceIdMultiChipTestCase]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    HBM device ID read failure or mismatch
  probable_causes:
    - HBM stack not responding on its IEEE 1500 port
    - "Stack absent, misassembled, or not trained during bring-up"
    - Sohu HBM controller fault on the affected stack
  capture:
    - suite_run_url
    - per-test log naming the failing stack and the underlying error
    - reported HBM system info (stack count and channel ids)
    - recorded per-stack serial_number / manufacturer / density measurements
    - "unit_sn, fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "escalate", action: "Escalate to Etched with the failing stack id and the reported HBM system info.", retest: "n/a", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [HBM_STACK, ASIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-HBM-0013
  name: HBM_LANE_REPAIR_INCONCLUSIVE
  version: 1
  category: 3  # CATEGORY_MEMORY
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.HBM.HBM
  test_cases: [SohuLaneRepairMultiChipTestCase]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    HBM lane repair sweep did not complete, so repairability was never
    established
  probable_causes:
    - "Test-system, station, or configuration problem covering HBM"
    - "Missing, malformed, or unreachable input required by HBM_LANE_REPAIR_INCONCLUSIVE"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): SohuLaneRepairMultiChipTestCase"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [HBM]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-INFRA-0001
  name: RACKSIM_UNAVAILABLE
  version: 1
  category: 9  # CATEGORY_RESOURCE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.INFRA.STATION
  test_cases: [RunInStress, ExecBinaryTestCase]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The configured racksim service or simulated server endpoint is unavailable.
  probable_causes:
    - "Test-system, station, or configuration problem covering Test Infrastructure"
    - "Missing, malformed, or unreachable input required by RACKSIM_UNAVAILABLE"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): RunInStress, ExecBinaryTestCase"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [TEST_INFRASTRUCTURE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-INFRA-0002
  name: REMOTE_EXECUTION_TRANSPORT_FAILED
  version: 1
  category: 1  # CATEGORY_NETWORK
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.INFRA.STATION
  test_cases: [CpldDiagnosticsRemoteTestCase, ExecBinaryTestCase]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The infrastructure transport used to execute a remote L10 operation failed
    before a DUT verdict was obtained.
  probable_causes:
    - "Test-system, station, or configuration problem covering Test Infrastructure"
    - "Missing, malformed, or unreachable input required by REMOTE_EXECUTION_TRANSPORT_FAILED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): CpldDiagnosticsRemoteTestCase, ExecBinaryTestCase"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [TEST_INFRASTRUCTURE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-INFRA-0003
  name: INFRA_NETWORK_PATH_UNAVAILABLE
  version: 1
  category: 1  # CATEGORY_NETWORK
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.INFRA.STATION
  test_cases: [ModelRegistryNfsMountTestCase, BmcConnectivityCheck]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    A required station-side network path, route, or service endpoint is
    unavailable.
  probable_causes:
    - "Test-system, station, or configuration problem covering Test Infrastructure"
    - "Missing, malformed, or unreachable input required by INFRA_NETWORK_PATH_UNAVAILABLE"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): ModelRegistryNfsMountTestCase, BmcConnectivityCheck"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [TEST_INFRASTRUCTURE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-INFRA-0004
  name: TEST_RESOURCE_EXHAUSTED
  version: 1
  category: 9  # CATEGORY_RESOURCE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.INFRA.STATION
  test_cases: [StageModelRegistryToShmTestCase, StageWeightsTestCase, RunInStress]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    A station-side resource limit prevented the L10 test from running or
    completing.
  probable_causes:
    - "Test-system, station, or configuration problem covering Test Infrastructure"
    - "Missing, malformed, or unreachable input required by TEST_RESOURCE_EXHAUSTED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): StageModelRegistryToShmTestCase, StageWeightsTestCase, RunInStress"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [TEST_INFRASTRUCTURE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-MEM-0001
  name: MEMORY_INVENTORY_QUERY_FAILED
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.MEM.HOST_DIMM
  test_cases: [CheckMemory]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The host memory inventory command failed; no DIMM verdict is available.
  probable_causes:
    - "Test-system, station, or configuration problem covering Host Memory"
    - "Missing, malformed, or unreachable input required by MEMORY_INVENTORY_QUERY_FAILED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): CheckMemory"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [HOST_MEMORY]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-MEM-0002
  name: MEMORY_INVENTORY_OUTPUT_INVALID
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.MEM.HOST_DIMM
  test_cases: [CheckMemory]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The host memory inventory output was malformed or could not be parsed.
  probable_causes:
    - "Test-system, station, or configuration problem covering Host Memory"
    - "Missing, malformed, or unreachable input required by MEMORY_INVENTORY_OUTPUT_INVALID"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): CheckMemory"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [HOST_MEMORY]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-MEM-0003
  name: NO_DIMMS_DETECTED
  version: 1
  category: 3  # CATEGORY_MEMORY
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.MEM.HOST_DIMM
  test_cases: [CheckMemory]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    No populated DIMMs were detected although the L10 BOM requires host memory.
  probable_causes:
    - Defective or out-of-spec Host Memory on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckMemory"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [HOST_MEMORY]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-MEM-0004
  name: DIMM_COUNT_MISMATCH
  version: 1
  category: 3  # CATEGORY_MEMORY
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.MEM.HOST_DIMM
  test_cases: [CheckMemory]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The number of populated DIMMs does not match the expected L10 BOM.
  probable_causes:
    - Defective or out-of-spec Host Memory on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckMemory"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [HOST_MEMORY]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-MEM-0005
  name: DIMM_IDENTITY_OR_CAPACITY_MISMATCH
  version: 1
  category: 11  # CATEGORY_DATA
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.MEM.HOST_DIMM
  test_cases: [CheckMemory]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    A DIMM manufacturer, part number, serial number, location, or capacity does
    not match the expected BOM.
  probable_causes:
    - Defective or out-of-spec Host Memory on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckMemory"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [HOST_MEMORY]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-MEM-0006
  name: MIXED_DIMM_CONFIGURATION
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.MEM.HOST_DIMM
  test_cases: [CheckMemory]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The installed DIMMs are not homogeneous where the platform configuration
    requires matching parts and capacities.
  probable_causes:
    - Defective or out-of-spec Host Memory on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckMemory"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [HOST_MEMORY]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-MEM-0007
  name: MEMORY_STRESS_FAILED
  version: 1
  category: 3  # CATEGORY_MEMORY
  severity: 2  # ERROR_ISOLATE
  quick_action: 6  # QA_RETRY
  component: L10.MEM.HOST_DIMM
  test_cases: [MemoryStressNg, MemoryStress]
  disposition: FAILED
  retryable: true
  max_auto_retry: 1
  description: >
    The host memory stress workload reported data errors, verification failures,
    or a workload-failure exit status.
  probable_causes:
    - Defective or out-of-spec Host Memory on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): MemoryStressNg, MemoryStress"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Retry the affected L10 subset once and capture logs from the failing run.", retest: "affected_subset", est_time_min: 10}
    - {step: 2, who: "escalate", action: "If persistent, isolate the unit; attach logs; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [HOST_MEMORY]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-MEM-0008
  name: MEMORY_STRESS_METRICS_INVALID
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.MEM.HOST_DIMM
  test_cases: [MemoryStressNg, MemoryStress]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The memory stress tool completed without usable result statistics or
    produced malformed metrics, so the verdict is inconclusive.
  probable_causes:
    - "Test-system, station, or configuration problem covering Host Memory"
    - "Missing, malformed, or unreachable input required by MEMORY_STRESS_METRICS_INVALID"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): MemoryStressNg, MemoryStress"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [HOST_MEMORY]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-MEM-0009
  name: NUMA_TOPOLOGY_PRECONDITION_NOT_MET
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 5  # ERROR_CONFIG
  quick_action: 9  # QA_ESCALATE
  component: L10.MEM.HOST_DIMM
  test_cases: [IoStress]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The host NUMA/SNC topology does not provide the nodes required by the
    configured cross-NUMA I/O stress.
  probable_causes:
    - "Test-system, station, or configuration problem covering Host Memory"
    - "Missing, malformed, or unreachable input required by NUMA_TOPOLOGY_PRECONDITION_NOT_MET"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): IoStress"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [HOST_MEMORY]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-MEM-0010
  name: CROSS_NUMA_MEMORY_STRESS_FAILED
  version: 1
  category: 3  # CATEGORY_MEMORY
  severity: 2  # ERROR_ISOLATE
  quick_action: 6  # QA_RETRY
  component: L10.MEM.HOST_DIMM
  test_cases: [IoStress]
  disposition: FAILED
  retryable: true
  max_auto_retry: 1
  description: >
    The cross-NUMA stream or memory-transfer workload failed its execution or
    acceptance criteria.
  probable_causes:
    - Defective or out-of-spec Host Memory on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): IoStress"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Retry the affected L10 subset once and capture logs from the failing run.", retest: "affected_subset", est_time_min: 10}
    - {step: 2, who: "escalate", action: "If persistent, isolate the unit; attach logs; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [HOST_MEMORY]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-MEZZ-0004
  name: MEZZ_CPLD_ACCESS_FAILED
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 2  # ERROR_ISOLATE
  quick_action: 6  # QA_RETRY
  component: L10.MEZZ.CPLD
  test_cases: [CpldDiagnosticsRemoteTestCase]
  disposition: FAILED
  retryable: true
  max_auto_retry: 1
  description: >
    Mezz/eval CPLD register access or write failure
  probable_causes:
    - Mezzanine or eval-board CPLD register access failed
  capture:
    - suite_run_url
    - per-test log and transcript
    - CPLD operation transcript
    - "harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Retest the affected test once.", retest: "affected_subset"}
    - {step: 2, who: "escalate", action: "Escalate to Etched if the failure repeats.", retest: "n/a"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [MEZZ]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-NIC-0001
  name: NIC_EXPECTED_CONFIG_UNAVAILABLE
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 5  # ERROR_CONFIG
  quick_action: 9  # QA_ESCALATE
  component: L10.NIC.NIC
  test_cases: [CheckNic, CheckLomNicMac, LoopbackTopologyTest, NetworkBwRdmaTest, CheckNicAsicRails]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The expected NIC/BOM/network configuration could not be loaded; no NIC
    verdict is available.
  probable_causes:
    - "Test-system, station, or configuration problem covering NIC"
    - "Missing, malformed, or unreachable input required by NIC_EXPECTED_CONFIG_UNAVAILABLE"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): CheckNic, CheckLomNicMac, LoopbackTopologyTest, NetworkBwRdmaTest, CheckNicAsicRails"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-NIC-0002
  name: NIC_INVENTORY_QUERY_FAILED
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.NIC.NIC
  test_cases: [CheckNic, CheckLomNicMac, LoopbackTopologyTest, NetworkBwRdmaTest]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    NIC inventory or interface discovery failed.
  probable_causes:
    - "Test-system, station, or configuration problem covering NIC"
    - "Missing, malformed, or unreachable input required by NIC_INVENTORY_QUERY_FAILED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): CheckNic, CheckLomNicMac, LoopbackTopologyTest, NetworkBwRdmaTest"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-NIC-0003
  name: NIC_DISCOVERY_SERVICE_UNAVAILABLE
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.NIC.NIC
  test_cases: [CheckNic, CheckInterfaceLinkStatus, CheckNetdReadiness]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The NIC discovery/netd service or its Unix-domain socket was not ready or
    reachable.
  probable_causes:
    - "Test-system, station, or configuration problem covering NIC"
    - "Missing, malformed, or unreachable input required by NIC_DISCOVERY_SERVICE_UNAVAILABLE"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): CheckNic, CheckInterfaceLinkStatus, CheckNetdReadiness"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-NIC-0004
  name: NIC_COUNT_MISMATCH
  version: 1
  category: 1  # CATEGORY_NETWORK
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.NIC.NIC
  test_cases: [CheckNic]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The discovered NIC count does not match the expected L10 BOM.
  probable_causes:
    - Defective or out-of-spec NIC on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckNic"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-NIC-0005
  name: NIC_IDENTITY_MISMATCH
  version: 1
  category: 11  # CATEGORY_DATA
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.NIC.NIC
  test_cases: [CheckNic]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    A NIC vendor, device, name, speed capability, or expected grouping does not
    match configuration.
  probable_causes:
    - Defective or out-of-spec NIC on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckNic"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-NIC-0006
  name: NIC_DUPLICATE_PCI_ADDRESS
  version: 1
  category: 11  # CATEGORY_DATA
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.NIC.NIC
  test_cases: [CheckNic]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    Multiple NIC records resolve to the same PCI address.
  probable_causes:
    - Defective or out-of-spec NIC on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckNic"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-NIC-0007
  name: UNEXPECTED_NIC_DETECTED
  version: 1
  category: 1  # CATEGORY_NETWORK
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.NIC.NIC
  test_cases: [CheckNic]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    A NIC that is not present in the expected L10 configuration was discovered.
  probable_causes:
    - Defective or out-of-spec NIC on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckNic"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-NIC-0008
  name: LOM_NIC_NOT_DETECTED
  version: 1
  category: 1  # CATEGORY_NETWORK
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.NIC.LOM
  test_cases: [CheckLomNicMac]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The configured LAN-on-motherboard interface was not found.
  probable_causes:
    - Defective or out-of-spec LOM NIC on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckLomNicMac"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [LOM_NIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-NIC-0009
  name: LOM_MAC_ADDRESS_MISMATCH
  version: 1
  category: 11  # CATEGORY_DATA
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.NIC.LOM
  test_cases: [CheckLomNicMac]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The LAN-on-motherboard MAC address does not match the expected service/BOM
    value.
  probable_causes:
    - Defective or out-of-spec LOM NIC on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckLomNicMac"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [LOM_NIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-NIC-0010
  name: NIC_DRIVER_QUERY_FAILED
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.NIC.NIC
  test_cases: [CheckBroadcomDrivers]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The installed NIC driver or firmware version could not be queried.
  probable_causes:
    - "Test-system, station, or configuration problem covering NIC"
    - "Missing, malformed, or unreachable input required by NIC_DRIVER_QUERY_FAILED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): CheckBroadcomDrivers"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-NIC-0011
  name: NIC_DRIVER_VERSION_MISMATCH
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 5  # ERROR_CONFIG
  quick_action: 9  # QA_ESCALATE
  component: L10.NIC.NIC
  test_cases: [CheckBroadcomDrivers]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The installed NIC kernel driver does not match the required L10 release.
  probable_causes:
    - Defective or out-of-spec NIC on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckBroadcomDrivers"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-NIC-0012
  name: NIC_FIRMWARE_VERSION_MISMATCH
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 5  # ERROR_CONFIG
  quick_action: 9  # QA_ESCALATE
  component: L10.NIC.NIC
  test_cases: [CheckBroadcomDrivers]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The installed NIC firmware does not match the required L10 release.
  probable_causes:
    - Defective or out-of-spec NIC on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckBroadcomDrivers"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-NIC-0013
  name: NIC_VERSION_OUTPUT_INVALID
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.NIC.NIC
  test_cases: [CheckBroadcomDrivers]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    NIC driver/firmware version output was incomplete or could not be parsed.
  probable_causes:
    - "Test-system, station, or configuration problem covering NIC"
    - "Missing, malformed, or unreachable input required by NIC_VERSION_OUTPUT_INVALID"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): CheckBroadcomDrivers"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-NIC-0014
  name: NIC_INTERFACE_LINK_DOWN
  version: 1
  category: 1  # CATEGORY_NETWORK
  severity: 2  # ERROR_ISOLATE
  quick_action: 6  # QA_RETRY
  component: L10.NIC.NIC
  test_cases: [CheckInterfaceLinkStatus, LoopbackTopologyTest]
  disposition: FAILED
  retryable: true
  max_auto_retry: 1
  description: >
    A required NIC interface has no carrier or is not operationally up.
  probable_causes:
    - Defective or out-of-spec NIC on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckInterfaceLinkStatus, LoopbackTopologyTest"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Retry the affected L10 subset once and capture logs from the failing run.", retest: "affected_subset", est_time_min: 10}
    - {step: 2, who: "escalate", action: "If persistent, isolate the unit; attach logs; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-NIC-0015
  name: NETD_NOT_READY
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.NIC.NIC
  test_cases: [CheckNetdReadiness]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    netd did not reach its ready state within the configured wait interval.
  probable_causes:
    - "Test-system, station, or configuration problem covering NIC"
    - "Missing, malformed, or unreachable input required by NETD_NOT_READY"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): CheckNetdReadiness"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-NIC-0016
  name: NETD_ACTIVE_SLAVE_MISMATCH
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 5  # ERROR_CONFIG
  quick_action: 9  # QA_ESCALATE
  component: L10.NIC.NIC
  test_cases: [CheckNetdReadiness]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The active bond slave reported by netd is not the configured expected
    interface.
  probable_causes:
    - Defective or out-of-spec NIC on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckNetdReadiness"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-NIC-0017
  name: LOOPBACK_GROUP_COUNT_MISMATCH
  version: 1
  category: 1  # CATEGORY_NETWORK
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.NIC.NIC
  test_cases: [LoopbackTopologyTest, NetworkBwRdmaTest]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    A configured loopback NIC group does not contain the expected number of
    interfaces.
  probable_causes:
    - Defective or out-of-spec NIC on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): LoopbackTopologyTest, NetworkBwRdmaTest"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-NIC-0018
  name: LOOPBACK_TOPOLOGY_INDEX_INVALID
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 5  # ERROR_CONFIG
  quick_action: 9  # QA_ESCALATE
  component: L10.NIC.NIC
  test_cases: [LoopbackTopologyTest, NetworkBwRdmaTest]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    A configured loopback client/server index is outside the discovered NIC
    group.
  probable_causes:
    - "Test-system, station, or configuration problem covering NIC"
    - "Missing, malformed, or unreachable input required by LOOPBACK_TOPOLOGY_INDEX_INVALID"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): LoopbackTopologyTest, NetworkBwRdmaTest"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-NIC-0019
  name: RDMA_DEVICE_UNAVAILABLE
  version: 1
  category: 1  # CATEGORY_NETWORK
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.NIC.NIC
  test_cases: [LoopbackTopologyTest, NetworkBwRdmaTest]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    A required interface has no RDMA device binding or the configured group has
    no RDMA-capable interface.
  probable_causes:
    - Defective or out-of-spec NIC on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): LoopbackTopologyTest, NetworkBwRdmaTest"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-NIC-0020
  name: LOOPBACK_PAIR_CONFIGURATION_FAILED
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.NIC.NIC
  test_cases: [LoopbackTopologyTest, NetworkBwRdmaTest]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The test could not configure the loopback pair IP, MTU, namespace, neighbor,
    or interface state.
  probable_causes:
    - "Test-system, station, or configuration problem covering NIC"
    - "Missing, malformed, or unreachable input required by LOOPBACK_PAIR_CONFIGURATION_FAILED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): LoopbackTopologyTest, NetworkBwRdmaTest"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-NIC-0021
  name: ROCE_GID_UNAVAILABLE
  version: 1
  category: 1  # CATEGORY_NETWORK
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.NIC.NIC
  test_cases: [LoopbackTopologyTest, NetworkBwRdmaTest]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    A required IPv4 RoCE v2 GID could not be found for one or both interfaces in
    the pair.
  probable_causes:
    - "Test-system, station, or configuration problem covering NIC"
    - "Missing, malformed, or unreachable input required by ROCE_GID_UNAVAILABLE"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): LoopbackTopologyTest, NetworkBwRdmaTest"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-NIC-0022
  name: LOOPBACK_PROBE_FAILED
  version: 1
  category: 1  # CATEGORY_NETWORK
  severity: 2  # ERROR_ISOLATE
  quick_action: 6  # QA_RETRY
  component: L10.NIC.NIC
  test_cases: [LoopbackTopologyTest]
  disposition: FAILED
  retryable: true
  max_auto_retry: 1
  description: >
    The short RDMA loopback probe failed to establish traffic or returned a
    workload failure.
  probable_causes:
    - Defective or out-of-spec NIC on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): LoopbackTopologyTest"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Retry the affected L10 subset once and capture logs from the failing run.", retest: "affected_subset", est_time_min: 10}
    - {step: 2, who: "escalate", action: "If persistent, isolate the unit; attach logs; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-NIC-0023
  name: NO_RDMA_PAIRS_TO_TEST
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 5  # ERROR_CONFIG
  quick_action: 9  # QA_ESCALATE
  component: L10.NIC.NIC
  test_cases: [NetworkBwRdmaTest]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    No valid RDMA-capable client/server pair remained after discovery and
    topology validation.
  probable_causes:
    - "Test-system, station, or configuration problem covering NIC"
    - "Missing, malformed, or unreachable input required by NO_RDMA_PAIRS_TO_TEST"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): NetworkBwRdmaTest"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-NIC-0024
  name: RDMA_SERVER_START_FAILED
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.NIC.NIC
  test_cases: [NetworkBwRdmaTest]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The ib_write_bw server process could not be started for an RDMA pair.
  probable_causes:
    - "Test-system, station, or configuration problem covering NIC"
    - "Missing, malformed, or unreachable input required by RDMA_SERVER_START_FAILED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): NetworkBwRdmaTest"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-NIC-0025
  name: RDMA_CLIENT_EXECUTION_FAILED
  version: 1
  category: 1  # CATEGORY_NETWORK
  severity: 2  # ERROR_ISOLATE
  quick_action: 6  # QA_RETRY
  component: L10.NIC.NIC
  test_cases: [NetworkBwRdmaTest]
  disposition: FAILED
  retryable: true
  max_auto_retry: 1
  description: >
    The ib_write_bw client timed out or returned a non-zero status for an RDMA
    pair.
  probable_causes:
    - Defective or out-of-spec NIC on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): NetworkBwRdmaTest"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Retry the affected L10 subset once and capture logs from the failing run.", retest: "affected_subset", est_time_min: 10}
    - {step: 2, who: "escalate", action: "If persistent, isolate the unit; attach logs; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-NIC-0026
  name: RDMA_BANDWIDTH_OUTPUT_INVALID
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.NIC.NIC
  test_cases: [NetworkBwRdmaTest]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The RDMA bandwidth tool did not emit a parseable average-bandwidth result.
  probable_causes:
    - "Test-system, station, or configuration problem covering NIC"
    - "Missing, malformed, or unreachable input required by RDMA_BANDWIDTH_OUTPUT_INVALID"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): NetworkBwRdmaTest"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-NIC-0027
  name: RDMA_BANDWIDTH_BELOW_THRESHOLD
  version: 1
  category: 1  # CATEGORY_NETWORK
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.NIC.NIC
  test_cases: [NetworkBwRdmaTest]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    Measured RDMA bandwidth for a configured pair is below the configured
    percentage of expected line rate.
  probable_causes:
    - Defective or out-of-spec NIC on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): NetworkBwRdmaTest"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-NIC-0028
  name: NIC_RAIL_CONFIG_UNAVAILABLE
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 5  # ERROR_CONFIG
  quick_action: 9  # QA_ESCALATE
  component: L10.NIC.ASIC_RAILS
  test_cases: [CheckNicAsicRails]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The expected NIC ASIC rail limits could not be loaded from product
    configuration.
  probable_causes:
    - "Test-system, station, or configuration problem covering NIC ASIC Rails"
    - "Missing, malformed, or unreachable input required by NIC_RAIL_CONFIG_UNAVAILABLE"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): CheckNicAsicRails"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NIC_ASIC_RAILS]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-NIC-0029
  name: NIC_RAIL_READING_UNAVAILABLE
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.NIC.ASIC_RAILS
  test_cases: [CheckNicAsicRails]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    A required NIC ASIC voltage-rail reading is unavailable or reported as not
    applicable.
  probable_causes:
    - "Test-system, station, or configuration problem covering NIC ASIC Rails"
    - "Missing, malformed, or unreachable input required by NIC_RAIL_READING_UNAVAILABLE"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): CheckNicAsicRails"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NIC_ASIC_RAILS]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-NIC-0030
  name: NIC_RAIL_OUT_OF_RANGE
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.NIC.ASIC_RAILS
  test_cases: [CheckNicAsicRails]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    A NIC ASIC voltage rail is below its configured minimum or above its
    configured maximum.
  probable_causes:
    - Defective or out-of-spec NIC ASIC Rails on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckNicAsicRails"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NIC_ASIC_RAILS]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-NIC-0031
  name: NIC_FIRMWARE_UPDATE_FAILED
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.NIC.NIC
  test_cases: [BroadcomUpdateFwAndConfigure]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The Broadcom NIC firmware update command failed.
  probable_causes:
    - Defective or out-of-spec NIC on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): BroadcomUpdateFwAndConfigure"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-NIC-0032
  name: NIC_CONFIGURATION_APPLY_FAILED
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.NIC.NIC
  test_cases: [BroadcomUpdateFwAndConfigure]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    Required NIC port, driver, or firmware configuration could not be applied.
  probable_causes:
    - Defective or out-of-spec NIC on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): BroadcomUpdateFwAndConfigure"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-NIC-0033
  name: NIC_POST_UPDATE_VERIFY_FAILED
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.NIC.NIC
  test_cases: [BroadcomUpdateFwAndConfigure]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    NIC firmware or configuration did not match the requested state after
    update.
  probable_causes:
    - Defective or out-of-spec NIC on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): BroadcomUpdateFwAndConfigure"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-NIC-0034
  name: NIC_MAC_ADDRESS_UNAVAILABLE
  version: 1
  category: 11  # CATEGORY_DATA
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.NIC.NIC
  test_cases: [LoopbackTopologyTest, NetworkBwRdmaTest]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    A required interface MAC address could not be read while constructing a
    loopback/RDMA pair.
  probable_causes:
    - "Test-system, station, or configuration problem covering NIC"
    - "Missing, malformed, or unreachable input required by NIC_MAC_ADDRESS_UNAVAILABLE"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): LoopbackTopologyTest, NetworkBwRdmaTest"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-NIC-0035
  name: NIC_LINK_RECOVERY_FAILED
  version: 1
  category: 1  # CATEGORY_NETWORK
  severity: 2  # ERROR_ISOLATE
  quick_action: 6  # QA_RETRY
  component: L10.NIC.NIC
  test_cases: [LoopbackTopologyTest]
  disposition: FAILED
  retryable: true
  max_auto_retry: 1
  description: >
    A carrier-down interface did not recover after the configured link-speed/FEC
    recovery sequence.
  probable_causes:
    - Defective or out-of-spec NIC on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): LoopbackTopologyTest"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Retry the affected L10 subset once and capture logs from the failing run.", retest: "affected_subset", est_time_min: 10}
    - {step: 2, who: "escalate", action: "If persistent, isolate the unit; attach logs; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [NIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test

# L10 cable inventory (CheckCable). The check records these codes in the order
# they are listed here and a run reports the earliest one it recorded, so a
# failing run carries a single code. One bad port can still record more than
# one: a port dropped by NIC_PORT_INTERFACE_MISSING or
# CABLE_MODULE_INFO_INCOMPLETE never reaches position checking, so if the BOM
# declares a cable at its position that position also records
# CABLE_MISSING_AT_POSITION, naming the dropped port as the cause. A
# precondition failure that stops the check before any cable is reached (no SSH
# target, no cables in the BOM, netd unreachable) reports TH-HAR-0003 instead:
# nothing NIC-specific was checked.
# Allocated from 0036 in the 1X registry before CheckCable had an L10 registry
# to live in; the identities are unchanged.
- error_code: TH-NIC-0036
  name: NIC_PORT_INTERFACE_MISSING
  version: 1
  category: 1  # CATEGORY_NETWORK
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE -- retry cannot bind a driver
  component: L10.NIC.PORT
  test_cases: [CheckCable]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    netd reported a NIC port with no kernel interface name, so ethtool cannot
    read that port's module EEPROM and the cable there cannot be verified.
  probable_causes:
    - NIC function present on the bus but no netdev created (driver not bound)
    - NIC firmware or driver failed to bring the port up
    - Port renamed or removed on the host outside the harness
  capture:
    - suite_run_url
    - netd GetNics response for the affected port (bdf, position)
    - ip link and lspci output for the NIC
    - unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp
  repair_actions:
    - {step: 1, who: operator, action: "Collect the suite/test logs and artifacts for this failure.", est_time_min: 5}
    - {step: 2, who: escalate, action: "Escalate to Etched engineer with the collected logs.", ticket_template: ETCH}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.224
  suspected_components: [NIC, NIC_DRIVER]
  vendor_error_id: []
  jira_component: ETCH / l10-test

# Both halves of this group leave the check without the data it needs to judge a
# module, so they share one code: an unusable position identifier from netd, and
# an EEPROM that omits a required identity field.
- error_code: TH-NIC-0037
  name: CABLE_MODULE_INFO_INCOMPLETE
  version: 1
  category: 11  # CATEGORY_DATA
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.NIC.CABLE
  test_cases: [CheckCable]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    A pluggable module was read but cannot be checked: netd gave its port no
    position identifier or one outside the known layout, or the module did not
    report every identity field the check requires.
  probable_causes:
    - Corrupt or non-compliant module EEPROM omitting Vendor PN/SN or Identifier
    - Module type whose EEPROM layout ethtool cannot fully decode
    - netd port-position mapping missing or stale for this server layout
  capture:
    - suite_run_url
    - full ethtool --module-info output for the affected interface
    - netd GetNics response for the affected port (bdf, position)
    - unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp
  repair_actions:
    - {step: 1, who: operator, action: "Collect the suite/test logs and artifacts for this failure.", est_time_min: 5}
    - {step: 2, who: escalate, action: "Escalate to Etched engineer with the collected logs.", ticket_template: ETCH}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.224
  suspected_components: [CABLE, NIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-NIC-0038
  name: CABLE_MISSING_AT_POSITION
  version: 1
  category: 1  # CATEGORY_NETWORK
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE -- an absent cable needs a human, not a retry
  component: L10.NIC.CABLE
  test_cases: [CheckCable]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The BOM declares a cable at a port position where none was found: no NIC
    reported that position, the port was pruned earlier, or the module read
    failed.
  probable_causes:
    - Cable not installed, or installed in the wrong cage
    - Cable or module not fully seated, so the cage reads as empty
    - Failed module EEPROM read on an otherwise populated cage
    - BOM declares a position the server build does not populate
  capture:
    - suite_run_url
    - the check's per-position summary naming the cause for this position
    - ethtool --module-info output or error for the affected interface
    - unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp
  repair_actions:
    - {step: 1, who: operator, action: "Verify the cable is installed and fully seated at the reported port position per the cabling SOP, then retest.", retest: affected_subset, est_time_min: 10}
    - {step: 2, who: operator, action: "Collect the suite/test logs and artifacts for this failure.", est_time_min: 5}
    - {step: 3, who: escalate, action: "Escalate to Etched engineer with the collected logs.", ticket_template: ETCH}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.224
  suspected_components: [CABLE, NIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test

# Two ports cannot occupy one physical position, so this is a station/config
# fault rather than a server verdict: disposition ERROR keeps it out of yield.
- error_code: TH-NIC-0039
  name: NIC_PORT_POSITION_AMBIGUOUS
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.NIC.PORT
  test_cases: [CheckCable]
  disposition: ERROR  # test-system problem; excluded from server yield
  retryable: false
  max_auto_retry: 0
  description: >
    More than one module reported the same port position, so the check cannot
    tell which one is physically there and prunes all of them rather than
    guessing.
  probable_causes:
    - netd port-position mapping assigns one position to several BDFs
    - Server layout changed without a matching netd configuration update
    - netd reported the same port twice in one GetNics response
  capture:
    - suite_run_url
    - full netd GetNics response (all bdf/position pairs)
    - the netd port-position configuration in use on the host
    - station_id, harness_version, operator_id, timestamp
  repair_actions:
    - {step: 1, who: operator, action: "Collect the suite/test logs and artifacts for this failure.", est_time_min: 5}
    - {step: 2, who: escalate, action: "Escalate to Etched engineer with the collected logs.", ticket_template: ETCH}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.224
  suspected_components: []
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-NIC-0040
  name: CABLE_IDENTITY_MISMATCH
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE -- the wrong part must be swapped
  component: L10.NIC.CABLE
  test_cases: [CheckCable]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    A module's identity fields do not match the BOM entry for its port
    position: Vendor PN (including the entry's alternate part numbers), Vendor
    name, Identifier, Transmitter technology, or a shopfloor-declared Vendor SN.
  probable_causes:
    - Wrong cable part or length installed at this position
    - Cables swapped between positions during assembly
    - Substitute part not yet listed in the BOM entry's alternate_part_numbers
    - Serial number on the installed cable differs from the shopfloor record
  capture:
    - suite_run_url
    - the check's per-field expected vs. observed values for this position
    - full ethtool --module-info output for the affected interface
    - the BOM cable entry for this position
    - unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp
  repair_actions:
    - {step: 1, who: operator, action: "Compare the installed cable against the BOM part for this position and install the correct part, then retest.", retest: affected_subset, est_time_min: 15}
    - {step: 2, who: operator, action: "Collect the suite/test logs and artifacts for this failure.", est_time_min: 5}
    - {step: 3, who: escalate, action: "Escalate to Etched engineer with the collected logs.", ticket_template: ETCH}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.224
  suspected_components: [CABLE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-NIC-0041
  name: CABLE_UNEXPECTED_AT_POSITION
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.NIC.CABLE
  test_cases: [CheckCable]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    A module was found at a port position where the BOM declares no cable.
  probable_causes:
    - Cable installed in a position the build does not use
    - Cable intended for another position plugged here
    - BOM missing an entry for a position this build populates
  capture:
    - suite_run_url
    - the position and interface of the unexpected module
    - full ethtool --module-info output for the affected interface
    - the BOM cable entries for this server
    - unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp
  repair_actions:
    - {step: 1, who: operator, action: "Remove or relocate the cable at the reported position per the cabling SOP, then retest.", retest: affected_subset, est_time_min: 10}
    - {step: 2, who: operator, action: "Collect the suite/test logs and artifacts for this failure.", est_time_min: 5}
    - {step: 3, who: escalate, action: "Escalate to Etched engineer with the collected logs.", ticket_template: ETCH}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.224
  suspected_components: [CABLE]
- error_code: TH-PCIE-0001
  name: PCIE_LINK_STATE_MISMATCH
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 2  # ERROR_ISOLATE
  quick_action: 6  # QA_RETRY
  component: L10.PCIE.HOST_PCIE
  test_cases: [CheckPcieTopology]
  disposition: FAILED
  retryable: true
  max_auto_retry: 1
  description: >
    PCIe link state mismatch (not L0 / Gen5 / x16)
  probable_causes:
    - PCIe link degraded or retrained at a lower speed/width
    - Marginal PCIe channel or connector seating
  capture:
    - suite_run_url
    - per-test log and transcript
    - "unit_sn, fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Rerun the PCIe subset up to 3 times; capture lspci and AER state before retry.", retest: "affected_subset", est_time_min: 10}
    - {step: 2, who: "escalate", action: "Isolate the module after retry exhaustion; escalate to Etched with the captured evidence.", retest: "n/a", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: []
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-PCIE-0003
  name: PCIE_CSRAM_DMA_THROUGHPUT_BELOW_THRESHOLD
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.PCIE.HOST_PCIE
  test_cases: [SohuDmaMultiChipTestCase]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    PCIe->CSRAM DMA throughput below threshold (50GB/s)
  probable_causes:
    - PCIe link degraded (lane width / speed below Gen5 x16)
    - Marginal PCIe channel forcing correctable-error retries
    - DMA engine or arbitration issue limiting transfer rate
  capture:
    - suite_run_url
    - measured read/write throughput vs configured floor
    - lspci link state and AER counters
    - "unit_sn, fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Rerun the PCIe/DMA subset up to 3 times; reseat the module if failures persist.", retest: "PCIE_subset", est_time_min: 10}
    - {step: 2, who: "escalate", action: "Capture lspci + AER state; isolate the module and route to triage.", retest: "n/a", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [SOHU_ASIC, PCIE_LINK]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-PCIE-0020
  name: PCIE_HBM_DMA_DATA_MISMATCH
  version: 1
  category: 11  # CATEGORY_DATA
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.PCIE.HOST_PCIE
  test_cases: [SohuDmaMultiChipTestCase]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    PCIe->HBM DMA data integrity mismatch
  probable_causes:
    - HBM stack or pseudo-channel data fault
    - Marginal PCIe or on-chip path corrupting transfers to HBM
    - DMA addressing fault targeting the wrong block/offset
  capture:
    - suite_run_url
    - "full dma_test_app stdout/stderr, exit code, and RNG seed"
    - failing region/blocks/offset from the verification context
    - "unit_sn, fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Rerun the DMA subset up to 3 times with the reported seed.", retest: "DMA_subset", est_time_min: 10}
    - {step: 2, who: "escalate", action: "Preserve unit; attach output and failing block map; isolate the module and route to triage.", retest: "n/a", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [SOHU_ASIC, HBM_STACK]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-PCIE-0005
  name: PCIE_HBM_DMA_THROUGHPUT_BELOW_THRESHOLD
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.PCIE.HOST_PCIE
  test_cases: [SohuDmaMultiChipTestCase]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    PCIe->HBM DMA throughput below threshold (16GB/s)
  probable_causes:
    - PCIe link degraded (lane width / speed below Gen5 x16)
    - HBM stack or controller limiting sustained bandwidth
    - DMA engine or arbitration issue limiting transfer rate
  capture:
    - suite_run_url
    - measured per-region throughput vs configured floor
    - "full dma_test_app stdout/stderr, exit code, and RNG seed"
    - "unit_sn, fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Rerun the DMA subset up to 3 times; reseat the module if failures persist.", retest: "DMA_subset", est_time_min: 10}
    - {step: 2, who: "escalate", action: "Capture lspci link state and throughput numbers; isolate the module and route to triage.", retest: "n/a", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [SOHU_ASIC, PCIE_LINK, HBM_STACK]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-PCIE-0008
  name: SOHU_ENDPOINT_NOT_ENUMERATED
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 2  # ERROR_ISOLATE
  quick_action: 6  # QA_RETRY
  component: L10.PCIE.HOST_PCIE
  test_cases: [CheckPcieTopology, SohuVfioPingMultiChipTestCase]
  disposition: FAILED
  retryable: true
  max_auto_retry: 1
  description: >
    Sohu endpoint not enumerated (absent from PCIe tree / RPC proxy)
  probable_causes:
    - Sohu endpoint is absent from the PCIe tree or RPC proxy
  capture:
    - suite_run_url
    - per-test log and transcript
    - PCIe topology and RPC proxy state
    - "harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "escalate", action: "Escalate to Etched.", retest: "affected_subset"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [PCIE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-PCIE-0009
  name: PCIE_EXPECTED_TOPOLOGY_UNAVAILABLE
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 5  # ERROR_CONFIG
  quick_action: 9  # QA_ESCALATE
  component: L10.PCIE.HOST_PCIE
  test_cases: [CheckPcieTopology]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The expected PCIe topology or device specification could not be loaded; no
    PCIe DUT verdict is available.
  probable_causes:
    - "Test-system, station, or configuration problem covering PCIe"
    - "Missing, malformed, or unreachable input required by PCIE_EXPECTED_TOPOLOGY_UNAVAILABLE"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): CheckPcieTopology"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [PCIE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-PCIE-0010
  name: PCIE_INVENTORY_QUERY_FAILED
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.PCIE.HOST_PCIE
  test_cases: [CheckPcieTopology]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The PCIe inventory command failed, so the test could not enumerate devices
    or links.
  probable_causes:
    - "Test-system, station, or configuration problem covering PCIe"
    - "Missing, malformed, or unreachable input required by PCIE_INVENTORY_QUERY_FAILED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): CheckPcieTopology"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [PCIE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-PCIE-0011
  name: PCIE_INVENTORY_OUTPUT_INVALID
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.PCIE.HOST_PCIE
  test_cases: [CheckPcieTopology]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The PCIe inventory output was empty, incomplete, or could not be parsed into
    a reliable topology.
  probable_causes:
    - "Test-system, station, or configuration problem covering PCIe"
    - "Missing, malformed, or unreachable input required by PCIE_INVENTORY_OUTPUT_INVALID"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): CheckPcieTopology"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [PCIE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-PCIE-0012
  name: PCIE_TOPOLOGY_SPEC_INVALID
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 5  # ERROR_CONFIG
  quick_action: 9  # QA_ESCALATE
  component: L10.PCIE.HOST_PCIE
  test_cases: [CheckPcieTopology]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The configured PCIe topology contains invalid or conflicting device
    identifiers, counts, or topology indices.
  probable_causes:
    - "Test-system, station, or configuration problem covering PCIe"
    - "Missing, malformed, or unreachable input required by PCIE_TOPOLOGY_SPEC_INVALID"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): CheckPcieTopology"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [PCIE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-PCIE-0013
  name: PCIE_TOPOLOGY_MISMATCH
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 2  # ERROR_ISOLATE
  quick_action: 6  # QA_RETRY
  component: L10.PCIE.HOST_PCIE
  test_cases: [CheckPcieTopology]
  disposition: FAILED
  retryable: true
  max_auto_retry: 1
  description: >
    The discovered PCIe device count or parent/child topology does not match the
    expected server topology.
  probable_causes:
    - Defective or out-of-spec PCIe on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckPcieTopology"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Retry the affected L10 subset once and capture logs from the failing run.", retest: "affected_subset", est_time_min: 10}
    - {step: 2, who: "escalate", action: "If persistent, isolate the unit; attach logs; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [PCIE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-PCIE-0014
  name: PCIE_DEVICE_IDENTITY_MISMATCH
  version: 1
  category: 11  # CATEGORY_DATA
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.PCIE.HOST_PCIE
  test_cases: [CheckPcieTopology]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    A discovered PCIe device has an unexpected vendor ID, device ID, subsystem
    ID, or configured device name.
  probable_causes:
    - Defective or out-of-spec PCIe on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckPcieTopology"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [PCIE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-PCIE-0015
  name: PCIE_DUPLICATE_BDF
  version: 1
  category: 11  # CATEGORY_DATA
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.PCIE.HOST_PCIE
  test_cases: [CheckPcieTopology]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    Two expected or discovered PCIe devices resolve to the same
    bus/device/function address.
  probable_causes:
    - Defective or out-of-spec PCIe on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckPcieTopology"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [PCIE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-PCIE-0016
  name: PLX_DAEMON_VERSION_QUERY_FAILED
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.PCIE.PLX_SWITCH
  test_cases: [CheckPlxDaemonFirmwareVersion]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The PLX daemon firmware/version query failed or returned no usable switch
    information.
  probable_causes:
    - "Test-system, station, or configuration problem covering PLX switch"
    - "Missing, malformed, or unreachable input required by PLX_DAEMON_VERSION_QUERY_FAILED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): CheckPlxDaemonFirmwareVersion"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [PLX_SWITCH]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-PCIE-0017
  name: PLX_DAEMON_FIRMWARE_VERSION_MISMATCH
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 5  # ERROR_CONFIG
  quick_action: 9  # QA_ESCALATE
  component: L10.PCIE.PLX_SWITCH
  test_cases: [CheckPlxDaemonFirmwareVersion]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The PLX daemon or switch firmware version does not match the expected L10
    release.
  probable_causes:
    - Defective or out-of-spec PLX switch on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckPlxDaemonFirmwareVersion"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [PLX_SWITCH]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-PCIE-0018
  name: PLX_DAEMON_FIRMWARE_UPDATE_FAILED
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.PCIE.PLX_SWITCH
  test_cases: [UpdatePlxDaemonFirmwareVersion]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The PLX daemon firmware update or activation command failed.
  probable_causes:
    - Defective or out-of-spec PLX switch on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): UpdatePlxDaemonFirmwareVersion"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [PLX_SWITCH]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-PCIE-0019
  name: PLX_DAEMON_POST_UPDATE_VERIFY_FAILED
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.PCIE.PLX_SWITCH
  test_cases: [UpdatePlxDaemonFirmwareVersion]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    PLX daemon firmware did not report the requested version after the update
    completed.
  probable_causes:
    - Defective or out-of-spec PLX switch on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): UpdatePlxDaemonFirmwareVersion"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [PLX_SWITCH]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-PWR-0007
  name: UNEXPECTED_DVFS_RAMPDOWN
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.PWR.POWER
  test_cases: [SohuPowerVirusMultiChipTestCase]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    Unexpected DVFS rampdown event during workload
  probable_causes:
    - Power or thermal excursion triggering DVFS
    - Marginal rail under workload transients
  capture:
    - suite_run_url
    - per-test log and transcript
    - "unit_sn, fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture DVFS/power telemetry; rerun up to 3 times, then isolate.", retest: "affected_subset", est_time_min: 10}
    - {step: 2, who: "escalate", action: "Isolate the module after retry exhaustion; escalate to Etched with the captured evidence.", retest: "n/a", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: []
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-PWR-0009
  name: BMC_POWER_CYCLE_COMMAND_FAILED
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.PWR.POWER
  test_cases: [BmcPowerCycle]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The BMC power-cycle command returned a failure.
  probable_causes:
    - "Test-system, station, or configuration problem covering Power"
    - "Missing, malformed, or unreachable input required by BMC_POWER_CYCLE_COMMAND_FAILED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): BmcPowerCycle"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [POWER]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-PWR-0010
  name: HOST_DID_NOT_POWER_OFF
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.PWR.POWER
  test_cases: [BmcPowerCycle]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The host remained reachable after the BMC power-off step and did not enter
    the expected off state.
  probable_causes:
    - Defective or out-of-spec Power on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): BmcPowerCycle"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [POWER]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-PWR-0011
  name: HOST_DID_NOT_RECOVER_AFTER_POWER_CYCLE
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.PWR.POWER
  test_cases: [BmcPowerCycle]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The host did not become reachable within the configured interval after BMC
    power-on.
  probable_causes:
    - Defective or out-of-spec Power on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): BmcPowerCycle"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [POWER]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-PWR-0012
  name: HOST_REBOOT_COMMAND_FAILED
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.PWR.POWER
  test_cases: [SohuHostRebootTestCase]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    The host reboot request failed before the reboot transition could be
    observed.
  probable_causes:
    - "Test-system, station, or configuration problem covering Power"
    - "Missing, malformed, or unreachable input required by HOST_REBOOT_COMMAND_FAILED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): SohuHostRebootTestCase"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [POWER]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-PWR-0013
  name: HOST_REBOOT_NOT_OBSERVED
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.PWR.POWER
  test_cases: [SohuHostRebootTestCase]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The host never became unreachable after the reboot request, so the expected
    reboot transition was not observed.
  probable_causes:
    - Defective or out-of-spec Power on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): SohuHostRebootTestCase"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [POWER]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-PWR-0014
  name: HOST_DID_NOT_RECOVER_AFTER_REBOOT
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.PWR.POWER
  test_cases: [SohuHostRebootTestCase]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The host did not return to a reachable and usable state within the
    configured reboot recovery interval.
  probable_causes:
    - Defective or out-of-spec Power on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): SohuHostRebootTestCase"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [POWER]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-SOHU-0002
  name: SOHU_NOT_PINGABLE
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.SOHU.SOHU_CHIP
  test_cases: [SohuPingTestCase, SohuVfioPingMultiChipTestCase, SohuFlrMultiChipTestCase]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    Sohu not pingable / no response from firmware
  probable_causes:
    - Sohu firmware hung or crashed and stopped servicing RPCs
    - Device dropped off the PCIe tree / vfio proxy
    - Marginal reset or power sequencing leaving the ASIC unresponsive
  capture:
    - suite_run_url
    - per-test log naming the unresponsive device index
    - UART log for the run window
    - lspci / proxy device list at the time of the failure
    - "unit_sn, fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "escalate", action: "Escalate to Etched with the unresponsive device index and the UART log.", retest: "n/a", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [ASIC, MEZZ_MODULE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-SOHU-0029
  name: SOHU_SRAM_TEST_FAILED
  version: 1
  category: 3  # CATEGORY_MEMORY
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.SOHU.SOHU_CHIP
  test_cases: [SohuSramMemoryMultiChipTestCase]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    On-chip SRAM test failure (stuck bits / readback mismatch)
  probable_causes:
    - Defective on-die SRAM cells (stuck bits) in ISC/NLU/theta/softmax SRAMs
    - Marginal SRAM timing under nominal voltage/frequency
  capture:
    - suite_run_url
    - failing SRAM names (affected_instances)
    - March C+ binary stdout/stderr and exit code
    - "unit_sn, fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Rerun the SRAM subset once (SRAM names in affected_instances).", retest: "affected_subset", est_time_min: 10}
    - {step: 2, who: "escalate", action: "Isolate module; attach failing SRAM list and March C+ log; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [SOHU_CHIP]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-SOHU-0030
  name: SOHU_WORKLOAD_TIMEOUT
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.SOHU.SOHU_CHIP
  test_cases: [SohuMlpTp8TestCase, SohuLlama70bForwardIteratedTestCase, SohuLlama70bHp8ForwardIteratedTestCase, SohuLlamaHp2Up4CachedInferenceMaxTestCase, SohuLlamaHp2Up4NospecInferenceMaxTestCase, SohuLlamaHp8CachedInferenceMaxTestCase, Llama8bFp8Tp1SmokeTestCase, SohuPowerVirusMultiChipTestCase]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    Workload hang / operation timeout
  probable_causes:
    - SOHU workload or operation did not complete within its timeout
  capture:
    - suite_run_url
    - per-test log and transcript
    - device state captured before recovery
    - "unit_sn, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture state before recovery and retry up to three times.", retest: "affected_subset"}
    - {step: 2, who: "operator", action: "Isolate the module after retry exhaustion.", retest: "n/a"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [SOHU]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-SOHU-0031
  name: SOHU_WEIGHT_LOAD_MISMATCH
  version: 1
  category: 11  # CATEGORY_DATA
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.SOHU.SOHU_CHIP
  test_cases: [SohuLlama70bForwardIteratedTestCase, SohuLlama70bHp8ForwardIteratedTestCase, SohuLlamaHp2Up4CachedInferenceMaxTestCase, SohuLlamaHp2Up4NospecInferenceMaxTestCase, SohuLlamaHp8CachedInferenceMaxTestCase, Llama8bFp8Tp1SmokeTestCase]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    Weight load spot-check mismatch
  probable_causes:
    - HBM write/read path corrupting weights
    - DMA addressing fault
  capture:
    - suite_run_url
    - per-test log and transcript
    - "unit_sn, fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Rerun up to 3 times; isolate (subset offsets in raw_data).", retest: "affected_subset", est_time_min: 10}
    - {step: 2, who: "escalate", action: "Isolate the module after retry exhaustion; escalate to Etched with the captured evidence.", retest: "n/a", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: []
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-SOHU-0032
  name: SOHU_INFERENCE_THROUGHPUT_BELOW_THRESHOLD
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.SOHU.SOHU_CHIP
  test_cases: [SohuLlamaHp2Up4CachedInferenceMaxTestCase, SohuLlamaHp2Up4NospecInferenceMaxTestCase, SohuLlamaHp8CachedInferenceMaxTestCase]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    Sustained inference throughput on the million-token test did not reach 95%
    of the suite's expected_chip_tokens_per_second.
  probable_causes:
    - Chip sustaining a lower clock than expected under continuous load
    - Thermal or power throttling during the run
    - Marginal SA/HBM path reducing effective inference rate
  capture:
    - suite_run_url
    - full llama_mtt_app stdout/stderr with the measured tokens/s
    - configured expected_chip_tokens_per_second and the 0.95 lower bound
    - DVFS and thermal telemetry captured by the monitor during the run
    - "unit_sn, fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "escalate", action: "Escalate to Etched with the measured tokens/s, the configured expectation, and the DVFS/thermal telemetry.", retest: "n/a", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [ASIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-SOHU-0033
  name: SOHU_INFERENCE_NUMERIC_MISMATCH
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.SOHU.SOHU_CHIP
  test_cases: [SohuMlpTp8TestCase, SohuLlama70bForwardIteratedTestCase, SohuLlama70bHp8ForwardIteratedTestCase, SohuLlamaHp2Up4CachedInferenceMaxTestCase, SohuLlamaHp2Up4NospecInferenceMaxTestCase, SohuLlamaHp8CachedInferenceMaxTestCase, Llama8bFp8Tp1SmokeTestCase]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    End-to-end inference numeric mismatch vs reference
  probable_causes:
    - SOHU inference output did not match the expected reference
  capture:
    - suite_run_url
    - per-test log and transcript
    - numeric comparison output
    - "unit_sn, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Retry up to three times.", retest: "affected_subset"}
    - {step: 2, who: "operator", action: "Isolate the module after retry exhaustion.", retest: "n/a"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [SOHU]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-SOHU-0034
  name: SOHU_FLR_FAILED
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.SOHU.SOHU_CHIP
  test_cases: [SohuFlrMultiChipTestCase]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    FLR failed / flr_timeout_s 5.0s / recovery_timeout_s 10.0s / not pingable
    between FLRs
  probable_causes:
    - Sohu firmware not re-initialising cleanly after FLR
    - PCIe link failing to retrain within the recovery window
    - Marginal reset or power sequencing on the module
  capture:
    - suite_run_url
    - per-test log with the failing iteration index and measured FLR durations
    - PCIe link state and lspci output after the failing reset
    - "unit_sn, fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "escalate", action: "Escalate to Etched with the failing iteration index, measured FLR durations, and post-reset link state.", retest: "n/a", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [ASIC, MEZZ_MODULE]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-SOHU-0035
  name: SOHU_SNAPSHOT_CAPTURE_FAILED
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.SOHU.SOHU_CHIP
  test_cases: [SohuSnapTestCase]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    Chip state snapshot capture failed
  probable_causes:
    - Chip not responding to the snapshot request
    - Snapshot tooling fault
  capture:
    - suite_run_url
    - per-test log and transcript
    - "unit_sn, fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Rerun up to 3 times.", retest: "affected_subset", est_time_min: 10}
    - {step: 2, who: "escalate", action: "Escalate to Etched with the snapshot log.", retest: "n/a", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [SOHU_ASIC]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-SOHU-0036
  name: SOHU_VFIO_PING_LATENCY_EXCEEDED
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.SOHU.SOHU_CHIP
  test_cases: [SohuVfioPingMultiChipTestCase]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    VFIO ping responses received but none within the allowed latency (device
    pingable, latency out of spec)
  probable_causes:
    - Defective or out-of-spec SOHU on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): SohuVfioPingMultiChipTestCase"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [SOHU]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-SOHU-0028
  name: SOHU_WORKLOAD_BINARY_FAILED
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.SOHU.SOHU_CHIP
  test_cases: [SohuMlpTp8TestCase, SohuLlama70bForwardIteratedTestCase, SohuLlama70bHp8ForwardIteratedTestCase, SohuLlamaHp2Up4CachedInferenceMaxTestCase, SohuLlamaHp2Up4NospecInferenceMaxTestCase, SohuLlamaHp8CachedInferenceMaxTestCase, Llama8bFp8Tp1SmokeTestCase]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    MTT binary exited non-zero for a reason the harness cannot determine
  probable_causes:
    - Defective or out-of-spec SOHU on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): SohuMlpTp8TestCase, SohuLlama70bForwardIteratedTestCase, SohuLlama70bHp8ForwardIteratedTestCase, SohuLlamaHp2Up4CachedInferenceMaxTestCase, SohuLlamaHp2Up4NospecInferenceMaxTestCase, SohuLlamaHp8CachedInferenceMaxTestCase, Llama8bFp8Tp1SmokeTestCase"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [SOHU]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-THM-0012
  name: THERMAL_DIODE_READING_INVALID
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 2  # ERROR_ISOLATE
  quick_action: 6  # QA_RETRY
  component: L10.THM.THERMAL
  test_cases: [SohuDtsMultiChipTestCase]
  disposition: FAILED
  retryable: true
  max_auto_retry: 1
  description: >
    Thermal diode temperature reading out of the qualified range, reported
    invalid by the sensor, or an uncalibrated mezzanine diode reading below 0 C.
  probable_causes:
    - Thermal forcing head not seated / poor contact
    - Bad solder joint on an uncalibrated mezzanine diode channel
    - Faulty TMP464 sensor or ASIC remote diode
    - Station ambient out of spec
  capture:
    - suite_run_url
    - per-channel temperatures and validity flags (recorded measurements)
    - configured min/max bounds
    - "unit_sn, fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Verify thermal head contact and station ambient; rerun the thermal subset once (channel in affected_instances).", retest: "affected_subset", est_time_min: 5}
    - {step: 2, who: "technician", action: "Inspect mezzanine diode solder joints if an uncalibrated channel reads below 0 C.", retest: "affected_subset", est_time_min: 15}
    - {step: 3, who: "escalate", action: "Isolate module; attach per-channel readings; route to FA; open Jira.", retest: "n/a", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [MEZZ_MODULE, THERMAL_HEAD, SOHU_CHIP]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-THM-0013
  name: DTS_SPREAD_EXCEEDED
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 2  # ERROR_ISOLATE
  quick_action: 6  # QA_RETRY
  component: L10.THM.THERMAL
  test_cases: [SohuDtsMultiChipTestCase]
  disposition: FAILED
  retryable: true
  max_auto_retry: 1
  description: >
    The peak-to-peak spread across usable Sohu DTS readings exceeded the
    configured deviation limit.
  probable_causes:
    - Uneven thermal forcing head seating across the die
    - TIM voids or uneven bond line
    - Defective DTS sensor reading high or low while still reporting no fault
  capture:
    - suite_run_url
    - per-sensor converted readings and the computed peak_to_peak value
    - configured deviation limit for the run
    - "unit_sn, fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Verify thermal forcing head seating, rerun the DTS test up to 3 times.", retest: "sensors_subset", est_time_min: 10}
    - {step: 2, who: "technician", action: "Reseat the thermal forcing head and reapply TIM per assembly SOP.", retest: "sensors_subset", est_time_min: 20}
    - {step: 3, who: "escalate", action: "Isolate the module after retry exhaustion; escalate to Etched with the per-sensor spread.", retest: "n/a", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [SOHU_ASIC, THERMAL_HEAD, TIM]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-THM-0014
  name: CATASTROPHIC_THERMAL_TRIP
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 2  # ERROR_ISOLATE
  quick_action: 6  # QA_RETRY
  component: L10.THM.THERMAL
  test_cases: [SohuPowerVirusMultiChipTestCase]
  disposition: FAILED
  retryable: true
  max_auto_retry: 1
  description: >
    Catastrophic trip event during stress workload
  probable_causes:
    - Thermal excursion under power virus
    - TIM or cold-plate contact degradation
  capture:
    - suite_run_url
    - per-test log and transcript
    - "unit_sn, fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture thermal telemetry before retry; isolate if repeated.", retest: "affected_subset", est_time_min: 10}
    - {step: 2, who: "escalate", action: "Isolate the module after retry exhaustion; escalate to Etched with the captured evidence.", retest: "n/a", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: []
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-THM-0015
  name: DTS_DEVIATION_UNDER_STRESS
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.THM.THERMAL
  test_cases: [SohuPowerVirusMultiChipTestCase]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    DTS peak-to-peak deviation under power virus out of bounds
  probable_causes:
    - TIM or thermal head contact variation
    - Threshold still under review
  capture:
    - suite_run_url
    - per-test log and transcript
    - "unit_sn, fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Check TIM / thermal head seating; rerun; triage against the tentative bounds.", retest: "affected_subset", est_time_min: 10}
    - {step: 2, who: "escalate", action: "Isolate the module after retry exhaustion; escalate to Etched with the captured evidence.", retest: "n/a", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: []
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-THM-0016
  name: PEAK_DTS_TEMPERATURE_EXCEEDED
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.THM.THERMAL
  test_cases: [SohuPowerVirusMultiChipTestCase]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    Peak DTS temperature under power virus exceeds expected maximum
  probable_causes:
    - Unit did not complete the required 45-minute TIM baking process
    - TIM or cold-plate contact degradation
  capture:
    - suite_run_url
    - thermal telemetry for the run
    - "unit_sn, fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Verify the unit completed the required 45-minute TIM baking process (updated from 15 minutes).", retest: "affected_subset", est_time_min: 10}
    - {step: 2, who: "escalate", action: "If baked and still out of spec, escalate to Etched for FA.", retest: "n/a", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [SOHU_ASIC, TIM]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-THM-0008
  name: FAN_LOCATION_MISMATCH
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.THM.FAN
  test_cases: [BmcCheckFanLocation]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    Fan location identifiers read back from the BMC do not match the configured
    physical fan positions.
  probable_causes:
    - Defective or out-of-spec Fan on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): BmcCheckFanLocation"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [FAN]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-THM-0009
  name: FAN_PWM_SET_FAILED
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.THM.FAN
  test_cases: [BmcCheckFanControl]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The requested fan PWM value could not be applied through the BMC control
    interface.
  probable_causes:
    - Defective or out-of-spec Fan on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): BmcCheckFanControl"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [FAN]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-THM-0010
  name: FAN_RPM_RESPONSE_OUT_OF_RANGE
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.THM.FAN
  test_cases: [BmcCheckFanControl]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    A fan RPM reading is outside the expected range or does not increase as
    required after a PWM change.
  probable_causes:
    - Defective or out-of-spec Fan on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): BmcCheckFanControl"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [FAN]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-THM-0011
  name: FAN_AUTOMATIC_MODE_RESTORE_FAILED
  version: 1
  category: 6  # CATEGORY_HARDWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.THM.FAN
  test_cases: [BmcCheckFanControl]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    Fan control could not be returned to automatic mode after the manual fan
    test.
  probable_causes:
    - "Test-system, station, or configuration problem covering Fan"
    - "Missing, malformed, or unreachable input required by FAN_AUTOMATIC_MODE_RESTORE_FAILED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): BmcCheckFanControl"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [FAN]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-VBB-0001
  name: BASEBOARD_INVENTORY_QUERY_FAILED
  version: 1
  category: 7  # CATEGORY_SOFTWARE
  severity: 4  # ERROR_TRIAGE
  quick_action: 9  # QA_ESCALATE
  component: L10.VBB.BASEBOARD
  test_cases: [CheckBaseboard, CheckBaseboardVersion]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    Baseboard/VBB inventory or version information could not be queried.
  probable_causes:
    - "Test-system, station, or configuration problem covering VBB"
    - "Missing, malformed, or unreachable input required by BASEBOARD_INVENTORY_QUERY_FAILED"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): CheckBaseboard, CheckBaseboardVersion"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [VBB]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-VBB-0002
  name: BASEBOARD_IDENTITY_MISMATCH
  version: 1
  category: 11  # CATEGORY_DATA
  severity: 2  # ERROR_ISOLATE
  quick_action: 9  # QA_ESCALATE
  component: L10.VBB.BASEBOARD
  test_cases: [CheckBaseboard]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    Baseboard manufacturer, product, serial, or board identity does not match
    the expected BOM.
  probable_causes:
    - Defective or out-of-spec VBB on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckBaseboard"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [VBB]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-VBB-0003
  name: BASEBOARD_VERSION_MISMATCH
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 5  # ERROR_CONFIG
  quick_action: 9  # QA_ESCALATE
  component: L10.VBB.BASEBOARD
  test_cases: [CheckBaseboardVersion]
  disposition: FAILED
  retryable: false
  max_auto_retry: 0
  description: >
    The baseboard/VBB hardware or firmware version does not match the expected
    release.
  probable_causes:
    - Defective or out-of-spec VBB on the L10 unit
    - "Firmware, BIOS, or configuration state not at the required L10 release"
    - "Station fixture, cable, or environmental issue reproducing as a DUT fail"
  capture:
    - suite_run_url
    - "failing test case(s): CheckBaseboardVersion"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "operator", action: "Capture logs and isolate the unit; do not continue the L10 suite on this code.", retest: "n/a", est_time_min: 5}
    - {step: 2, who: "escalate", action: "Attach the per-test log and suite URL; route to FA; open Jira.", retest: "n/a", stop_condition: "No further rework before root cause confirmed.", ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [VBB]
  vendor_error_id: []
  jira_component: ETCH / l10-test
- error_code: TH-VBB-0004
  name: BASEBOARD_EXPECTED_CONFIG_UNAVAILABLE
  version: 1
  category: 5  # CATEGORY_CONFIG
  severity: 5  # ERROR_CONFIG
  quick_action: 9  # QA_ESCALATE
  component: L10.VBB.BASEBOARD
  test_cases: [CheckBaseboard, CheckBaseboardVersion]
  disposition: ERROR
  retryable: false
  max_auto_retry: 0
  description: >
    Expected baseboard/VBB BOM or version data could not be loaded; no DUT
    verdict is available.
  probable_causes:
    - "Test-system, station, or configuration problem covering VBB"
    - "Missing, malformed, or unreachable input required by BASEBOARD_EXPECTED_CONFIG_UNAVAILABLE"
    - Suite or image provisioning gap rather than a DUT hardware fault
  capture:
    - suite_run_url
    - "failing test case(s): CheckBaseboard, CheckBaseboardVersion"
    - per-test log and transcript
    - "unit_sn, bios/fw_version, harness_version, station_id, operator_id, timestamp"
  repair_actions:
    - {step: 1, who: "engineer", action: "Fix the station, image, or suite configuration; do NOT route the unit to rework.", retest: "affected_subset", est_time_min: 15, ticket_template: "ETCH"}
  owner: supercomputing-sw
  doc_url: https://docs.google.com/document/d/1-_YC_cXvi3FtKd51jIetY9pER8JtCxH5dBE9UvndZWU/edit
  since: harness 2026.226
  suspected_components: [VBB]
  vendor_error_id: []
  jira_component: ETCH / l10-test