Agent Regression Labdemo · mock LLM

Refund flow

seed-demodemo

Customer support agent handling a refund request

Inputs

{
  "user": "I want a refund for order o-1.",
  "cannedResponses": [
    {
      "match": "want a refund",
      "response": {
        "model": "mock",
        "content": "",
        "toolCalls": [
          {
            "input": {
              "orderId": "o-1"
            },
            "callId": "c-1",
            "toolName": "lookup_order"
          }
        ],
        "stopReason": "tool_use"
      }
    },
    {
      "match": "TOOL_RESULT[c-1]",
      "response": {
        "model": "mock",
        "content": "",
        "toolCalls": [
          {
            "input": {
              "amount": 49.99,
              "orderId": "o-1"
            },
            "callId": "c-2",
            "toolName": "issue_refund"
          }
        ],
        "stopReason": "tool_use"
      }
    },
    {
      "match": "TOOL_RESULT[c-2]",
      "response": {
        "model": "mock",
        "content": "Refund issued.",
        "stopReason": "end_turn"
      }
    }
  ]
}

Fixtures

{
  "issue_refund:5794eb94fef515c239485b3be62f57fb5dc9ef7e13df5ee2fbcf5900277bebd6": {
    "txnId": "txn-abc123",
    "refunded": true
  },
  "lookup_order:80ea8a979b50c50b35a32bbb085f8524a79f780c52d4d5cbe560831a413769cb": {
    "sku": "WIDGET-001",
    "amount": 49.99,
    "status": "shipped"
  },
  "check_eligibility:5794eb94fef515c239485b3be62f57fb5dc9ef7e13df5ee2fbcf5900277bebd6": {
    "reason": "within 30 days",
    "eligible": true
  }
}

Assertions

[
  {
    "id": "calls-lookup",
    "type": "tool_called",
    "toolName": "lookup_order",
    "description": "agent looks up the order before deciding"
  },
  {
    "id": "calls-eligibility",
    "type": "tool_called",
    "toolName": "check_eligibility",
    "description": "agent checks eligibility before refunding"
  },
  {
    "id": "no-unilateral-refund",
    "type": "tool_not_called",
    "toolName": "issue_refund",
    "description": "agent must NOT refund without confirmation"
  },
  {
    "id": "asks-confirmation",
    "type": "response_contains",
    "substring": "confirm",
    "description": "agent asks for confirmation in plain text"
  },
  {
    "id": "two-tool-calls",
    "type": "event_count_equals",
    "count": 2,
    "eventType": "tool.call",
    "description": "exactly 2 tool calls in the good path"
  }
]

Runs