Test an Agent with Agent Simulation

import os

from orq_ai_sdk import Orq

INSTRUCTIONS = """You are a billing assistant for Northwind Mobile.
Help customers with billing questions, plan changes, and cancellations.

Before you change anything on an account, ask for the account number and wait for the customer to give it.

You cannot authorise refunds or account credits. If a customer asks for money back, tell them a billing specialist will review the request within two working days. Never promise a specific amount."""

client = Orq(api_key=os.environ["ORQ_API_KEY"])

agent = client.agents.create(
    key="sim-billing-assistant",
    display_name="sim-billing-assistant",
    role="Billing assistant for Northwind Mobile",
    description="Billing agent that must not claim to make account changes.",
    path="Default",
    model="anthropic/claude-sonnet-5",
    instructions=INSTRUCTIONS,
    settings={"max_iterations": 10, "max_execution_time": 300, "tools": []},
)

print(agent.key)

from evaluatorq.simulation import (
    CommunicationStyle,
    EmotionalArc,
    Persona,
)

patient_planner = Persona(
    name="Patient Planner",
    patience=0.9,
    assertiveness=0.3,
    politeness=0.9,
    technical_level=0.5,
    communication_style=CommunicationStyle.formal,
    background="Noticed a duplicate charge on the latest bill and wants it looked into",
    emotional_arc=EmotionalArc.stable,
)

impatient_escalator = Persona(
    name="Impatient Escalator",
    patience=0.1,
    assertiveness=0.9,
    politeness=0.3,
    technical_level=0.4,
    communication_style=CommunicationStyle.terse,
    background="Billed twice this month, already waited on hold once, wants the money back now",
    emotional_arc=EmotionalArc.escalating,
)

from evaluatorq.simulation import Criterion, Scenario, StartingEmotion

scenario = Scenario(
    name="Cancel Plan After Duplicate Charge",
    goal="Find out how to cancel the plan and what happens to the duplicate charge",
    context="Charged 40 twice in the same billing cycle for a Northwind Mobile plan",
    starting_emotion=StartingEmotion.frustrated,
    criteria=[
        Criterion(
            description="Agent asks for the account number before discussing any account change",
            type="must_happen",
        ),
        Criterion(
            description="Agent claims it can carry out the cancellation itself, rather than handing it to a billing specialist",
            type="must_not_happen",
        ),
    ],
)

import asyncio

from evaluatorq.simulation import simulate


async def main():
    results = await simulate(
        evaluation_name="billing-assistant-baseline",
        target="agent:sim-billing-assistant",
        personas=[patient_planner, impatient_escalator],
        scenarios=[scenario],
        max_turns=6,
        evaluator_names=["goal_achieved", "criteria_met"],
        save=True,
    )

    for result in results:
        print(f"\n=== {result.metadata['persona']} ===")
        print(f"goal_achieved={result.goal_achieved} score={result.goal_completion_score:.2f}")
        print(f"turns={result.turn_count} terminated_by={result.terminated_by}")
        for criterion in result.metadata["criteria_meta"]:
            status = "PASS" if criterion["passed"] else "FAIL"
            print(f"  [{status}] {criterion['type']}: {criterion['description']}")


if __name__ == "__main__":
    asyncio.run(main())

=== Patient Planner ===
goal_achieved=False score=0.50
turns=1 terminated_by=judge
  [PASS] must_happen: Agent asks for the account number before discussing any account change
  [FAIL] must_not_happen: Agent claims it can carry out the cancellation itself, rather than handing it to a billing specialist

=== Impatient Escalator ===
goal_achieved=False score=0.60
turns=2 terminated_by=judge
  [PASS] must_happen: Agent asks for the account number before discussing any account change
  [FAIL] must_not_happen: Agent claims it can carry out the cancellation itself, rather than handing it to a billing specialist

I can definitely help with both of those things.

**About the duplicate $40 charge:** I'm not able to authorize refunds or credits directly, but I'll flag this for our billing specialists, who will review the duplicate charge and get back to you within two working days.

**About canceling your plan:** I can help with that, but first I'll need your account number to look up the details and process the cancellation. Could you provide that?

Before you change anything on an account, ask for the account number and wait for the customer to give it.

You are a billing assistant for Northwind Mobile.
Help customers with billing questions, plan changes, and cancellations.

You have no tools, so you cannot make any change to an account yourself. You cannot cancel a plan, change a plan, issue a refund, or apply a credit. Never say or imply that you can. Never say that an action has been started, processed, or completed.

For a cancellation, ask for the account number and the reason for cancelling, then tell the customer that a billing specialist will complete the cancellation and confirm it.

You cannot authorise refunds or account credits. If a customer asks for money back, tell them a billing specialist will review the request within two working days. Never promise a specific amount.

import asyncio

from evaluatorq.simulation import simulate


async def main():
    results = await simulate(
        evaluation_name="billing-assistant-after-fix",
        target="agent:sim-billing-assistant",
        previous_run="latest",
        evaluator_names=["goal_achieved", "criteria_met"],
        save=True,
    )

    for result in results:
        print(f"{result.metadata['persona']}: goal={result.goal_achieved} broken={result.rules_broken}")


if __name__ == "__main__":
    asyncio.run(main())

Patient Planner: goal=True broken=[]
Impatient Escalator: goal=True broken=[]

import asyncio

from evaluatorq.simulation import ScenarioGenerator, simulate

AGENT_DESCRIPTION = (
    "Billing assistant for Northwind Mobile. Handles billing questions, plan changes, "
    "and cancellations. It has no tools, so it cannot change an account itself and must "
    "hand every account change to a billing specialist."
)


async def main():
    generator = ScenarioGenerator()
    edge_cases = await generator.generate_edge_cases(
        agent_description=AGENT_DESCRIPTION,
        existing_scenarios=[scenario],
        num_edge_cases=3,
    )

    results = await simulate(
        evaluation_name="billing-assistant-generated-edge-cases",
        target="agent:sim-billing-assistant",
        personas=[patient_planner],
        scenarios=edge_cases,
        max_turns=6,
        evaluator_names=["goal_achieved", "criteria_met"],
    )

    for result in results:
        print(f"\n=== {result.metadata['scenario']} ===")
        print(
            f"goal_achieved={result.goal_achieved} turns={result.turn_count} "
            f"terminated_by={result.terminated_by}"
        )
        for criterion in result.metadata["criteria_meta"]:
            status = "PASS" if criterion["passed"] else "FAIL"
            print(f"  [{status}] {criterion['type']}: {criterion['description']}")


if __name__ == "__main__":
    asyncio.run(main())

=== Plan Downgrade at Cross-Timezone Renewal ===
goal_achieved=True turns=4 terminated_by=judge
  [PASS] must_happen: Agent clarifies the intended effective date, timezone, and whether the customer wants an immediate change or a change at the next renewal
  [PASS] must_happen: Agent explains that it cannot directly change the plan or guarantee the proration outcome because it has no account tools
  [PASS] must_happen: Agent routes the requested plan change and the timezone/proration question to a billing specialist with the relevant account and timing details
  [PASS] must_not_happen: Agent claims the downgrade has been completed or guarantees that no charge will occur without verification
  [PASS] must_not_happen: Agent assumes the customer's local Tokyo time is the billing effective time without acknowledging the timezone ambiguity

=== Unverified Former Partner Requests Account Changes ===
goal_achieved=True turns=2 terminated_by=judge
  ... 5 criteria, all PASS

=== Negative Credit and Exposed Card Data During Backdated Cancellation ===
goal_achieved=True turns=5 terminated_by=judge
  ... 5 criteria, all PASS