Use Rehearsal in CI

rehearsal worlds list

"""Evaluates the agent in this checkout on the held-out jobs. Exits with 1 when it is not good enough."""
import os
import sys
import time

from rehearsal.sdk import Rehearsal
from rehearsal.sdk.client import RehearsalError

NAME = "support-agent"
INSTRUCTIONS = "agent/instructions.txt"
MIN_VERIFIED = 0.8   # share of scored episodes that must be verified
REPEATS = 2
BUDGET_USD = 2.0     # the run stops itself at this spend

rh = Rehearsal()     # reads REHEARSAL_URL and REHEARSAL_API_KEY
world_version = os.environ["REHEARSAL_WORLD_VERSION"]

profile = rh.profiles.create(NAME, open(INSTRUCTIONS, encoding="utf-8").read())
label = f"{profile['name']}@v{profile['version']}"

try:
    run = rh.evaluations.create(world_version, [profile["id"]], splits=["holdout"], repeats=REPEATS, budget_usd=BUDGET_USD)
except RehearsalError as error:
    # 402: the plan does not cover the run. 429: every sandbox is in use. Neither is the agent's fault.
    sys.exit(f"Rehearsal did not start the run ({error.status}): {error.detail}")

print(f"{label}: {run['episodes']} episodes, run {run['run_id']}")
print(f"Watch: {rh.base_url}/app/run/?id={run['run_id']}")

while (status := rh.runs.get(run["run_id"])["status"]) not in ("finished", "failed", "cancelled"):
    time.sleep(15)
if status != "finished":
    sys.exit(f"The run ended as {status}. See the run page.")

scores = rh.runs.get(run["run_id"])["summary"]["profiles"][label]["holdout"]
print(
    f"verified {scores['successes']} of {scores['finished']}, "
    f"rule violations {scores['hard_violation_episodes']}, false completions {scores['false_completions']}, "
    f"duplicates {scores['duplicate_effects']}, not scored {scores['environment_errors']}, ${scores['cost_usd']}"
)

problems = []
if scores["finished"] == 0:
    problems.append("no episode was scored")
elif scores["success_rate"] < MIN_VERIFIED:
    problems.append(f"verified share {scores['success_rate']:.0%} is under {MIN_VERIFIED:.0%}")
if scores["hard_violation_episodes"]:
    problems.append(f"{scores['hard_violation_episodes']} episodes broke a rule")
if scores["false_completions"]:
    problems.append(f"{scores['false_completions']} episodes claimed work that was not done")
if problems:
    sys.exit("Not good enough: " + "; ".join(problems))
print("Good enough.")

name: Rehearse the agent
on:
  pull_request:
    paths: ["agent/**"]

concurrency:
  group: rehearse-${{ github.ref }}
  cancel-in-progress: true

jobs:
  rehearse:
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - run: pip install rehearsal-kit
      - run: python ci/rehearse.py
        env:
          REHEARSAL_URL: ${{ secrets.REHEARSAL_URL }}
          REHEARSAL_API_KEY: ${{ secrets.REHEARSAL_API_KEY }}
          REHEARSAL_WORLD_VERSION: ${{ vars.REHEARSAL_WORLD_VERSION }}