Basic Usage

Core operations with the Python SDK

Environment Management

Create Environment

from agent_diff import AgentDiff

client = AgentDiff()

env = client.init_env(
    templateService="slack",
    templateName="slack_default",
    impersonateUserId="U01AGENBOT9",
    ttlSeconds=3600
)

print(f"Environment ID: {env.environmentId}")
print(f"API URL: {env.environmentUrl}")
print(f"Schema: {env.schemaName}")
print(f"Expires: {env.expiresAt}")

Parameters:

Parameter	Type	Required	Description
templateService	string	Yes*	Service type: "slack" or "linear"
templateName	string	Yes*	Template to clone from (e.g., "slack_default")
impersonateUserId	string	Yes*	User ID the agent will act as
testId	UUID	Yes*	Test ID (alternative to template params)
ttlSeconds	int	No	Auto-cleanup time in seconds (default: 1800)

Either provide templateService + templateName + impersonateUserId, OR provide testId (which includes all template info).

Delete Environment

client.delete_env(envId=env.environmentId)

Always delete environments when done to free up resources. Environments auto-expire after TTL, but explicit cleanup is faster.

---

Run Management

Start Run

# Start a run (takes "before" snapshot)
run = client.start_run(envId=env.environmentId)

print(f"Run ID: {run.runId}")
print(f"Status: {run.status}")  # "running"

Parameters:

Parameter	Type	Required	Description
envId	string	Yes	Environment ID to run in
testId	UUID	No	Test ID (for evaluation with assertions)

Get Diff

After your agent has made changes:

# Compute diff between before and after states
diff = client.diff_run(runId=run.runId)

print("New records:", diff.diff['inserts'])
print("Modified records:", diff.diff['updates'])
print("Deleted records:", diff.diff['deletes'])

Parameters:

Parameter	Type	Required	Description
runId	string	Yes	Run ID to compute diff for

Evaluate Run

If you have a test with assertions:

# Run assertions against the diff (uses test suite's expectedOutput)
result = client.evaluate_run(runId=run.runId)

print(f"Passed: {result.passed}")
print(f"Score: {result.score}")

Or pass assertions explicitly (no test suite needed):

# Evaluate with explicit expected output
result = client.evaluate_run(
    runId=run.runId,
    expectedOutput={
        "assertions": [{
            "diff_type": "added",
            "entity": "issues",
            "where": {"title": {"contains": "Fix bug"}}
        }],
        "ignore_fields": {
            "global": ["id", "created_at", "updated_at"]
        }
    }
)

Parameters:

Parameter	Type	Required	Description
runId	string	Yes	Run ID to evaluate
expectedOutput	object	No	Explicit assertions (if not using test suite)

Expected Output Structure:

Field	Type	Description
assertions	array	List of assertions to check
ignore_fields	object	Fields to ignore: {"global": [...], "entity": [...]}
strict	boolean	Fail on extra changes (default: false)

Get Results

results = client.get_results_for_run(runId=run.runId)

print(f"Status: {results.status}")
print(f"Failures: {results.failures}")
print(f"Diff: {results.diff}")

Parameters:

Parameter	Type	Required	Description
runId	string	Yes	Run ID to get results for

---

Template Management

List Templates

templates = client.list_templates()

for t in templates.templates:
    print(f"- {t.name} ({t.service})")
    print(f"  Description: {t.description}")
    print(f"  Visibility: {t.visibility}")

Get Template

from uuid import UUID

template = client.get_template(UUID("your-template-uuid"))
print(f"Template: {template.name}")
print(f"Service: {template.service}")

Create Custom Template

# First, create and customize an environment
env = client.init_env(
    templateService="slack",
    templateName="slack_default"
)

# Make customizations via API...
# (add users, channels, etc.)

# Save as new template
custom = client.create_template_from_environment(
    environmentId=env.environmentId,
    service="slack",
    name="my_custom_template",
    description="Customized workspace with extra channels",
    visibility="private"
)

print(f"Created template: {custom.templateId}")

Parameters:

Parameter	Type	Required	Description
environmentId	string	Yes	Environment ID to create template from
service	string	Yes	Service type: "slack" or "linear"
name	string	Yes	Name for the new template
description	string	No	Description of the template
visibility	string	No	"public" or "private" (default: "private")

---

Test Suite Management

List Test Suites

# List all visible suites
suites = client.list_test_suites()

# Filter by name
suites = client.list_test_suites(name="Slack Bench")

# Filter by visibility
suites = client.list_test_suites(visibility="public")

for suite in suites.testSuites:
    print(f"- {suite.name} (ID: {suite.id})")

Get Test Suite with Tests

# Get suite with expanded test details
suite = client.get_test_suite(suite_id, expand=True)

print(f"Suite: {suite.name}")
for test in suite.tests:
    print(f"  - {test.name}: {test.prompt}")

Get Individual Test

test = client.get_test(test_id="your-test-uuid")
print(f"Test: {test.name}")
print(f"Prompt: {test.prompt}")

Create Test Suite

suite = client.create_test_suite(
    name="My Agent Tests",
    description="Custom tests for my Slack agent",
    visibility="private"
)

print(f"Created suite: {suite.id}")

Parameters:

Parameter	Type	Required	Description
name	string	Yes	Name for the test suite
description	string	Yes	Description of the suite
visibility	string	No	"public" or "private" (default: "private")

Add Tests to Suite

# Add a single test
test = client.create_test(suite.id, {
    "name": "Post message",
    "prompt": "Post 'Hello' to #general",
    "type": "actionEval",
    "environmentTemplate": "slack_default",
    "impersonateUserId": "U01AGENBOT9",
    "expected_output": {
        "assertions": [{
            "diff_type": "added",
            "entity": "messages",
            "where": {"message_text": {"contains": "Hello"}}
        }],
        "ignore_fields": {
            "global": ["ts", "message_id", "created_at"]
        }
    }
})

# Add another test
test2 = client.create_test(suite.id, {
    "name": "Add reaction",
    "prompt": "React with 👍 to the latest message",
    "type": "actionEval",
    "environmentTemplate": "slack_default",
    "impersonateUserId": "U01AGENBOT9",
    "expected_output": {
        "assertions": [{
            "diff_type": "added",
            "entity": "reactions"
        }]
    }
})

Run Test from Suite

# Get a specific test
test = suite.tests[0]

# Create environment for this test
env = client.init_env(testId=test.id)

# Start run
run = client.start_run(envId=env.environmentId, testId=test.id)

# Run your agent with test.prompt
# ...

# Evaluate against test assertions
result = client.evaluate_run(runId=run.runId)

print(f"Test: {test.name}")
print(f"Passed: {result.passed}")
print(f"Failures: {result.failures}")

---

Complete Example

from agent_diff import AgentDiff
import requests

client = AgentDiff()

# 1. Create environment
env = client.init_env(
    templateService="slack",
    templateName="slack_default",
    impersonateUserId="U01AGENBOT9"
)

# 2. Start run
run = client.start_run(envId=env.environmentId)

# 3. Make API calls to the isolated environment
response = requests.post(
    f"{env.environmentUrl}/chat.postMessage",
    json={
        "channel": "C01GENERAL99",
        "text": "Hello from my agent!"
    }
)
print(f"API Response: {response.json()}")

# 4. Get diff
diff = client.diff_run(runId=run.runId)

print("\n📊 Changes detected:")
for insert in diff.diff['inserts']:
    print(f"  + [{insert['__table__']}] {insert.get('message_text', insert)}")

# 5. Cleanup
client.delete_env(envId=env.environmentId)
print("\n✓ Environment cleaned up")

Next Steps

Code Executors

Use proxy tools for agent frameworks

OpenAI Agents

Integrate with OpenAI Agents SDK