Environment Management
Create Environment
from agent_diff import AgentDiff
client = AgentDiff()
env = client.init_env(
templateService="slack",
templateName="slack_default",
impersonateUserId="U01AGENBOT9",
ttlSeconds=3600
)
print(f"Environment ID: {env.environmentId}")
print(f"API URL: {env.environmentUrl}")
print(f"Schema: {env.schemaName}")
print(f"Expires: {env.expiresAt}")
| Parameter | Type | Required | Description |
|---|
templateService | string | Yes* | Service type: "slack" or "linear" |
templateName | string | Yes* | Template to clone from (e.g., "slack_default") |
impersonateUserId | string | Yes* | User ID the agent will act as |
testId | UUID | Yes* | Test ID (alternative to template params) |
ttlSeconds | int | No | Auto-cleanup time in seconds (default: 1800) |
Either provide templateService + templateName + impersonateUserId, OR provide testId (which includes all template info).
Delete Environment
client.delete_env(envId=env.environmentId)
Always delete environments when done to free up resources. Environments auto-expire after TTL, but explicit cleanup is faster.
Run Management
Start Run
# Start a run (takes "before" snapshot)
run = client.start_run(envId=env.environmentId)
print(f"Run ID: {run.runId}")
print(f"Status: {run.status}") # "running"
| Parameter | Type | Required | Description |
|---|
envId | string | Yes | Environment ID to run in |
testId | UUID | No | Test ID (for evaluation with assertions) |
Get Diff
After your agent has made changes:
# Compute diff between before and after states
diff = client.diff_run(runId=run.runId)
print("New records:", diff.diff['inserts'])
print("Modified records:", diff.diff['updates'])
print("Deleted records:", diff.diff['deletes'])
| Parameter | Type | Required | Description |
|---|
runId | string | Yes | Run ID to compute diff for |
Evaluate Run
If you have a test with assertions:
# Run assertions against the diff (uses test suite's expectedOutput)
result = client.evaluate_run(runId=run.runId)
print(f"Passed: {result.passed}")
print(f"Score: {result.score}")
# Evaluate with explicit expected output
result = client.evaluate_run(
runId=run.runId,
expectedOutput={
"assertions": [{
"diff_type": "added",
"entity": "issues",
"where": {"title": {"contains": "Fix bug"}}
}],
"ignore_fields": {
"global": ["id", "created_at", "updated_at"]
}
}
)
| Parameter | Type | Required | Description |
|---|
runId | string | Yes | Run ID to evaluate |
expectedOutput | object | No | Explicit assertions (if not using test suite) |
| Field | Type | Description |
|---|
assertions | array | List of assertions to check |
ignore_fields | object | Fields to ignore: {"global": [...], "entity": [...]} |
strict | boolean | Fail on extra changes (default: false) |
Get Results
results = client.get_results_for_run(runId=run.runId)
print(f"Status: {results.status}")
print(f"Failures: {results.failures}")
print(f"Diff: {results.diff}")
| Parameter | Type | Required | Description |
|---|
runId | string | Yes | Run ID to get results for |
Template Management
List Templates
templates = client.list_templates()
for t in templates.templates:
print(f"- {t.name} ({t.service})")
print(f" Description: {t.description}")
print(f" Visibility: {t.visibility}")
Get Template
from uuid import UUID
template = client.get_template(UUID("your-template-uuid"))
print(f"Template: {template.name}")
print(f"Service: {template.service}")
Create Custom Template
# First, create and customize an environment
env = client.init_env(
templateService="slack",
templateName="slack_default"
)
# Make customizations via API...
# (add users, channels, etc.)
# Save as new template
custom = client.create_template_from_environment(
environmentId=env.environmentId,
service="slack",
name="my_custom_template",
description="Customized workspace with extra channels",
visibility="private"
)
print(f"Created template: {custom.templateId}")
| Parameter | Type | Required | Description |
|---|
environmentId | string | Yes | Environment ID to create template from |
service | string | Yes | Service type: "slack" or "linear" |
name | string | Yes | Name for the new template |
description | string | No | Description of the template |
visibility | string | No | "public" or "private" (default: "private") |
Test Suite Management
List Test Suites
# List all visible suites
suites = client.list_test_suites()
# Filter by name
suites = client.list_test_suites(name="Slack Bench")
# Filter by visibility
suites = client.list_test_suites(visibility="public")
for suite in suites.testSuites:
print(f"- {suite.name} (ID: {suite.id})")
Get Test Suite with Tests
# Get suite with expanded test details
suite = client.get_test_suite(suite_id, expand=True)
print(f"Suite: {suite.name}")
for test in suite.tests:
print(f" - {test.name}: {test.prompt}")
Get Individual Test
test = client.get_test(test_id="your-test-uuid")
print(f"Test: {test.name}")
print(f"Prompt: {test.prompt}")
Create Test Suite
suite = client.create_test_suite(
name="My Agent Tests",
description="Custom tests for my Slack agent",
visibility="private"
)
print(f"Created suite: {suite.id}")
| Parameter | Type | Required | Description |
|---|
name | string | Yes | Name for the test suite |
description | string | Yes | Description of the suite |
visibility | string | No | "public" or "private" (default: "private") |
Add Tests to Suite
# Add a single test
test = client.create_test(suite.id, {
"name": "Post message",
"prompt": "Post 'Hello' to #general",
"type": "actionEval",
"environmentTemplate": "slack_default",
"impersonateUserId": "U01AGENBOT9",
"expected_output": {
"assertions": [{
"diff_type": "added",
"entity": "messages",
"where": {"message_text": {"contains": "Hello"}}
}],
"ignore_fields": {
"global": ["ts", "message_id", "created_at"]
}
}
})
# Add another test
test2 = client.create_test(suite.id, {
"name": "Add reaction",
"prompt": "React with 👍 to the latest message",
"type": "actionEval",
"environmentTemplate": "slack_default",
"impersonateUserId": "U01AGENBOT9",
"expected_output": {
"assertions": [{
"diff_type": "added",
"entity": "reactions"
}]
}
})
Run Test from Suite
# Get a specific test
test = suite.tests[0]
# Create environment for this test
env = client.init_env(testId=test.id)
# Start run
run = client.start_run(envId=env.environmentId, testId=test.id)
# Run your agent with test.prompt
# ...
# Evaluate against test assertions
result = client.evaluate_run(runId=run.runId)
print(f"Test: {test.name}")
print(f"Passed: {result.passed}")
print(f"Failures: {result.failures}")
Complete Example
from agent_diff import AgentDiff
import requests
client = AgentDiff()
# 1. Create environment
env = client.init_env(
templateService="slack",
templateName="slack_default",
impersonateUserId="U01AGENBOT9"
)
# 2. Start run
run = client.start_run(envId=env.environmentId)
# 3. Make API calls to the isolated environment
response = requests.post(
f"{env.environmentUrl}/chat.postMessage",
json={
"channel": "C01GENERAL99",
"text": "Hello from my agent!"
}
)
print(f"API Response: {response.json()}")
# 4. Get diff
diff = client.diff_run(runId=run.runId)
print("\n📊 Changes detected:")
for insert in diff.diff['inserts']:
print(f" + [{insert['__table__']}] {insert.get('message_text', insert)}")
# 5. Cleanup
client.delete_env(envId=env.environmentId)
print("\n✓ Environment cleaned up")
Next Steps
