import asyncio
import json
import os
import re
import anthropic
from boxlite import ComputerBox
vision = anthropic.Anthropic(
base_url=os.environ["ANTHROPIC_BASE_URL"], # <YOUR_ANTHROPIC_COMPATIBLE_BASE_URL>
api_key=os.environ["ANTHROPIC_API_KEY"], # <YOUR_API_KEY>
)
VISION_MODEL = "<YOUR_VISION_MODEL>" # any model id that accepts images
# The action vocabulary maps one-to-one onto ComputerBox methods
SYSTEM = """You operate a Linux desktop (1024x768) via screenshots.
Look at the screenshot and decide the SINGLE next action toward the goal.
Return ONLY one JSON object, no prose:
{"action":"mouse_move","x":<int>,"y":<int>}
{"action":"left_click"}
{"action":"double_click"}
{"action":"type","text":"<str>"}
{"action":"key","text":"<xdotool keyname, e.g. Return, ctrl+a>"}
{"action":"scroll","x":<int>,"y":<int>,"direction":"up|down","amount":<int>}
"""
def text_of(response) -> str:
"""Return the last text block — some models emit a thinking block first."""
out = None
for block in response.content:
if block.type == "text":
out = block.text
return out or ""
def extract_action(text: str) -> dict:
match = re.search(r"\{.*\}", text, re.S)
return json.loads(match.group(0) if match else text)
async def run_action(desktop, action: dict) -> None:
"""Apply one model-chosen action to the real desktop."""
kind = action["action"]
if kind == "mouse_move":
await desktop.mouse_move(int(action["x"]), int(action["y"]))
elif kind == "left_click":
await desktop.left_click()
elif kind == "double_click":
await desktop.double_click()
elif kind == "type":
await desktop.type(action["text"])
elif kind == "key":
await desktop.key(action["text"]) # e.g. "Return", "ctrl+a"
elif kind == "scroll":
await desktop.scroll(int(action["x"]), int(action["y"]),
action["direction"], int(action.get("amount", 3)))
print(f"executed: {kind} {action}")
async def main() -> None:
goal = ("As a concrete first step, move the mouse to the center "
"of the screen at coordinates (512, 384).")
try:
async with ComputerBox(cpu=2, memory=2048) as desktop:
# The first run pulls a large image; be generous with the timeout
await desktop.wait_until_ready(timeout=180)
# 1) Observe — screenshot() returns a dict with base64 PNG in ["data"]
shot = await desktop.screenshot()
print(f"screenshot {shot['width']}x{shot['height']} {shot['format']}")
# 2) Think — hand the image to the vision model
response = vision.messages.create(
model=VISION_MODEL,
max_tokens=512,
system=SYSTEM,
messages=[{"role": "user", "content": [
{"type": "image", "source": {
"type": "base64", "media_type": "image/png",
"data": shot["data"],
}},
{"type": "text", "text": f"Goal: {goal}\nWhat is the next single action?"},
]}],
)
action = extract_action(text_of(response))
print(f"model chose: {action}")
# 3) Act — then verify by reading the cursor and re-shooting
await run_action(desktop, action)
x, y = await desktop.cursor_position()
print(f"cursor now at ({x}, {y})")
await desktop.screenshot()
except TimeoutError:
print("the desktop did not become ready in time — raise the timeout")
except RuntimeError as exc:
print(f"sandbox failed to start: {exc}")
asyncio.run(main())
