Cookbook: Browser Automation

import { ComputerUseCapability } from '@allternit/sdk/ai-runtime/capabilities/computer-use';
import { ToolRegistry } from '@allternit/sdk/ai-runtime/tools';

const registry = new ToolRegistry();

const computer = new ComputerUseCapability({
  displayWidthPx: 1280,
  displayHeightPx: 720,
});

registry.registerTool(computer.getTool(), { strict: true });

const response = await fetch('http://127.0.0.1:8013/api/aci/run', {
  method: 'POST',
  headers: { 'Content-Type': 'application/json' },
  body: JSON.stringify({
    goal: 'Go to https://news.ycombinator.com and extract the top 10 story titles and URLs',
    model: 'gizzi-pro',
    allowedSites: ['news.ycombinator.com'],
    autoVerify: false,
  }),
});

const { sessionId } = await response.json();
console.log('Run started:', sessionId);

const eventSource = new EventSource(
  `http://127.0.0.1:8013/api/aci/stream/${sessionId}`
);

eventSource.onmessage = (event) => {
  const data = JSON.parse(event.data);

  switch (data.type) {
    case 'trace':
      console.log('Progress:', data.message);
      break;
    case 'state':
      if (data.requiresApproval) {
        console.log('Action needs approval:', data.action);
        // Approve or deny
        approveAction(sessionId, data.actionId, true);
      }
      break;
    case 'done':
      console.log('Run completed:', data.result);
      eventSource.close();
      break;
  }
};

async function approveAction(
  sessionId: string,
  actionId: string,
  approved: boolean
) {
  await fetch(
    `http://127.0.0.1:8013/api/aci/approve/${sessionId}?deny=${!approved}`,
    {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({ actionId }),
    }
  );
}

// After the run completes, extract the data
const result = data.result;
const stories = result.extractedData;

console.table(stories);
// ┌─────────┬────────────────────────────────┬──────────────────────────┐
// │ (index) │ title                          │ url                      │
// ├─────────┼────────────────────────────────┼──────────────────────────┤
// │ 0       │ 'Show HN: My new project'      │ 'https://...'            │
// │ 1       │ 'The future of AI agents'      │ 'https://...'            │
// └─────────┴────────────────────────────────┴──────────────────────────┘