Agent Tools at Runtime

With function calling, a model can only use the tools you wrote ahead of time. When a customer asks for something slightly different, like "how much have I spent this year?", the agent has to string together calls it wasn't designed for, or give up.

Here the agent's tools live in a Stof document. Your app turns them into tool definitions for the model, runs the model's tool calls inside the document, and lets the agent write a new tool when it needs one. The new tool is sandboxed like everything else, and it's there for the rest of the conversation.

The Agent Document

Tools are functions tagged #[tool('description')]. They reach your systems only through functions your app provides (App.orders, App.refund). The document also has the app's two helpers: tool_specs turns the tools into JSON tool definitions, and call_tool runs one. Press Run to see the agent add a tool and use it. In this demo, a root named App stands in for your app's functions (so the tools call App.orders with a dot, which reaches a root or a library; App::orders would only reach the library):

agent.stofPlayground ↗

tools: {
  #[tool("List a customer's recent orders")]
  fn recent_orders(customer_id: str) -> list {
      await App.orders(customer_id)
  }

  #[tool('Refund an order')]
  fn refund(order_id: str, amount: float) -> str {
      await App.refund(order_id, amount)
  }

  #[tool('Add a tool for yourself. source is Stof: one function tagged #[tool(description)] that uses App functions.')]
  fn add_tool(source: str) -> str {
      try {
          using staging = new {};
          parse(source, staging);
          const added = staging.funcs('tool');
          if (added.len() != 1) return 'add exactly one function tagged #[tool]';
          parse(source, self);
          return 'added ' + added[0].name();
      } catch (error) {
          return 'that tool did not parse: ' + str(error);
      }
  }
}

fn json_type(stof_type: str) -> str {
  switch (stof_type) {
      case 'str': 'string'
      case 'int': 'integer'
      case 'float': 'number'
      case 'bool': 'boolean'
      case 'list': 'array'
      default: 'object'
  }
}

/// The tools, as JSON tool definitions for the model's API.
fn tool_specs() -> str {
  using specs = new { tools: [] };
  for (const tool in self.tools.funcs('tool')) {
      const spec = new {
          name: tool.name(),
          description: tool.attributes().get('tool'),
          input_schema: new { type: 'object', properties: new {}, required: [] },
      } on specs;
      for (const (name, type) in tool.params()) {
          spec.input_schema.properties.insert(name, new { type: self.json_type(type) });
          spec.input_schema.required.push_back(name);
      }
      specs.tools.push_back(spec);
  }
  stringify('json', specs)
}

/// Runs a tool the model called, with the arguments it gave.
fn call_tool(name: str, args: map) -> unknown {
  const tool = self.tools.get(name);
  if (tool == null || !tool.has_attribute('tool')) throw(`no tool named ${name}`);
  const values = [];
  for (const (param, _) in tool.params()) values.push_back(args.get(param));
  tool.call_expanded(values)
}

// ---- for this demo only: stands in for the functions your app provides ----
root App {
  fn orders(customer_id: str) -> list { [{'id': 'o_1', 'total': 40}, {'id': 'o_2', 'total': 250}] }
  fn refund(order_id: str, amount: float) -> str { 'refunded ' + amount + ' on ' + order_id }
}

#[main]
fn main() {
  // the model asks for a tool it doesn't have, then uses it
  pln(self.call_tool('add_tool', {'source': r#"
      #[tool('Total a customer has spent across their recent orders')]
      fn total_spent(customer_id: str) -> float {
          let total = 0;
          for (const order in await App.orders(customer_id)) total += order.get('total');
          total
      }
  "#}));
  pln(self.call_tool('total_spent', {'customer_id': 'c_9'}));

  // a broken tool comes back as a message the model can act on
  pln(self.call_tool('add_tool', {'source': 'fn broken( {'}));
}

Output

Giving the Agent Your Systems

The functions you give the document are the only way its tools can reach anything. Put your limits in them, not in the tools, because the agent can write new tools that call them:

import { StofDoc } from '@formata/stof';

function connect(doc: StofDoc, customerId: string) {
    // scoped to this customer: a tool can't ask for anyone else's orders
    doc.lib('App', 'orders', async () => db.orders.forCustomer(customerId), true);

    doc.lib('App', 'refund', async (orderId: string, amount: number) => {
        if (amount > 100) throw new Error('refunds over $100 need a person to approve them');
        return payments.refund(orderId, amount);
    }, true);

    doc.setMaxExecutionTime(2000);   // no tool runs longer than 2 seconds
}

The document has no network, file, or environment access, so App.orders and App.refund are all of its reach. A refund over $100 fails the same way whether it comes from the built-in refund tool or one the agent wrote.

The Conversation Loop

Each turn, give the model the current tools, and run whatever it calls:

const doc = await StofDoc.new(AGENT_SOURCE);
connect(doc, customerId);

// Maps from Stof become plain objects in JSON
const toJSON = (value: unknown) => JSON.stringify(value, (_, v) => v instanceof Map ? Object.fromEntries(v) : v);

while (true) {
    const tools = JSON.parse(await doc.call('tool_specs') as string).tools;
    const response = await model.chat({ messages, tools });
    messages.push(response.message);
    if (!response.toolCalls.length) break;

    for (const call of response.toolCalls) {
        let result: string;
        try {
            result = toJSON(await doc.call('call_tool', call.name, call.input));
        } catch (error) {
            result = `error: ${error}`;   // a plain message the model can act on
        }
        messages.push({ role: 'tool', toolCallId: call.id, content: result });
    }
}

Because tool_specs reads the document each turn, a tool the agent adds with add_tool shows up in the very next request, with its description and parameter types. Tell the model in the system prompt that it can write tools, and give it the Stof skill so it writes them correctly.

Keeping the Agent Between Requests

The agent is a document, so save it like data. Binary Stof keeps everything, including the tools it wrote:

// end of a request
await sessions.save(sessionId, doc.blobify('bstf'));
doc.dispose();

// next request
const doc = await StofDoc.new(await sessions.load(sessionId), 'bstf');
connect(doc, customerId);    // your functions aren't saved with it; connect them again

You can keep more than tools in the same document: notes the agent takes, the customer's preferences, a running summary of the conversation. It's all one value to store, and it moves between services or into a different language (a Python worker, say) without any translation.

Why This Works

Tools are data. They're listed, described, saved, and sent like any other part of the document, so the tool definitions can't drift from the tools.
New tools are safe to accept. A tool the agent writes can only do what your functions let it, within a time limit.
Mistakes are recoverable. A tool that doesn't parse, or one that throws, comes back to the model as a plain message, and the model fixes it.

To have a model write logic once and run it many times, rather than during a conversation, see Run Rules a Model Writes.

The Agent Document​

Giving the Agent Your Systems​

The Conversation Loop​

Keeping the Agent Between Requests​

Why This Works​

The Agent Document

Giving the Agent Your Systems

The Conversation Loop

Keeping the Agent Between Requests

Why This Works