Tool Calling

📖

This page describes features extending the AI Proxy, which provides a unified API for accessing multiple AI providers. To learn more, see AI Proxy.

Quick Start

Enable AI models to call external functions with structured parameters.

const tools = [
  {
    type: "function",
    function: {
      name: "get_weather",
      description: "Get current weather for a location",
      parameters: {
        type: "object",
        properties: {
          location: { type: "string", description: "City and state" },
          unit: { type: "string", enum: ["celsius", "fahrenheit"] },
        },
        required: ["location"],
      },
    },
  },
];

const response = await openai.chat.completions.create({
  model: "openai/gpt-4o",
  messages: [{ role: "user", content: "What's the weather in NYC?" }],
  tools,
  tool_choice: "auto",
});

// Handle tool calls
if (response.choices[0].message.tool_calls) {
  const toolCall = response.choices[0].message.tool_calls[0];
  const args = JSON.parse(toolCall.function.arguments);

  // Execute function
  const result = await getWeather(args.location, args.unit);

  // Continue conversation with result
  const finalResponse = await openai.chat.completions.create({
    model: "openai/gpt-4o",
    messages: [
      { role: "user", content: "What's the weather in NYC?" },
      response.choices[0].message,
      {
        role: "tool",
        tool_call_id: toolCall.id,
        content: JSON.stringify(result),
      },
    ],
  });
}

Configuration

Tool Definition

ParameterTypeRequiredDescription
type"function"YesTool type
function.namestringYesFunction identifier
function.descriptionstringYesWhat the function does
function.parametersobjectYesJSON Schema for parameters

Tool Choice Options

ValueBehavior
"auto"Model decides when to use tools
"none"Disable tool usage
"required"Force tool usage
{type: "function", function: {name: "tool_name"}}Force specific tool

Supported Models

ProviderModelSupportParallel Calls
OpenAIgpt-4o✅ Full✅ Yes
OpenAIgpt-4✅ Full✅ Yes
OpenAIgpt-3.5-turbo✅ Full❌ No
Anthropicclaude-3-5-sonnet✅ Full✅ Yes
Googlegemini-1.5-pro✅ Full⚠️ Limited
Mistralmistral-large✅ Full✅ Yes

Code examples

curl -X POST https://api.orq.ai/v2/proxy/chat/completions \
  -H "Authorization: Bearer $ORQ_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "model": "openai/gpt-4o",
    "messages": [
      {
        "role": "user",
        "content": "What is the weather like in San Francisco?"
      }
    ],
    "tools": [
      {
        "type": "function",
        "function": {
          "name": "get_weather",
          "description": "Get the current weather for a location",
          "parameters": {
            "type": "object",
            "properties": {
              "location": {
                "type": "string",
                "description": "The city and state, e.g. San Francisco, CA"
              },
              "unit": {
                "type": "string",
                "enum": ["celsius", "fahrenheit"],
                "description": "The temperature unit"
              }
            },
            "required": ["location"]
          }
        }
      }
    ],
    "tool_choice": "auto"
  }'
from openai import OpenAI
import json
import os

openai = OpenAI(
  api_key=os.environ.get("ORQ_API_KEY"),
  base_url="https://api.orq.ai/v2/proxy"
)

# Define your tools
tools = [
    {
        "type": "function",
        "function": {
            "name": "get_weather",
            "description": "Get the current weather for a location",
            "parameters": {
                "type": "object",
                "properties": {
                    "location": {
                        "type": "string",
                        "description": "The city and state, e.g. San Francisco, CA"
                    },
                    "unit": {
                        "type": "string",
                        "enum": ["celsius", "fahrenheit"],
                        "description": "The temperature unit"
                    }
                },
                "required": ["location"]
            }
        }
    }
]

messages = [
    {"role": "user", "content": "What's the weather like in San Francisco?"}
]

# First API call - model decides to use the tool
response = openai.chat.completions.create(
    model="openai/gpt-4o",
    messages=messages,
    tools=tools,
    tool_choice="auto"
)

# Check if the model wants to use a tool
if response.choices[0].message.tool_calls:
    tool_call = response.choices[0].message.tool_calls[0]

    # Parse the arguments
    arguments = json.loads(tool_call.function.arguments)

    # Execute your function (mock response here)
    weather_result = {
        "temperature": 72,
        "unit": "fahrenheit",
        "description": "Sunny with light clouds"
    }

    # Add the assistant's message with tool call to history
    messages.append(response.choices[0].message)

    # Add the tool result to messages
    messages.append({
        "role": "tool",
        "tool_call_id": tool_call.id,
        "content": json.dumps(weather_result)
    })

    # Second API call - get final response with tool result
    final_response = openai.chat.completions.create(
        model="openai/gpt-4o",
        messages=messages
    )

    print(final_response.choices[0].message.content)
import OpenAI from "openai";

const openai = new OpenAI({
  apiKey: process.env.ORQ_API_KEY,
  baseURL: "https://api.orq.ai/v2/proxy",
});

// Define your tools
const tools = [
  {
    type: "function" as const,
    function: {
      name: "get_weather",
      description: "Get the current weather for a location",
      parameters: {
        type: "object",
        properties: {
          location: {
            type: "string",
            description: "The city and state, e.g. San Francisco, CA",
          },
          unit: {
            type: "string",
            enum: ["celsius", "fahrenheit"],
            description: "The temperature unit",
          },
        },
        required: ["location"],
      },
    },
  },
];

const messages = [
  { role: "user" as const, content: "What's the weather like in San Francisco?" },
];

// First API call - model decides to use the tool
const response = await openai.chat.completions.create({
  model: "openai/gpt-4o",
  messages,
  tools,
  tool_choice: "auto",
});

// Check if the model wants to use a tool
if (response.choices[0].message.tool_calls) {
  const toolCall = response.choices[0].message.tool_calls[0];

  // Parse the arguments
  const arguments = JSON.parse(toolCall.function.arguments);

  // Execute your function (mock response here)
  const weatherResult = {
    temperature: 72,
    unit: "fahrenheit",
    description: "Sunny with light clouds",
  };

  // Add the assistant's message with tool call to history
  messages.push(response.choices[0].message);

  // Add the tool result to messages
  messages.push({
    role: "tool" as const,
    tool_call_id: toolCall.id,
    content: JSON.stringify(weatherResult),
  });

  // Second API call - get final response with tool result
  const finalResponse = await openai.chat.completions.create({
    model: "openai/gpt-4o",
    messages,
  });

  console.log(finalResponse.choices[0].message.content);
}

Function Execution Patterns

Basic Tool Handler

class ToolHandler {
  constructor() {
    this.tools = new Map();
  }

  register(name, func, schema) {
    this.tools.set(name, { func, schema });
  }

  async execute(toolCall) {
    const tool = this.tools.get(toolCall.function.name);
    if (!tool) throw new Error(`Unknown tool: ${toolCall.function.name}`);

    const args = JSON.parse(toolCall.function.arguments);
    const result = await tool.func(args);

    return {
      role: "tool",
      tool_call_id: toolCall.id,
      content: JSON.stringify(result),
    };
  }
}

// Usage
const handler = new ToolHandler();
handler.register("get_weather", getWeatherAPI, weatherSchema);
handler.register("search_web", searchWebAPI, searchSchema);

// Execute tool calls
const toolResults = await Promise.all(
  response.choices[0].message.tool_calls.map((call) => handler.execute(call)),
);

Parallel Tool Execution

import asyncio
import json

async def execute_tools_parallel(tool_calls):
    async def execute_single_tool(tool_call):
        function_name = tool_call.function.name
        arguments = json.loads(tool_call.function.arguments)

        # Route to appropriate function
        if function_name == "get_weather":
            result = await get_weather_async(arguments)
        elif function_name == "search_products":
            result = await search_products_async(arguments)
        elif function_name == "check_inventory":
            result = await check_inventory_async(arguments)
        else:
            result = {"error": f"Unknown function: {function_name}"}

        return {
            "role": "tool",
            "tool_call_id": tool_call.id,
            "content": json.dumps(result)
        }

    # Execute all tools concurrently
    results = await asyncio.gather(
        *[execute_single_tool(call) for call in tool_calls]
    )

    return results

# Usage
if response.choices[0].message.tool_calls:
    tool_results = await execute_tools_parallel(
        response.choices[0].message.tool_calls
    )

    # Add to conversation
    messages.extend(tool_results)

Advanced Use Cases

Database Integration

tools = [
    {
        "type": "function",
        "function": {
            "name": "query_database",
            "description": "Query the customer database",
            "parameters": {
                "type": "object",
                "properties": {
                    "query": {
                        "type": "string",
                        "description": "SQL query to execute"
                    },
                    "limit": {
                        "type": "integer",
                        "description": "Maximum number of results"
                    }
                },
                "required": ["query"]
            }
        }
    }
]

async def query_database(args):
    # Sanitize and execute SQL safely
    query = sanitize_sql(args["query"])
    limit = args.get("limit", 10)

    results = await db.execute(f"{query} LIMIT {limit}")
    return {"results": results.fetchall()}

API Integration

const apiTools = [
  {
    type: "function",
    function: {
      name: "send_email",
      description: "Send an email to a recipient",
      parameters: {
        type: "object",
        properties: {
          to: { type: "string", description: "Email address" },
          subject: { type: "string", description: "Email subject" },
          body: { type: "string", description: "Email content" },
        },
        required: ["to", "subject", "body"],
      },
    },
  },
  {
    type: "function",
    function: {
      name: "create_calendar_event",
      description: "Create a calendar event",
      parameters: {
        type: "object",
        properties: {
          title: { type: "string" },
          start_time: { type: "string", format: "date-time" },
          duration: { type: "integer", description: "Duration in minutes" },
          attendees: { type: "array", items: { type: "string" } },
        },
        required: ["title", "start_time"],
      },
    },
  },
];

const executeApiTool = async (toolCall) => {
  const { name } = toolCall.function;
  const args = JSON.parse(toolCall.function.arguments);

  switch (name) {
    case "send_email":
      return await emailAPI.send(args);
    case "create_calendar_event":
      return await calendarAPI.createEvent(args);
    default:
      throw new Error(`Unknown API tool: ${name}`);
  }
};

Multi-Steps Workflows

class WorkflowEngine:
    def __init__(self):
        self.tools = {}
        self.conversation = []

    def register_tool(self, name, func, schema):
        self.tools[name] = {"func": func, "schema": schema}

    async def execute_workflow(self, initial_prompt, max_steps=10):
        self.conversation = [{"role": "user", "content": initial_prompt}]

        for step in range(max_steps):
            response = await openai.chat.completions.create(
                model="openai/gpt-4o",
                messages=self.conversation,
                tools=list(self.tools.values()),
                tool_choice="auto"
            )

            self.conversation.append(response.choices[0].message)

            # Check if tools need to be executed
            if response.choices[0].message.tool_calls:
                for tool_call in response.choices[0].message.tool_calls:
                    result = await self.execute_tool(tool_call)
                    self.conversation.append(result)
            else:
                # No tools called, workflow complete
                return response.choices[0].message.content

        return "Workflow exceeded maximum steps"

    async def execute_tool(self, tool_call):
        tool_name = tool_call.function.name
        arguments = json.loads(tool_call.function.arguments)

        if tool_name in self.tools:
            result = await self.tools[tool_name]["func"](arguments)
        else:
            result = {"error": f"Unknown tool: {tool_name}"}

        return {
            "role": "tool",
            "tool_call_id": tool_call.id,
            "content": json.dumps(result)
        }

Error Handling

const safeToolExecution = async (toolCall) => {
  try {
    const args = JSON.parse(toolCall.function.arguments);

    // Validate arguments
    const validation = validateArgs(toolCall.function.name, args);
    if (!validation.valid) {
      return {
        role: "tool",
        tool_call_id: toolCall.id,
        content: JSON.stringify({
          error: "Invalid arguments",
          details: validation.errors,
        }),
      };
    }

    // Execute with timeout
    const result = await Promise.race([
      executeFunction(toolCall.function.name, args),
      new Promise((_, reject) =>
        setTimeout(() => reject(new Error("Tool execution timeout")), 30000),
      ),
    ]);

    return {
      role: "tool",
      tool_call_id: toolCall.id,
      content: JSON.stringify(result),
    };
  } catch (error) {
    console.error(`Tool execution failed: ${error.message}`);

    return {
      role: "tool",
      tool_call_id: toolCall.id,
      content: JSON.stringify({
        error: "Tool execution failed",
        message: error.message,
      }),
    };
  }
};

Best Practices

Tool design

  • Use clear, descriptive function names
  • Provide detailed parameter descriptions
  • Include examples in descriptions
  • Make tools idempotent when possible

Schema design

{
  "type": "object",
  "properties": {
    "location": {
      "type": "string",
      "description": "City and state (e.g., 'San Francisco, CA')",
      "examples": ["New York, NY", "London, UK"]
    },
    "units": {
      "type": "string",
      "enum": ["metric", "imperial"],
      "description": "Temperature unit system",
      "default": "metric"
    }
  },
  "required": ["location"]
}

Security considerations

  • Never expose destructive operations directly
  • Validate all inputs thoroughly
  • Use allowlists for sensitive operations
  • Implement proper authentication
  • Log all tool executions

Troubleshooting

Tool not being called
  • Check tool descriptions are clear
  • Verify parameter schemas are correct
  • Ensure tool_choice is set appropriately
  • Try more explicit prompts
Invalid arguments
  • Validate JSON Schema thoroughly
  • Add parameter examples
  • Check required fields are marked
  • Simplify complex parameter structures
Execution failures
  • Implement proper error handling
  • Add timeout protection
  • Validate inputs before execution
  • Return structured error messages

Limitations

LimitationImpactWorkaround
Tool limitMax ~20 tools per requestGroup related functions
Parameter sizeLarge schemas may failSimplify parameter structure
Execution timeTools block responseUse async patterns
Error propagationFailures can break workflowImplement error recovery
Model differencesVarying tool calling qualityTest across models

Performance Optimization

Tool caching

class CachedToolExecutor {
  constructor() {
    this.cache = new Map();
    this.cacheTTL = 300000; // 5 minutes
  }

  getCacheKey(toolCall) {
    return `${toolCall.function.name}:${toolCall.function.arguments}`;
  }

  async execute(toolCall) {
    const key = this.getCacheKey(toolCall);
    const cached = this.cache.get(key);

    if (cached && Date.now() - cached.timestamp < this.cacheTTL) {
      return cached.result;
    }

    const result = await this.executeFunction(toolCall);
    this.cache.set(key, { result, timestamp: Date.now() });

    return result;
  }
}

Batch operations

# Instead of multiple individual calls
def get_weather_batch(locations):
    return {loc: get_weather(loc) for loc in locations}

# Tool that accepts multiple inputs
{
    "name": "get_weather_batch",
    "parameters": {
        "type": "object",
        "properties": {
            "locations": {
                "type": "array",
                "items": {"type": "string"}
            }
        }
    }
}