File Embedding with Agents

Embedding Files

Multiple types of files can be embedded in an Agent Run.

Images: Via URL (uri) or base64 encoding (bytes)
PDFs: Only supported via base64 encoding (bytes) - URI links are not supported for PDFs
MIME Types: Required - Must specify correct mimeType (e.g., image/jpeg, application/pdf)

❗️
Always verify that your chosen model supports the file types you're using. Vision models are typically required for image processing and PDF processing

Examples

Embedding an Image

curl -X POST https://api.orq.ai/v2/agents/run \
  -H "Authorization: Bearer $ORQ_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
  "key": "image-classifier-1",
  "role": "Image Analyst",
  "description": "Analyzes images and identifies visual content",
  "instructions": "Analyze images and describe what you can see in detail",
  "settings": {
    "max_iterations": 5,
    "max_execution_time": 600,
    "tools": [
      {
        "type": "current_date"
      }
    ]
  },
  "message": {
    "role": "user",
    "parts": [
      {
        "kind": "text",
        "text": "Look at this map image and tell me what cities, regions, and geographical features you can identify."
      },
      {
        "kind": "file",
        "file": {
          "uri": "https://upload.wikimedia.org/wikipedia/commons/7/73/Herman_Moll_A_New_Map_of_Europe_According_to_the_Newest_Observations_1721.JPG",
          "mimeType": "image/jpeg",
          "name": "europe-map.jpg"
        }
      }
    ]
  },
  "model": "openai/gpt-4o",
  "path": "Default/agents"
}'

from orq_ai_sdk import Orq
import os

with Orq(
    api_key=os.getenv("ORQ_API_KEY", ""),
) as orq:

    res = orq.agents.run(request={
        "key": "image-classifier-1",
        "role": "Image Analyst",
        "description": "Analyzes images and identifies visual content",
        "instructions": "Analyze images and describe what you can see in detail",
        "settings": {
            "max_iterations": 5,
            "max_execution_time": 600,
            "tools": [
                {
                    "type": "current_date"
                }
            ]
        },
        "message": {
            "role": "user",
            "parts": [
                {
                    "kind": "text",
                    "text": "Look at this map image and tell me what cities, regions, and geographical features you can identify."
                },
                {
                    "kind": "file",
                    "file": {
                        "uri": "https://upload.wikimedia.org/wikipedia/commons/7/73/Herman_Moll_A_New_Map_of_Europe_According_to_the_Newest_Observations_1721.JPG",
                        "mimeType": "image/jpeg",
                        "name": "europe-map.jpg"
                    }
                }
            ]
        },
        "model": "openai/gpt-4o",
        "path": "Default/agents"
    })

    assert res is not None
    print(res)

import { Orq } from "@orq-ai/node";

const orq = new Orq({
  apiKey: process.env["ORQ_API_KEY"] ?? "",
});

async function run() {
  const result = await orq.agents.run({
    key: "image-classifier-1",
    role: "Image Analyst",
    description: "Analyzes images and identifies visual content",
    instructions: "Analyze images and describe what you can see in detail",
    settings: {
      maxIterations: 5,
      maxExecutionTime: 600,
      tools: [
        {
          type: "current_date"
        }
      ]
    },
    message: {
      role: "user",
      parts: [
        {
          kind: "text",
          text: "Look at this map image and tell me what cities, regions, and geographical features you can identify."
        },
        {
          kind: "file",
          file: {
            uri: "https://upload.wikimedia.org/wikipedia/commons/7/73/Herman_Moll_A_New_Map_of_Europe_According_to_the_Newest_Observations_1721.JPG",
            mimeType: "image/jpeg",
            name: "europe-map.jpg"
          }
        }
      ]
    },
    model: "openai/gpt-4o",
    path: "Default/agents"
  });

  console.log(result);
}

run();

Embedding a PDF

curl -X POST https://api.orq.ai/v2/agents/run \
  -H "Authorization: Bearer $ORQ_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
  "key": "pdf-analyzer-1",
  "role": "Document Analyst",
  "description": "Analyzes PDF documents and extracts information",
  "instructions": "Analyze the provided PDF document and answer questions about its content",
  "settings": {
    "max_iterations": 5,
    "max_execution_time": 600,
    "tools": [
      {
        "type": "current_date"
      }
    ]
  },
  "message": {
    "role": "user",
    "parts": [
      {
        "kind": "text",
        "text": "Please analyze this PDF document and summarize its key points"
      },
      {
        "kind": "file",
        "file": {
          "bytes": "JVBERi0xLjQKJeLjz9MKMyAwIG9iago8PC9MZW5ndGggMTE0L0ZpbHRlci9GbGF0ZURlY29kZT4+CnN0cmVhbQp4nD2OywoCMQxF3/mKu3YRk7ZJk3YpqCOIAzKCH2DSVgp2BtvR/3emDt7lOZyTl0CgVo0KBFhKZRWYClbwBHtwBg7gCM7AX8E3eMGHkMbmdjVdzLfx/XG13Kzn80U8G4+Sny9JyCmJMI25EFFGUs8P/SBJY7+XZElIo36c+72kl6ZJPOglWRgNe8Mw6oZJHqU0HSQvv9vn+wdDDxLsCmVuZHN0cmVhbQplbmRvYmoKMSAwIG9iago8PC9UeXBlL1BhZ2UvTWVkaWFCb3hbMCAwIDYxMiA3OTJdL1Jlc291cmNlczw8L0ZvbnQ8PC9GMSAyIDAgUj4+Pj4vQ29udGVudHMgMyAwIFIvUGFyZW50IDQgMCBSPj4KZW5kb2JqCjIgMCBvYmoKPDwvVHlwZS9Gb250L1N1YnR5cGUvVHlwZTEvQmFzZUZvbnQvSGVsdmV0aWNhPj4KZW5kb2JqCjQgMCBvYmoKPDwvVHlwZS9QYWdlcy9Db3VudCAxL0tpZHNbMSAwIFJdPj4KZW5kb2JqCjUgMCBvYmoKPDwvVHlwZS9DYXRhbG9nL1BhZ2VzIDQgMCBSPj4KZW5kb2JqCjYgMCBvYmoKPDwvUHJvZHVjZXIoU2FtcGxlIFBERikvQ3JlYXRpb25EYXRlKEQ6MjAyNDAxMDEwMDAwMDApPj4KZW5kb2JqCnhyZWYKMCA3CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDE5NyAwMDAwMCBuIAowMDAwMDAwMzA0IDAwMDAwIG4gCjAwMDAwMDAwMTUgMDAwMDAgbiAKMDAwMDAwMDM4NSAwMDAwMCBuIAowMDAwMDAwNDQwIDAwMDAwIG4gCjAwMDAwMDA0ODcgMDAwMDAgbiAKdHJhaWxlcgo8PC9TaXplIDcvUm9vdCA1IDAgUi9JbmZvIDYgMCBSPj4Kc3RhcnR4cmVmCjU3MQolJUVPRgo=",
          "mimeType": "application/pdf",
          "name": "sample-document.pdf"
        }
      }
    ]
  },
  "model": "openai/gpt-4o",
  "path": "Default/agents",
  "memory_stores": [],
  "team_of_agents": []
}'

from orq_ai_sdk import Orq
import os

with Orq(
    api_key=os.getenv("ORQ_API_KEY", ""),
) as orq:

    res = orq.agents.run(request={
        "key": "pdf-analyzer-1",
        "role": "Document Analyst",
        "description": "Analyzes PDF documents and extracts information",
        "instructions": "Analyze the provided PDF document and answer questions about its content",
        "settings": {
            "max_iterations": 5,
            "max_execution_time": 600,
            "tools": [
                {
                    "type": "current_date"
                }
            ]
        },
        "message": {
            "role": "user",
            "parts": [
                {
                    "kind": "text",
                    "text": "Please analyze this PDF document and summarize its key points"
                },
                {
                    "kind": "file",
                    "file": {
                        "bytes": "JVBERi0xLjQKJeLjz9MKMyAwIG9iago8PC9MZW5ndGggMTE0L0ZpbHRlci9GbGF0ZURlY29kZT4+CnN0cmVhbQp4nD2OywoCMQxF3/mKu3YRk7ZJk3YpqCOIAzKCH2DSVgp2BtvR/3emDt7lOZyTl0CgVo0KBFhKZRWYClbwBHtwBg7gCM7AX8E3eMGHkMbmdjVdzLfx/XG13Kzn80U8G4+Sny9JyCmJMI25EFFGUs8P/SBJY7+XZElIo36c+72kl6ZJPOglWRgNe8Mw6oZJHqU0HSQvv9vn+wdDDxLsCmVuZHN0cmVhbQplbmRvYmoKMSAwIG9iago8PC9UeXBlL1BhZ2UvTWVkaWFCb3hbMCAwIDYxMiA3OTJdL1Jlc291cmNlczw8L0ZvbnQ8PC9GMSAyIDAgUj4+Pj4vQ29udGVudHMgMyAwIFIvUGFyZW50IDQgMCBSPj4KZW5kb2JqCjIgMCBvYmoKPDwvVHlwZS9Gb250L1N1YnR5cGUvVHlwZTEvQmFzZUZvbnQvSGVsdmV0aWNhPj4KZW5kb2JqCjQgMCBvYmoKPDwvVHlwZS9QYWdlcy9Db3VudCAxL0tpZHNbMSAwIFJdPj4KZW5kb2JqCjUgMCBvYmoKPDwvVHlwZS9DYXRhbG9nL1BhZ2VzIDQgMCBSPj4KZW5kb2JqCjYgMCBvYmoKPDwvUHJvZHVjZXIoU2FtcGxlIFBERikvQ3JlYXRpb25EYXRlKEQ6MjAyNDAxMDEwMDAwMDApPj4KZW5kb2JqCnhyZWYKMCA3CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDE5NyAwMDAwMCBuIAowMDAwMDAwMzA0IDAwMDAwIG4gCjAwMDAwMDAwMTUgMDAwMDAgbiAKMDAwMDAwMDM4NSAwMDAwMCBuIAowMDAwMDAwNDQwIDAwMDAwIG4gCjAwMDAwMDA0ODcgMDAwMDAgbiAKdHJhaWxlcgo8PC9TaXplIDcvUm9vdCA1IDAgUi9JbmZvIDYgMCBSPj4Kc3RhcnR4cmVmCjU3MQolJUVPRko=",
                        "mimeType": "application/pdf",
                        "name": "sample-document.pdf"
                    }
                }
            ]
        },
        "model": "openai/gpt-4o",
        "path": "Default/agents",
        "memory_stores": [],
        "team_of_agents": []
    })

    assert res is not None
    print(res)

import { Orq } from "@orq-ai/node";

const orq = new Orq({
  apiKey: process.env["ORQ_API_KEY"] ?? "",
});

async function run() {
  const result = await orq.agents.run({
    key: "pdf-analyzer-1",
    role: "Document Analyst",
    description: "Analyzes PDF documents and extracts information",
    instructions: "Analyze the provided PDF document and answer questions about its content",
    settings: {
      maxIterations: 5,
      maxExecutionTime: 600,
      tools: [
        {
          type: "current_date"
        }
      ]
    },
    message: {
      role: "user",
      parts: [
        {
          kind: "text",
          text: "Please analyze this PDF document and summarize its key points"
        },
        {
          kind: "file",
          file: {
            bytes: "JVBERi0xLjQKJeLjz9MKMyAwIG9iago8PC9MZW5ndGggMTE0L0ZpbHRlci9GbGF0ZURlY29kZT4+CnN0cmVhbQp4nD2OywoCMQxF3/mKu3YRk7ZJk3YpqCOIAzKCH2DSVgp2BtvR/3emDt7lOZyTl0CgVo0KBFhKZRWYClbwBHtwBg7gCM7AX8E3eMGHkMbmdjVdzLfx/XG13Kzn80U8G4+Sny9JyCmJMI25EFFGUs8P/SBJY7+XZElIo36c+72kl6ZJPOglWRgNe8Mw6oZJHqU0HSQvv9vn+wdDDxLsCmVuZHN0cmVhbQplbmRvYmoKMSAwIG9iago8PC9UeXBlL1BhZ2UvTWVkaWFCb3hbMCAwIDYxMiA3OTJdL1Jlc291cmNlczw8L0ZvbnQ8PC9GMSAyIDAgUj4+Pj4vQ29udGVudHMgMyAwIFIvUGFyZW50IDQgMCBSPj4KZW5kb2JqCjIgMCBvYmoKPDwvVHlwZS9Gb250L1N1YnR5cGUvVHlwZTEvQmFzZUZvbnQvSGVsdmV0aWNhPj4KZW5kb2JqCjQgMCBvYmoKPDwvVHlwZS9QYWdlcy9Db3VudCAxL0tpZHNbMSAwIFJdPj4KZW5kb2JqCjUgMCBvYmoKPDwvVHlwZS9DYXRhbG9nL1BhZ2VzIDQgMCBSPj4KZW5kb2JqCjYgMCBvYmoKPDwvUHJvZHVjZXIoU2FtcGxlIFBERikvQ3JlYXRpb25EYXRlKEQ6MjAyNDAxMDEwMDAwMDApPj4KZW5kb2JqCnhyZWYKMCA3CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDE5NyAwMDAwMCBuIAowMDAwMDAwMzA0IDAwMDAwIG4gCjAwMDAwMDAwMTUgMDAwMDAgbiAKMDAwMDAwMDM4NSAwMDAwMCBuIAowMDAwMDAwNDQwIDAwMDAwIG4gCjAwMDAwMDA0ODcgMDAwMDAgbiAKdHJhaWxlcgo8PC9TaXplIDcvUm9vdCA1IDAgUi9JbmZvIDYgMCBSPj4Kc3RhcnR4cmVmCjU3MQolJUVPRko=",
            mimeType: "application/pdf",
            name: "sample-document.pdf"
          }
        }
      ]
    },
    model: "openai/gpt-4o",
    path: "Default/agents",
    memoryStores: [],
    teamOfAgents: []
  });

  console.log(result);
}

run();