Router.Chat.Completions
Create a Completion
Creates a model response for the given chat conversation with support for retries, fallbacks, prompts, and variables.from orq_ai_sdk import Orq
import os
with Orq(
api_key=os.getenv("ORQ_API_KEY", ""),
) as orq:
res = orq.router.chat.completions.create(messages=[], model="Model 3", fallbacks=[
{
"model": "openai/gpt-4o-mini",
},
], retry={
"on_codes": [
429,
500,
502,
503,
504,
],
}, cache={
"ttl": 3600,
"type": "exact_match",
}, load_balancer={
"type": "weight_based",
"models": [
{
"model": "openai/gpt-4o",
"weight": 0.7,
},
{
"model": "anthropic/claude-3-5-sonnet",
"weight": 0.3,
},
],
}, timeout={
"call_timeout": 30000,
}, variables={
"customer_name": "John Smith",
"product_name": "Premium Plan",
}, stream=False)
with res as event_stream:
for event in event_stream:
# handle event
print(event, flush=True)
import { Orq } from "@orq-ai/node";
const orq = new Orq({
apiKey: process.env["ORQ_API_KEY"] ?? "",
});
async function run() {
const result = await orq.router.chat.completions.create({
messages: [],
model: "Model 3",
fallbacks: [
{
model: "openai/gpt-4o-mini",
},
],
retry: {
onCodes: [
429,
500,
502,
503,
504,
],
},
cache: {
ttl: 3600,
type: "exact_match",
},
loadBalancer: {
type: "weight_based",
models: [
{
model: "openai/gpt-4o",
weight: 0.7,
},
{
model: "anthropic/claude-3-5-sonnet",
weight: 0.3,
},
],
},
timeout: {
callTimeout: 30000,
},
variables: {
"customer_name": "John Smith",
"product_name": "Premium Plan",
},
});
console.log(result);
}
run();
Show Parameters
Show Parameters
{
"messages": List[Union[CreateChatCompletionMessagesSystemMessage, CreateChatCompletionMessagesDeveloperMessage, CreateChatCompletionMessagesUserMessage, CreateChatCompletionMessagesAssistantMessage, CreateChatCompletionMessagesToolMessage]], # required
"model": str, # required
"metadata": Dict[str, str], # optional
"name": Optional[str],
"audio": { # optional
"voice": Literal["alloy", "echo", "fable", "onyx", "nova", "shimmer"], # required
"format_": Literal["wav", "mp3", "flac", "opus", "pcm16"], # required
},
"frequency_penalty": Optional[float],
"max_tokens": Optional[int],
"max_completion_tokens": Optional[int],
"logprobs": Optional[bool],
"top_logprobs": Optional[int],
"n": Optional[int],
"presence_penalty": Optional[float],
"response_format": Union[CreateChatCompletionResponseFormatText, CreateChatCompletionResponseFormatJSONObject, CreateChatCompletionResponseFormatJSONSchema], # optional
"reasoning_effort": Optional[Literal["none", "minimal", "low", "medium", "high", "xhigh"]],
"verbosity": Optional[str],
"seed": Optional[float],
"stop": Union[str, List[str]], # optional
"stream_options": { # optional
"include_usage": Optional[bool],
},
"thinking": Union[ThinkingConfigDisabledSchema, ThinkingConfigEnabledSchema, ThinkingConfigAdaptiveSchema], # optional
"temperature": Optional[float],
"top_p": Optional[float],
"top_k": Optional[float],
"tools": [{ # optional
"type": Optional[Literal["function"]],
"function": { # required
"name": str, # required
"description": Optional[str],
"parameters": { # optional
"type": Literal["object"], # required
"properties": Dict[str, Any], # required
"required": List[str], # optional
"additional_properties": Optional[bool],
},
"strict": Optional[bool],
},
}],
"tool_choice": Union[CreateChatCompletionToolChoice1, CreateChatCompletionToolChoice2], # optional
"parallel_tool_calls": Optional[bool],
"modalities": List[Literal["text", "audio"]], # optional
"guardrails": [{ # optional
"id": Union[CreateChatCompletionID1, str], # required
"execute_on": Literal["input", "output"], # required
}],
"plugins": List[Union[PIIRedactionPluginAuto, PIIRedactionPluginEn, PIIRedactionPluginNl, ResponseHealingPlugin, TraceScrubbingPlugin]], # optional
"fallbacks": [{ # optional
"model": str, # required
}],
"retry": { # optional
"count": Optional[float],
"on_codes": List[float], # optional
},
"cache": { # optional
"ttl": Optional[float],
"type": Literal["exact_match"], # required
},
"load_balancer": Union[CreateChatCompletionLoadBalancer1], # optional
"timeout": { # optional
"call_timeout": float, # required
},
"variables": Dict[str, Any], # optional
"cache_control": { # optional
"type": Literal["ephemeral"], # required
"ttl": Optional[Literal["5m", "1h"]],
},
"prompt_cache_key": Optional[str],
"orq": { # optional
"name": Optional[str],
"retry": { # optional
"count": Optional[float],
"on_codes": List[float], # optional
},
"fallbacks": [{ # optional
"model": str, # required
}],
"prompt": { # optional
"id": str, # required
"version": Literal["latest"], # required
},
"identity": { # optional
"id": str, # required
"display_name": Optional[str],
"email": Optional[str],
"metadata": List[Dict[str, Any]], # optional
"logo_url": Optional[str],
"tags": List[str], # optional
},
"contact": { # optional
"id": str, # required
"display_name": Optional[str],
"email": Optional[str],
"metadata": List[Dict[str, Any]], # optional
"logo_url": Optional[str],
"tags": List[str], # optional
},
"thread": { # optional
"id": str, # required
"tags": List[str], # optional
},
"inputs": Union[Dict[str, Any], List[Inputs2]], # optional
"cache": { # optional
"ttl": Optional[float],
"type": Literal["exact_match"], # required
},
"knowledge_bases": [{ # optional
"top_k": Optional[int],
"threshold": Optional[float],
"search_type": Optional[Literal["vector_search", "keyword_search", "hybrid_search"]],
"filter_by": Union[Dict[str, CreateChatCompletionFilterBy1], CreateChatCompletionFilterByAnd, CreateChatCompletionFilterByOr], # optional
"search_options": { # optional
"include_vectors": Optional[bool],
"include_metadata": Optional[bool],
"include_scores": Optional[bool],
},
"rerank_config": { # optional
"model": str, # required
"threshold": Optional[float],
"top_k": Optional[int],
},
"agentic_rag_config": { # optional
"model": str, # required
},
"knowledge_id": str, # required
"query": Optional[str],
}],
"load_balancer": Union[CreateChatCompletionLoadBalancerRouterChatCompletions1], # optional
"timeout": { # optional
"call_timeout": float, # required
},
},
"stream": Optional[bool],
}
{
messages: (CreateChatCompletionMessagesSystemMessage | CreateChatCompletionMessagesDeveloperMessage | CreateChatCompletionMessagesUserMessage | CreateChatCompletionMessagesAssistantMessage | CreateChatCompletionMessagesToolMessage)[]; // required
model: string; // required
metadata?: Record<string, string>;
name?: string;
audio?: {
voice: "alloy" | "echo" | "fable" | "onyx" | "nova" | "shimmer"; // required
format: "wav" | "mp3" | "flac" | "opus" | "pcm16"; // required
};
frequencyPenalty?: number;
maxTokens?: number;
maxCompletionTokens?: number;
logprobs?: boolean;
topLogprobs?: number;
n?: number;
presencePenalty?: number;
responseFormat?: CreateChatCompletionResponseFormatText | CreateChatCompletionResponseFormatJSONObject | CreateChatCompletionResponseFormatJSONSchema;
reasoningEffort?: "none" | "minimal" | "low" | "medium" | "high" | "xhigh";
verbosity?: string;
seed?: number;
stop?: string | string[];
streamOptions?: {
includeUsage?: boolean;
};
thinking?: ThinkingConfigDisabledSchema | ThinkingConfigEnabledSchema | ThinkingConfigAdaptiveSchema;
temperature?: number;
topP?: number;
topK?: number;
tools?: {
type?: "function";
function: { // required
name: string; // required
description?: string;
parameters?: {
type: "object"; // required
properties: Record<string, any>; // required
required?: string[];
};
strict?: boolean;
};
}[];
toolChoice?: CreateChatCompletionToolChoice1 | CreateChatCompletionToolChoice2;
parallelToolCalls?: boolean;
modalities?: ("text" | "audio")[];
guardrails?: {
id: CreateChatCompletionId1 | string; // required
executeOn: "input" | "output"; // required
}[];
plugins?: (PIIRedactionPluginAuto | PIIRedactionPluginEn | PIIRedactionPluginNl | ResponseHealingPlugin | TraceScrubbingPlugin)[];
fallbacks?: {
model: string; // required
}[];
retry?: {
count?: number;
onCodes?: number[];
};
cache?: {
ttl?: number;
type: "exact_match"; // required
};
loadBalancer?: CreateChatCompletionLoadBalancer1;
timeout?: {
callTimeout: number; // required
};
variables?: Record<string, any>;
cacheControl?: {
type: "ephemeral"; // required
ttl?: "5m" | "1h";
};
promptCacheKey?: string;
orq?: {
name?: string;
retry?: {
count?: number;
onCodes?: number[];
};
fallbacks?: {
model: string; // required
}[];
prompt?: {
id: string; // required
version: "latest"; // required
};
identity?: {
id: string; // required
displayName?: string;
email?: string;
metadata?: Record<string, any>[];
logoUrl?: string;
tags?: string[];
};
contact?: {
id: string; // required
displayName?: string;
email?: string;
metadata?: Record<string, any>[];
logoUrl?: string;
tags?: string[];
};
thread?: {
id: string; // required
tags?: string[];
};
inputs?: { [k: string]: any } | Inputs2[];
cache?: {
ttl?: number;
type: "exact_match"; // required
};
knowledgeBases?: {
topK?: number;
threshold?: number;
searchType?: "vector_search" | "keyword_search" | "hybrid_search";
filterBy?: { [k: string]: CreateChatCompletionFilterBy1 } | CreateChatCompletionFilterByAnd | CreateChatCompletionFilterByOr;
searchOptions?: {
includeVectors?: boolean;
includeMetadata?: boolean;
includeScores?: boolean;
};
rerankConfig?: {
model: string; // required
threshold?: number;
topK?: number;
};
agenticRagConfig?: {
model: string; // required
};
knowledgeId: string; // required
query?: string;
}[];
loadBalancer?: CreateChatCompletionLoadBalancerRouterChatCompletions1;
timeout?: {
callTimeout: number; // required
};
};
stream?: boolean;
}
Show Response
Show Response
{
"id": str,
"choices": [{
"finish_reason": Nullable[CreateChatCompletionFinishReason],
"index": Optional[float],
"message": {
"content": Optional[str],
"refusal": Optional[str],
"tool_calls": [{ # optional
"index": Optional[float],
"id": Optional[str],
"type": Optional[Literal["function"]],
"function": { # optional
"name": Optional[str],
"arguments": Optional[str],
},
"thought_signature": Optional[str],
}],
"role": Optional[Literal["assistant"]],
"reasoning": Optional[str],
"reasoning_signature": Optional[str],
"redacted_reasoning": Optional[str],
"audio": { # optional
"id": str,
"expires_at": int,
"data": str,
"transcript": str,
},
},
"logprobs": { # optional
"content": [{
"token": str,
"logprob": float,
"bytes_": List[float],
"top_logprobs": [{
"token": str,
"logprob": float,
"bytes_": List[float],
}],
}],
"refusal": [{
"token": str,
"logprob": float,
"bytes_": List[float],
"top_logprobs": [{
"token": str,
"logprob": float,
"bytes_": List[float],
}],
}],
},
}],
"created": float,
"model": str,
"system_fingerprint": Optional[str],
"usage": { # optional
"completion_tokens": Optional[float],
"prompt_tokens": Optional[float],
"total_tokens": Optional[float],
"prompt_tokens_details": { # optional
"cached_tokens": Optional[int],
"cache_creation_tokens": Optional[int],
"audio_tokens": Optional[int],
},
"completion_tokens_details": { # optional
"reasoning_tokens": Optional[float],
"accepted_prediction_tokens": Optional[float],
"rejected_prediction_tokens": Optional[float],
"audio_tokens": Optional[int],
},
},
"object": Literal["chat.completion"],
}
{
id: string;
choices: {
finishReason: "stop" | "length" | "tool_calls" | "content_filter" | "function_call";
index?: number;
message: {
content?: string;
refusal?: string;
toolCalls?: {
index?: number;
id?: string;
type?: "function";
function?: {
name?: string;
arguments?: string;
};
thoughtSignature?: string;
}[];
role?: "assistant";
reasoning?: string;
reasoningSignature?: string;
redactedReasoning?: string;
audio?: {
id: string;
expiresAt: number;
data: string;
transcript: string;
};
};
logprobs?: {
content: {
token: string;
logprob: number;
bytes: number[];
topLogprobs: {
token: string;
logprob: number;
bytes: number[];
}[];
}[];
refusal: {
token: string;
logprob: number;
bytes: number[];
topLogprobs: {
token: string;
logprob: number;
bytes: number[];
}[];
}[];
};
}[];
created: number;
model: string;
systemFingerprint?: string;
usage?: {
completionTokens?: number;
promptTokens?: number;
totalTokens?: number;
promptTokensDetails?: {
cachedTokens?: number;
cacheCreationTokens?: number;
audioTokens?: number;
};
completionTokensDetails?: {
reasoningTokens?: number;
acceptedPredictionTokens?: number;
rejectedPredictionTokens?: number;
audioTokens?: number;
};
};
object: "chat.completion";
}