Evals SDK Reference

Evals

List Evals

Get all Evaluators

from orq_ai_sdk import Orq
import os

with Orq(
    api_key=os.getenv("ORQ_API_KEY", ""),
) as orq:

    res = orq.evals.all(limit=10)

    # Handle response
    print(res)

import { Orq } from "@orq-ai/node";

const orq = new Orq({
  apiKey: process.env["ORQ_API_KEY"] ?? "",
});

async function run() {
  const result = await orq.evals.all({});

  console.log(result);
}

run();

Show Parameters

{
    "limit": Optional[int],
    "starting_after": Optional[str],
    "ending_before": Optional[str],
    "search": Optional[str],
    "sort": Optional[Literal["asc", "desc"]],
    "project_id": Optional[str],
}

{
  limit?: number;
  startingAfter?: string;
  endingBefore?: string;
  search?: string;
  sort?: "asc" | "desc";
  projectId?: string;
}

Show Response

{
    "object": Literal["list"],
    "data": List[Union[EvaluatorResponseLlm, EvaluatorResponseJSONSchema, EvaluatorResponseHTTP, EvaluatorResponsePython, EvaluatorResponseFunction, EvaluatorResponseRagas, EvaluatorResponseTypescript]],
    "has_more": bool,
}

{
  object: "list";
  data: (EvaluatorResponseLlm | EvaluatorResponseJsonSchema | EvaluatorResponseHttp | EvaluatorResponsePython | EvaluatorResponseFunction | EvaluatorResponseRagas | EvaluatorResponseTypescript)[];
  hasMore: boolean;
}

Create an Eval

Create an Evaluator

from orq_ai_sdk import Orq
import os

with Orq(
    api_key=os.getenv("ORQ_API_KEY", ""),
) as orq:

    res = orq.evals.create(request={
        "code": "<value>",
        "type": "python_eval",
        "path": "Default",
        "description": "",
        "key": "<key>",
    })

    # Handle response
    print(res)

import { Orq } from "@orq-ai/node";

const orq = new Orq({
  apiKey: process.env["ORQ_API_KEY"] ?? "",
});

async function run() {
  const result = await orq.evals.create({
    code: "<value>",
    type: "python_eval",
    path: "Default",
    description: "",
    key: "<key>",
  });

  console.log(result);
}

run();

Show Parameters

{
    "guardrail_config": Optional[Any],
    "output_type": Optional[Literal["boolean", "categorical", "number", "string"]],
    "type": Literal["llm_eval"],  # required
    "repetitions": Optional[int],
    "prompt": str,  # required
    "categories": List[str],  # optional
    "categorical_labels": [{  # optional
        "value": str,  # required
        "description": Optional[str],
    }],
    "dataset_id": Optional[str],
    "path": Optional[str],
    "project_id": Optional[str],
    "description": Optional[str],
    "key": str,  # required
    "mode": Literal["single"],  # required
    "model": str,  # required
}

{
  guardrailConfig?: any;
  outputType?: "boolean" | "categorical" | "number" | "string";
  type: "llm_eval";  // required
  repetitions?: number;
  prompt: string;  // required
  categories?: string[];
  categoricalLabels?: {
    value: string;  // required
    description?: string;
  }[];
  datasetId?: string;
  path?: string;
  projectId?: string;
  description?: string;
  key: string;  // required
  mode: "single";  // required
  model: string;  // required
}

Show Response

{
    "id": str,
    "description": str,
    "created": Optional[str],
    "updated": Optional[str],
    "updated_by_id": Optional[str],
    "project_id": Optional[str],
    "guardrail_config": Optional[Any],
    "type": Literal["llm_eval"],
    "repetitions": Optional[int],
    "prompt": str,
    "categories": List[str],  # optional
    "categorical_labels": [{  # optional
        "value": str,
        "description": Optional[str],
    }],
    "dataset_id": Optional[str],
    "key": str,
    "mode": Literal["single", "jury"],
    "model": Optional[str],
    "jury": {  # optional
        "judges": [{
            "model": str,
            "retry": {  # optional
                "count": Optional[int],
                "on_codes": List[int],  # optional
            },
            "fallbacks": [{  # optional
                "model": str,
            }],
        }],
        "replacement_judges": [{  # optional
            "model": str,
            "retry": {  # optional
                "count": Optional[int],
                "on_codes": List[int],  # optional
            },
            "fallbacks": [{  # optional
                "model": str,
            }],
        }],
        "min_successful_judges": Optional[int],
        "tie_value": Optional[Literal["Tie"]],
    },
}

{
  id: string;
  description: string;
  created?: string;
  updated?: string;
  updatedById?: string;
  projectId?: string;
  guardrailConfig?: any;
  type: "llm_eval";
  repetitions?: number;
  prompt: string;
  categories?: string[];
  categoricalLabels?: {
    value: string;
    description?: string;
  }[];
  datasetId?: string;
  key: string;
  mode: "single" | "jury";
  model?: string;
  jury?: {
    judges: {
      model: string;
      retry?: {
        count?: number;
        onCodes?: number[];
      };
      fallbacks?: {
        model: string;
      }[];
    }[];
    replacementJudges?: {
      model: string;
      retry?: {
        count?: number;
        onCodes?: number[];
      };
      fallbacks?: {
        model: string;
      }[];
    }[];
    minSuccessfulJudges?: number;
    tieValue?: "Tie";
  };
}

Retrieve an Eval

Retrieve a single evaluator by its unique identifier. Returns the evaluator exactly as stored, including its type-specific configuration: prompt and model for LLM evaluators, source code for Python and TypeScript evaluators, the JSON Schema for schema evaluators, and so on. Use this when you already know the evaluator id (for example to refresh the state of a resource you manage declaratively). To discover evaluator ids, list them with GET /v2/evaluators. This endpoint returns the stored record, which carries more detail than the representation GET /v2/evaluators returns: display_name rather than key, model as an object rather than a provider-qualified string, plus the owner, domain_id, metadata, enabled and output_type fields.

from orq_ai_sdk import Orq
import os

with Orq(
    api_key=os.getenv("ORQ_API_KEY", ""),
) as orq:

    res = orq.evals.get(id="01JMDPA3QW5C1V0NJ1PW34T4E5")

    # Handle response
    print(res)

import { Orq } from "@orq-ai/node";

const orq = new Orq({
  apiKey: process.env["ORQ_API_KEY"] ?? "",
});

async function run() {
  const result = await orq.evals.get({
    id: "01JMDPA3QW5C1V0NJ1PW34T4E5",
  });

  console.log(result);
}

run();

Show Parameters

{
    "id": str,  # required
}

{
  id: string;  // required
}

Show Response

{
    "enabled": Optional[bool],
    "metadata": {
        "required_model_with_tools_support": Optional[bool],
        "required_retrieval_context": Optional[bool],
        "required_expected_output": Optional[bool],
        "supported_on_input_type": Optional[bool],
        "supported_on_output_type": Optional[bool],
        "support_use_as_guardrail": Optional[bool],
    },
    "id": str,
    "display_name": str,
    "description": str,
    "owner": str,
    "created": Optional[str],
    "updated": Optional[str],
    "created_by_id": Optional[str],
    "updated_by_id": Optional[str],
    "domain_id": str,
    "project_id": Optional[str],
    "guardrail_config": Optional[Any],
    "output_type": Optional[Literal["boolean", "categorical", "number", "string"]],
    "type": Literal["llm_eval"],
    "mode": Optional[Literal["single", "jury"]],
    "repetitions": Optional[int],
    "model": {  # optional
        "id": str,
        "integration_id": Optional[str],
        "model_parameters": {  # optional
            "temperature": Optional[float],
            "max_tokens": Optional[float],
            "top_k": Optional[float],
            "top_p": Optional[float],
            "frequency_penalty": Optional[float],
            "presence_penalty": Optional[float],
            "reasoning_effort": Optional[str],
            "budget_tokens": Optional[float],
        },
    },
    "jury": {  # optional
        "judges": [{
            "model": {
                "id": str,
                "integration_id": Optional[str],
                "model_parameters": {  # optional
                    "temperature": Optional[float],
                    "max_tokens": Optional[float],
                    "top_k": Optional[float],
                    "top_p": Optional[float],
                    "frequency_penalty": Optional[float],
                    "presence_penalty": Optional[float],
                    "reasoning_effort": Optional[str],
                    "budget_tokens": Optional[float],
                },
            },
            "retry": {  # optional
                "count": Optional[int],
                "on_codes": List[int],  # optional
            },
            "fallbacks": [{  # optional
                "model": {
                    "id": str,
                    "integration_id": Optional[str],
                    "model_parameters": {  # optional
                        "temperature": Optional[float],
                        "max_tokens": Optional[float],
                        "top_k": Optional[float],
                        "top_p": Optional[float],
                        "frequency_penalty": Optional[float],
                        "presence_penalty": Optional[float],
                        "reasoning_effort": Optional[str],
                        "budget_tokens": Optional[float],
                    },
                },
            }],
        }],
        "replacement_judges": [{  # optional
            "model": {
                "id": str,
                "integration_id": Optional[str],
                "model_parameters": {  # optional
                    "temperature": Optional[float],
                    "max_tokens": Optional[float],
                    "top_k": Optional[float],
                    "top_p": Optional[float],
                    "frequency_penalty": Optional[float],
                    "presence_penalty": Optional[float],
                    "reasoning_effort": Optional[str],
                    "budget_tokens": Optional[float],
                },
            },
            "retry": {  # optional
                "count": Optional[int],
                "on_codes": List[int],  # optional
            },
            "fallbacks": [{  # optional
                "model": {
                    "id": str,
                    "integration_id": Optional[str],
                    "model_parameters": {  # optional
                        "temperature": Optional[float],
                        "max_tokens": Optional[float],
                        "top_k": Optional[float],
                        "top_p": Optional[float],
                        "frequency_penalty": Optional[float],
                        "presence_penalty": Optional[float],
                        "reasoning_effort": Optional[str],
                        "budget_tokens": Optional[float],
                    },
                },
            }],
        }],
        "min_successful_judges": Optional[int],
        "tie_value": Optional[Literal["Tie"]],
    },
    "prompt": str,
    "categories": List[str],  # optional
    "categorical_labels": [{  # optional
        "value": str,
        "description": Optional[str],
    }],
    "dataset_id": Optional[str],
}

{
  enabled?: boolean;
  metadata: {
    requiredModelWithToolsSupport?: boolean;
    requiredRetrievalContext?: boolean;
    requiredExpectedOutput?: boolean;
    supportedOnInputType?: boolean;
    supportedOnOutputType?: boolean;
    supportUseAsGuardrail?: boolean;
  };
  id: string;
  displayName: string;
  description: string;
  owner: string;
  created?: string;
  updated?: string;
  createdById?: string;
  updatedById?: string;
  domainId: string;
  projectId?: string;
  guardrailConfig?: any;
  outputType?: "boolean" | "categorical" | "number" | "string";
  type: "llm_eval";
  mode?: "single" | "jury";
  repetitions?: number;
  model?: {
    id: string;
    integrationId?: string;
    modelParameters?: {
      temperature?: number;
      maxTokens?: number;
      topK?: number;
      topP?: number;
      frequencyPenalty?: number;
      presencePenalty?: number;
      reasoningEffort?: string;
      budgetTokens?: number;
    };
  };
  jury?: {
    judges: {
      model: {
        id: string;
        integrationId?: string;
        modelParameters?: {
          temperature?: number;
          maxTokens?: number;
          topK?: number;
          topP?: number;
          frequencyPenalty?: number;
          presencePenalty?: number;
          reasoningEffort?: string;
          budgetTokens?: number;
        };
      };
      retry?: {
        count?: number;
        onCodes?: number[];
      };
      fallbacks?: {
        model: {
          id: string;
          integrationId?: string;
          modelParameters?: {
            temperature?: number;
            maxTokens?: number;
            topK?: number;
            topP?: number;
            frequencyPenalty?: number;
            presencePenalty?: number;
            reasoningEffort?: string;
            budgetTokens?: number;
          };
        };
      }[];
    }[];
    replacementJudges?: {
      model: {
        id: string;
        integrationId?: string;
        modelParameters?: {
          temperature?: number;
          maxTokens?: number;
          topK?: number;
          topP?: number;
          frequencyPenalty?: number;
          presencePenalty?: number;
          reasoningEffort?: string;
          budgetTokens?: number;
        };
      };
      retry?: {
        count?: number;
        onCodes?: number[];
      };
      fallbacks?: {
        model: {
          id: string;
          integrationId?: string;
          modelParameters?: {
            temperature?: number;
            maxTokens?: number;
            topK?: number;
            topP?: number;
            frequencyPenalty?: number;
            presencePenalty?: number;
            reasoningEffort?: string;
            budgetTokens?: number;
          };
        };
      }[];
    }[];
    minSuccessfulJudges?: number;
    tieValue?: "Tie";
  };
  prompt: string;
  categories?: string[];
  categoricalLabels?: {
    value: string;
    description?: string;
  }[];
  datasetId?: string;
}

Update an Eval

Update an Evaluator

from orq_ai_sdk import Orq
import os

with Orq(
    api_key=os.getenv("ORQ_API_KEY", ""),
) as orq:

    res = orq.evals.update(id="<id>", path="Default", project_id="01JMDPA3QW5C1V0NJ1PW34T4E5")

    # Handle response
    print(res)

import { Orq } from "@orq-ai/node";

const orq = new Orq({
  apiKey: process.env["ORQ_API_KEY"] ?? "",
});

async function run() {
  const result = await orq.evals.update({
    id: "<id>",
    requestBody: {
      path: "Default",
      projectId: "01JMDPA3QW5C1V0NJ1PW34T4E5",
    },
  });

  console.log(result);
}

run();

Show Parameters

{
    "id": str,  # required
    "type": Optional[str],
    "path": Optional[str],
    "project_id": Optional[str],
    "key": Optional[str],
    "description": Optional[str],
    "prompt": Optional[str],
    "output_type": Optional[str],
    "categories": List[str],  # optional
    "categorical_labels": [{  # optional
        "value": str,  # required
        "description": Optional[str],
    }],
    "dataset_id": Optional[str],
    "repetitions": Optional[float],
    "mode": Optional[Literal["single", "jury"]],
    "model": Optional[str],
    "jury": {  # optional
        "judges": [{  # required
            "model": str,  # required
            "retry": {  # optional
                "count": Optional[int],
                "on_codes": List[int],  # optional
            },
            "fallbacks": [{  # optional
                "model": str,  # required
            }],
        }],
        "replacement_judges": [{  # optional
            "model": str,  # required
            "retry": {  # optional
                "count": Optional[int],
                "on_codes": List[int],  # optional
            },
            "fallbacks": [{  # optional
                "model": str,  # required
            }],
        }],
        "min_successful_judges": Optional[int],
        "tie_value": Optional[Literal["Tie"]],
    },
    "schema_": Optional[str],
    "url": Optional[str],
    "method": Optional[str],
    "headers": Dict[str, str],  # optional
    "payload": Dict[str, Any],  # optional
    "code": Optional[str],
    "guardrail_config": Optional[Any],
    "version_increment": Optional[Literal["major", "minor", "patch"]],
    "version_description": Optional[str],
}

{
  id: string;  // required
  requestBody?: {
    type?: string;
    path?: string;
    projectId?: string;
    key?: string;
    description?: string;
    prompt?: string;
    outputType?: string;
    categories?: string[];
    categoricalLabels?: {
      value: string;  // required
      description?: string;
    }[];
    datasetId?: string;
    repetitions?: number;
    mode?: "single" | "jury";
    model?: string;
    jury?: {
      judges: {  // required
        model: string;  // required
        retry?: {
          count?: number;
          onCodes?: number[];
        };
        fallbacks?: {
          model: string;  // required
        }[];
      }[];
      replacementJudges?: {
        model: string;  // required
        retry?: {
          count?: number;
          onCodes?: number[];
        };
        fallbacks?: {
          model: string;  // required
        }[];
      }[];
      minSuccessfulJudges?: number;
      tieValue?: "Tie";
    };
    schema?: string;
    url?: string;
    method?: string;
    headers?: Record<string, string>;
    payload?: Record<string, any>;
    code?: string;
    guardrailConfig?: any;
    versionIncrement?: "major" | "minor" | "patch";
    versionDescription?: string;
  };
}

Show Response

{
    "id": str,
    "description": str,
    "created": Optional[str],
    "updated": Optional[str],
    "updated_by_id": Optional[str],
    "project_id": Optional[str],
    "guardrail_config": Optional[Any],
    "type": Literal["llm_eval"],
    "repetitions": Optional[int],
    "prompt": str,
    "categories": List[str],  # optional
    "categorical_labels": [{  # optional
        "value": str,
        "description": Optional[str],
    }],
    "dataset_id": Optional[str],
    "key": str,
    "mode": Literal["single", "jury"],
    "model": Optional[str],
    "jury": {  # optional
        "judges": [{
            "model": str,
            "retry": {  # optional
                "count": Optional[int],
                "on_codes": List[int],  # optional
            },
            "fallbacks": [{  # optional
                "model": str,
            }],
        }],
        "replacement_judges": [{  # optional
            "model": str,
            "retry": {  # optional
                "count": Optional[int],
                "on_codes": List[int],  # optional
            },
            "fallbacks": [{  # optional
                "model": str,
            }],
        }],
        "min_successful_judges": Optional[int],
        "tie_value": Optional[Literal["Tie"]],
    },
}

{
  id: string;
  description: string;
  created?: string;
  updated?: string;
  updatedById?: string;
  projectId?: string;
  guardrailConfig?: any;
  type: "llm_eval";
  repetitions?: number;
  prompt: string;
  categories?: string[];
  categoricalLabels?: {
    value: string;
    description?: string;
  }[];
  datasetId?: string;
  key: string;
  mode: "single" | "jury";
  model?: string;
  jury?: {
    judges: {
      model: string;
      retry?: {
        count?: number;
        onCodes?: number[];
      };
      fallbacks?: {
        model: string;
      }[];
    }[];
    replacementJudges?: {
      model: string;
      retry?: {
        count?: number;
        onCodes?: number[];
      };
      fallbacks?: {
        model: string;
      }[];
    }[];
    minSuccessfulJudges?: number;
    tieValue?: "Tie";
  };
}

Delete an Eval

Delete an Evaluator

from orq_ai_sdk import Orq
import os

with Orq(
    api_key=os.getenv("ORQ_API_KEY", ""),
) as orq:

    orq.evals.delete(id="<id>")

    # Use the SDK ...

import { Orq } from "@orq-ai/node";

const orq = new Orq({
  apiKey: process.env["ORQ_API_KEY"] ?? "",
});

async function run() {
  await orq.evals.delete({
    id: "<id>",
  });

}

run();

Show Parameters

{
    "id": str,  # required
}

{
  id: string;  // required
}

Invoke an Eval

Invoke a Custom Evaluator

from orq_ai_sdk import Orq
import os

with Orq(
    api_key=os.getenv("ORQ_API_KEY", ""),
) as orq:

    res = orq.evals.invoke(id="<id>", messages=[
        {
            "role": "tool",
            "content": [],
        },
    ], variables={
        "locale": "en",
        "tags": [
            "alpha",
            "omega",
        ],
        "profile": {
            "tier": "gold",
            "active": True,
        },
    })

    # Handle response
    print(res)

import { Orq } from "@orq-ai/node";

const orq = new Orq({
  apiKey: process.env["ORQ_API_KEY"] ?? "",
});

async function run() {
  const result = await orq.evals.invoke({
    id: "<id>",
    requestBody: {
      messages: [
        {
          role: "tool",
          content: [],
        },
      ],
      variables: {
        "locale": "en",
        "tags": [
          "alpha",
          "omega",
        ],
        "profile": {
          "tier": "gold",
          "active": true,
        },
      },
    },
  });

  console.log(result);
}

run();

Show Parameters

{
    "id": str,  # required
    "query": Optional[str],
    "output": Optional[str],
    "reference": Optional[str],
    "retrievals": List[str],  # optional
    "messages": [{  # optional
        "role": Literal["system", "developer", "assistant", "user", "exception", "tool", "prompt", "correction", "expected_output"],  # required
        "content": Union[str, List[InvokeEvalContent2]],  # required
        "tool_calls": [{  # optional
            "id": Optional[str],
            "index": Optional[float],
            "type": Literal["function"],  # required
            "function": {  # required
                "name": str,  # required
                "arguments": str,  # required
            },
        }],
        "tool_call_id": Optional[str],
    }],
    "model": Optional[str],
    "variables": Union[str, float, bool, List[Any], Dict[str, Nullable[EvaluatorVariableValue]]],  # optional
}

{
  id: string;  // required
  requestBody?: {
    query?: string;
    output?: string;
    reference?: string;
    retrievals?: string[];
    messages?: {
      role: "system" | "developer" | "assistant" | "user" | "exception" | "tool" | "prompt" | "correction" | "expected_output";  // required
      content: string | Two[];  // required
      toolCalls?: {
        id?: string;
        index?: number;
        type: "function";  // required
        function: {  // required
          name: string;  // required
          arguments: string;  // required
        };
      }[];
      toolCallId?: string;
    }[];
    model?: string;
    variables?: Record<string, string | number | boolean | any[] | { [k: string]: EvaluatorVariableValue }>;
  };
}

Show Response

{
    "type": Literal["string"],
    "original_value": Optional[str],
    "value": Optional[str],
    "trace_id": Optional[str],
    "span_id": Optional[str],
}

{
  type: "string";
  originalValue?: string;
  value?: string;
  traceId?: string;
  spanId?: string;
}

List Evaluator Versions

Returns version history for a specific evaluator

from orq_ai_sdk import Orq
import os

with Orq(
    api_key=os.getenv("ORQ_API_KEY", ""),
) as orq:

    res = orq.evals.list_versions(id="<id>", limit=10)

    # Handle response
    print(res)

import { Orq } from "@orq-ai/node";

const orq = new Orq({
  apiKey: process.env["ORQ_API_KEY"] ?? "",
});

async function run() {
  const result = await orq.evals.listVersions({
    id: "<id>",
  });

  console.log(result);
}

run();

Show Parameters

{
    "id": str,  # required
    "limit": Optional[int],
    "starting_after": Optional[str],
    "ending_before": Optional[str],
}

{
  id: string;  // required
  limit?: number;
  startingAfter?: string;
  endingBefore?: string;
}

Show Response

{
    "object": Literal["list"],
    "data": [{
        "id": str,
        "created_at": str,
        "updated_at": str,
        "created_by_id": Optional[str],
        "updated_by_id": Optional[str],
        "version": str,
        "description": Optional[str],
        "checksum": str,
        "entity_type": str,
        "entity_id": str,
        "data": Dict[str, Any],
        "workspace_id": str,
    }],
    "has_more": bool,
}

{
  object: "list";
  data: {
    id: string;
    createdAt: string;
    updatedAt: string;
    createdById?: string;
    updatedById?: string;
    version: string;
    description?: string;
    checksum: string;
    entityType: string;
    entityId: string;
    data: Record<string, any>;
    workspaceId: string;
  }[];
  hasMore: boolean;
}

Orq SDK

Terminal & Agents

Terraform

Platform APIs

Management APIs

Evals

List Evals

Create an Eval

Retrieve an Eval

Update an Eval

Delete an Eval

Invoke an Eval

List Evaluator Versions

​Evals

​List Evals

​Create an Eval

​Retrieve an Eval

​Update an Eval

​Delete an Eval

​Invoke an Eval

​List Evaluator Versions

Evals

List Evals

Create an Eval

Retrieve an Eval

Update an Eval

Delete an Eval

Invoke an Eval

List Evaluator Versions