Evals SDK Reference

Evals

List Evals

List all evaluators in the workspace.

from orq_ai_sdk import Orq
import os

with Orq(
    api_key=os.getenv("ORQ_API_KEY", ""),
) as orq:

    res = orq.evals.all(limit=10)

    # Handle response
    print(res)

import { Orq } from "@orq-ai/node";

const orq = new Orq({
  apiKey: process.env["ORQ_API_KEY"] ?? "",
});

async function run() {
  const result = await orq.evals.all({});

  console.log(result);
}

run();

Show Parameters

{
    "limit": Optional[int],
    "starting_after": Optional[str],
    "ending_before": Optional[str],
    "search": Optional[str],
    "sort": Optional[Literal["asc", "desc"]],
    "project_id": Optional[str],
}

{
  limit?: number;
  startingAfter?: string;
  endingBefore?: string;
  search?: string;
  sort?: "asc" | "desc";
  projectId?: string;
}

Show Response

{
    "object": Literal["list"],
    "data": List[Union[EvaluatorResponseLlm, EvaluatorResponseJSONSchema, EvaluatorResponseHTTP, EvaluatorResponsePython, EvaluatorResponseFunction, EvaluatorResponseRagas, EvaluatorResponseTypescript]],
    "has_more": bool,
}

{
  object: "list";
  data: (EvaluatorResponseLlm | EvaluatorResponseJsonSchema | EvaluatorResponseHttp | EvaluatorResponsePython | EvaluatorResponseFunction | EvaluatorResponseRagas | EvaluatorResponseTypescript)[];
  hasMore: boolean;
}

Create an Eval

Create a new evaluator in the workspace.

from orq_ai_sdk import Orq
import os

with Orq(
    api_key=os.getenv("ORQ_API_KEY", ""),
) as orq:

    res = orq.evals.create(request={
        "code": "<value>",
        "type": "python_eval",
        "path": "Default",
        "description": "",
        "key": "<key>",
    })

    # Handle response
    print(res)

import { Orq } from "@orq-ai/node";

const orq = new Orq({
  apiKey: process.env["ORQ_API_KEY"] ?? "",
});

async function run() {
  const result = await orq.evals.create({
    code: "<value>",
    type: "python_eval",
    path: "Default",
    description: "",
    key: "<key>",
  });

  console.log(result);
}

run();

Show Parameters

{
    "guardrail_config": Optional[Any],
    "output_type": Optional[Literal["boolean", "categorical", "number", "string"]],
    "type": Literal["llm_eval"],  # required
    "repetitions": Optional[int],
    "prompt": str,  # required
    "categories": List[str],  # optional
    "categorical_labels": [{  # optional
        "value": str,  # required
        "description": Optional[str],
    }],
    "dataset_id": Optional[str],
    "path": Optional[str],
    "project_id": Optional[str],
    "description": Optional[str],
    "key": str,  # required
    "mode": Literal["single"],  # required
    "model": str,  # required
}

{
  guardrailConfig?: any;
  outputType?: "boolean" | "categorical" | "number" | "string";
  type: "llm_eval";  // required
  repetitions?: number;
  prompt: string;  // required
  categories?: string[];
  categoricalLabels?: {
    value: string;  // required
    description?: string;
  }[];
  datasetId?: string;
  path?: string;
  projectId?: string;
  description?: string;
  key: string;  // required
  mode: "single";  // required
  model: string;  // required
}

Show Response

{
    "id": str,
    "description": str,
    "created": Optional[str],
    "updated": Optional[str],
    "updated_by_id": Optional[str],
    "project_id": Optional[str],
    "guardrail_config": Optional[Any],
    "type": Literal["llm_eval"],
    "repetitions": Optional[int],
    "prompt": str,
    "categories": List[str],  # optional
    "categorical_labels": [{  # optional
        "value": str,
        "description": Optional[str],
    }],
    "dataset_id": Optional[str],
    "key": str,
    "mode": Literal["single", "jury"],
    "model": Optional[str],
    "jury": {  # optional
        "judges": [{
            "model": str,
            "retry": {  # optional
                "count": Optional[int],
                "on_codes": List[int],  # optional
            },
            "fallbacks": [{  # optional
                "model": str,
            }],
        }],
        "replacement_judges": [{  # optional
            "model": str,
            "retry": {  # optional
                "count": Optional[int],
                "on_codes": List[int],  # optional
            },
            "fallbacks": [{  # optional
                "model": str,
            }],
        }],
        "min_successful_judges": Optional[int],
        "tie_value": Optional[Literal["Tie"]],
    },
}

{
  id: string;
  description: string;
  created?: string;
  updated?: string;
  updatedById?: string;
  projectId?: string;
  guardrailConfig?: any;
  type: "llm_eval";
  repetitions?: number;
  prompt: string;
  categories?: string[];
  categoricalLabels?: {
    value: string;
    description?: string;
  }[];
  datasetId?: string;
  key: string;
  mode: "single" | "jury";
  model?: string;
  jury?: {
    judges: {
      model: string;
      retry?: {
        count?: number;
        onCodes?: number[];
      };
      fallbacks?: {
        model: string;
      }[];
    }[];
    replacementJudges?: {
      model: string;
      retry?: {
        count?: number;
        onCodes?: number[];
      };
      fallbacks?: {
        model: string;
      }[];
    }[];
    minSuccessfulJudges?: number;
    tieValue?: "Tie";
  };
}

Retrieve an Eval

Retrieve a single evaluator by ID with more detail than the list endpoint: full type-specific config, owner, domain_id, metadata, enabled, and output_type.

from orq_ai_sdk import Orq
import os

with Orq(
    api_key=os.getenv("ORQ_API_KEY", ""),
) as orq:

    res = orq.evals.get(id="01JMDPA3QW5C1V0NJ1PW34T4E5")

    # Handle response
    print(res)

import { Orq } from "@orq-ai/node";

const orq = new Orq({
  apiKey: process.env["ORQ_API_KEY"] ?? "",
});

async function run() {
  const result = await orq.evals.get({
    id: "01JMDPA3QW5C1V0NJ1PW34T4E5",
  });

  console.log(result);
}

run();

Show Parameters

{
    "id": str,  # required
}

{
  id: string;  // required
}

Show Response

{
    "enabled": Optional[bool],
    "metadata": {
        "required_model_with_tools_support": Optional[bool],
        "required_retrieval_context": Optional[bool],
        "required_expected_output": Optional[bool],
        "supported_on_input_type": Optional[bool],
        "supported_on_output_type": Optional[bool],
        "support_use_as_guardrail": Optional[bool],
    },
    "id": str,
    "display_name": str,
    "description": str,
    "owner": str,
    "created": Optional[str],
    "updated": Optional[str],
    "created_by_id": Optional[str],
    "updated_by_id": Optional[str],
    "domain_id": str,
    "project_id": Optional[str],
    "guardrail_config": Optional[Any],
    "output_type": Optional[Literal["boolean", "categorical", "number", "string"]],
    "type": Literal["llm_eval"],
    "mode": Optional[Literal["single", "jury"]],
    "repetitions": Optional[int],
    "model": {  # optional
        "id": str,
        "integration_id": Optional[str],
        "model_parameters": {  # optional
            "temperature": Optional[float],
            "max_tokens": Optional[float],
            "top_k": Optional[float],
            "top_p": Optional[float],
            "frequency_penalty": Optional[float],
            "presence_penalty": Optional[float],
            "reasoning_effort": Optional[str],
            "budget_tokens": Optional[float],
        },
    },
    "jury": {  # optional
        "judges": [{
            "model": {
                "id": str,
                "integration_id": Optional[str],
                "model_parameters": {  # optional
                    "temperature": Optional[float],
                    "max_tokens": Optional[float],
                    "top_k": Optional[float],
                    "top_p": Optional[float],
                    "frequency_penalty": Optional[float],
                    "presence_penalty": Optional[float],
                    "reasoning_effort": Optional[str],
                    "budget_tokens": Optional[float],
                },
            },
            "retry": {  # optional
                "count": Optional[int],
                "on_codes": List[int],  # optional
            },
            "fallbacks": [{  # optional
                "model": {
                    "id": str,
                    "integration_id": Optional[str],
                    "model_parameters": {  # optional
                        "temperature": Optional[float],
                        "max_tokens": Optional[float],
                        "top_k": Optional[float],
                        "top_p": Optional[float],
                        "frequency_penalty": Optional[float],
                        "presence_penalty": Optional[float],
                        "reasoning_effort": Optional[str],
                        "budget_tokens": Optional[float],
                    },
                },
            }],
        }],
        "replacement_judges": [{  # optional
            "model": {
                "id": str,
                "integration_id": Optional[str],
                "model_parameters": {  # optional
                    "temperature": Optional[float],
                    "max_tokens": Optional[float],
                    "top_k": Optional[float],
                    "top_p": Optional[float],
                    "frequency_penalty": Optional[float],
                    "presence_penalty": Optional[float],
                    "reasoning_effort": Optional[str],
                    "budget_tokens": Optional[float],
                },
            },
            "retry": {  # optional
                "count": Optional[int],
                "on_codes": List[int],  # optional
            },
            "fallbacks": [{  # optional
                "model": {
                    "id": str,
                    "integration_id": Optional[str],
                    "model_parameters": {  # optional
                        "temperature": Optional[float],
                        "max_tokens": Optional[float],
                        "top_k": Optional[float],
                        "top_p": Optional[float],
                        "frequency_penalty": Optional[float],
                        "presence_penalty": Optional[float],
                        "reasoning_effort": Optional[str],
                        "budget_tokens": Optional[float],
                    },
                },
            }],
        }],
        "min_successful_judges": Optional[int],
        "tie_value": Optional[Literal["Tie"]],
    },
    "prompt": str,
    "categories": List[str],  # optional
    "categorical_labels": [{  # optional
        "value": str,
        "description": Optional[str],
    }],
    "dataset_id": Optional[str],
}

{
  enabled?: boolean;
  metadata: {
    requiredModelWithToolsSupport?: boolean;
    requiredRetrievalContext?: boolean;
    requiredExpectedOutput?: boolean;
    supportedOnInputType?: boolean;
    supportedOnOutputType?: boolean;
    supportUseAsGuardrail?: boolean;
  };
  id: string;
  displayName: string;
  description: string;
  owner: string;
  created?: string;
  updated?: string;
  createdById?: string;
  updatedById?: string;
  domainId: string;
  projectId?: string;
  guardrailConfig?: any;
  outputType?: "boolean" | "categorical" | "number" | "string";
  type: "llm_eval";
  mode?: "single" | "jury";
  repetitions?: number;
  model?: {
    id: string;
    integrationId?: string;
    modelParameters?: {
      temperature?: number;
      maxTokens?: number;
      topK?: number;
      topP?: number;
      frequencyPenalty?: number;
      presencePenalty?: number;
      reasoningEffort?: string;
      budgetTokens?: number;
    };
  };
  jury?: {
    judges: {
      model: {
        id: string;
        integrationId?: string;
        modelParameters?: {
          temperature?: number;
          maxTokens?: number;
          topK?: number;
          topP?: number;
          frequencyPenalty?: number;
          presencePenalty?: number;
          reasoningEffort?: string;
          budgetTokens?: number;
        };
      };
      retry?: {
        count?: number;
        onCodes?: number[];
      };
      fallbacks?: {
        model: {
          id: string;
          integrationId?: string;
          modelParameters?: {
            temperature?: number;
            maxTokens?: number;
            topK?: number;
            topP?: number;
            frequencyPenalty?: number;
            presencePenalty?: number;
            reasoningEffort?: string;
            budgetTokens?: number;
          };
        };
      }[];
    }[];
    replacementJudges?: {
      model: {
        id: string;
        integrationId?: string;
        modelParameters?: {
          temperature?: number;
          maxTokens?: number;
          topK?: number;
          topP?: number;
          frequencyPenalty?: number;
          presencePenalty?: number;
          reasoningEffort?: string;
          budgetTokens?: number;
        };
      };
      retry?: {
        count?: number;
        onCodes?: number[];
      };
      fallbacks?: {
        model: {
          id: string;
          integrationId?: string;
          modelParameters?: {
            temperature?: number;
            maxTokens?: number;
            topK?: number;
            topP?: number;
            frequencyPenalty?: number;
            presencePenalty?: number;
            reasoningEffort?: string;
            budgetTokens?: number;
          };
        };
      }[];
    }[];
    minSuccessfulJudges?: number;
    tieValue?: "Tie";
  };
  prompt: string;
  categories?: string[];
  categoricalLabels?: {
    value: string;
    description?: string;
  }[];
  datasetId?: string;
}

Update an Eval

Update an evaluator by ID with the provided fields.

from orq_ai_sdk import Orq
import os

with Orq(
    api_key=os.getenv("ORQ_API_KEY", ""),
) as orq:

    res = orq.evals.update(id="<id>", path="Default", project_id="01JMDPA3QW5C1V0NJ1PW34T4E5")

    # Handle response
    print(res)

import { Orq } from "@orq-ai/node";

const orq = new Orq({
  apiKey: process.env["ORQ_API_KEY"] ?? "",
});

async function run() {
  const result = await orq.evals.update({
    id: "<id>",
    requestBody: {
      path: "Default",
      projectId: "01JMDPA3QW5C1V0NJ1PW34T4E5",
    },
  });

  console.log(result);
}

run();

Show Parameters

{
    "id": str,  # required
    "type": Optional[str],
    "path": Optional[str],
    "project_id": Optional[str],
    "key": Optional[str],
    "description": Optional[str],
    "prompt": Optional[str],
    "output_type": Optional[str],
    "categories": List[str],  # optional
    "categorical_labels": [{  # optional
        "value": str,  # required
        "description": Optional[str],
    }],
    "dataset_id": Optional[str],
    "repetitions": Optional[float],
    "mode": Optional[Literal["single", "jury"]],
    "model": Optional[str],
    "jury": {  # optional
        "judges": [{  # required
            "model": str,  # required
            "retry": {  # optional
                "count": Optional[int],
                "on_codes": List[int],  # optional
            },
            "fallbacks": [{  # optional
                "model": str,  # required
            }],
        }],
        "replacement_judges": [{  # optional
            "model": str,  # required
            "retry": {  # optional
                "count": Optional[int],
                "on_codes": List[int],  # optional
            },
            "fallbacks": [{  # optional
                "model": str,  # required
            }],
        }],
        "min_successful_judges": Optional[int],
        "tie_value": Optional[Literal["Tie"]],
    },
    "schema_": Optional[str],
    "url": Optional[str],
    "method": Optional[str],
    "headers": Dict[str, str],  # optional
    "payload": Dict[str, Any],  # optional
    "code": Optional[str],
    "guardrail_config": Optional[Any],
    "version_increment": Optional[Literal["major", "minor", "patch"]],
    "version_description": Optional[str],
}

{
  id: string;  // required
  requestBody?: {
    type?: string;
    path?: string;
    projectId?: string;
    key?: string;
    description?: string;
    prompt?: string;
    outputType?: string;
    categories?: string[];
    categoricalLabels?: {
      value: string;  // required
      description?: string;
    }[];
    datasetId?: string;
    repetitions?: number;
    mode?: "single" | "jury";
    model?: string;
    jury?: {
      judges: {  // required
        model: string;  // required
        retry?: {
          count?: number;
          onCodes?: number[];
        };
        fallbacks?: {
          model: string;  // required
        }[];
      }[];
      replacementJudges?: {
        model: string;  // required
        retry?: {
          count?: number;
          onCodes?: number[];
        };
        fallbacks?: {
          model: string;  // required
        }[];
      }[];
      minSuccessfulJudges?: number;
      tieValue?: "Tie";
    };
    schema?: string;
    url?: string;
    method?: string;
    headers?: Record<string, string>;
    payload?: Record<string, any>;
    code?: string;
    guardrailConfig?: any;
    versionIncrement?: "major" | "minor" | "patch";
    versionDescription?: string;
  };
}

Show Response

{
    "id": str,
    "description": str,
    "created": Optional[str],
    "updated": Optional[str],
    "updated_by_id": Optional[str],
    "project_id": Optional[str],
    "guardrail_config": Optional[Any],
    "type": Literal["llm_eval"],
    "repetitions": Optional[int],
    "prompt": str,
    "categories": List[str],  # optional
    "categorical_labels": [{  # optional
        "value": str,
        "description": Optional[str],
    }],
    "dataset_id": Optional[str],
    "key": str,
    "mode": Literal["single", "jury"],
    "model": Optional[str],
    "jury": {  # optional
        "judges": [{
            "model": str,
            "retry": {  # optional
                "count": Optional[int],
                "on_codes": List[int],  # optional
            },
            "fallbacks": [{  # optional
                "model": str,
            }],
        }],
        "replacement_judges": [{  # optional
            "model": str,
            "retry": {  # optional
                "count": Optional[int],
                "on_codes": List[int],  # optional
            },
            "fallbacks": [{  # optional
                "model": str,
            }],
        }],
        "min_successful_judges": Optional[int],
        "tie_value": Optional[Literal["Tie"]],
    },
}

{
  id: string;
  description: string;
  created?: string;
  updated?: string;
  updatedById?: string;
  projectId?: string;
  guardrailConfig?: any;
  type: "llm_eval";
  repetitions?: number;
  prompt: string;
  categories?: string[];
  categoricalLabels?: {
    value: string;
    description?: string;
  }[];
  datasetId?: string;
  key: string;
  mode: "single" | "jury";
  model?: string;
  jury?: {
    judges: {
      model: string;
      retry?: {
        count?: number;
        onCodes?: number[];
      };
      fallbacks?: {
        model: string;
      }[];
    }[];
    replacementJudges?: {
      model: string;
      retry?: {
        count?: number;
        onCodes?: number[];
      };
      fallbacks?: {
        model: string;
      }[];
    }[];
    minSuccessfulJudges?: number;
    tieValue?: "Tie";
  };
}

Delete an Eval

Delete an evaluator by its unique identifier.

from orq_ai_sdk import Orq
import os

with Orq(
    api_key=os.getenv("ORQ_API_KEY", ""),
) as orq:

    orq.evals.delete(id="<id>")

    # Use the SDK ...

import { Orq } from "@orq-ai/node";

const orq = new Orq({
  apiKey: process.env["ORQ_API_KEY"] ?? "",
});

async function run() {
  await orq.evals.delete({
    id: "<id>",
  });

}

run();

Show Parameters

{
    "id": str,  # required
}

{
  id: string;  // required
}

Invoke an Eval

Runs an evaluator that already exists in the workspace. Accepts either a conversation or the structured input and output fields; when both are present the conversation wins.

from orq_ai_sdk import Orq
import os

with Orq(
    api_key=os.getenv("ORQ_API_KEY", ""),
) as orq:

    res = orq.evals.invoke(id="<id>")

    # Handle response
    print(res)

import { Orq } from "@orq-ai/node";

const orq = new Orq({
  apiKey: process.env["ORQ_API_KEY"] ?? "",
});

async function run() {
  const result = await orq.evals.invoke({
    id: "<id>",
    invokeEvaluatorRequest: {},
  });

  console.log(result);
}

run();

Show Parameters

{
    "id": str,  # required
    "context": {  # optional
        "messages": List[Dict[str, Any]],  # optional
        "input": {  # optional
            "system_instructions": Optional[str],
            "user_query": Optional[str],
            "retrievals": List[str],  # optional
            "expected_output": Optional[str],
        },
        "output": {  # optional
            "response": Optional[str],
            "tools_called": [{  # optional
                "name": Optional[str],
                "arguments": Optional[str],
                "output": Optional[str],
            }],
        },
        "variables": Dict[str, Any],  # optional
    },
    "model": Optional[str],
    "query": Optional[str],
    "output": Optional[str],
    "reference": Optional[str],
    "retrievals": List[str],  # optional
    "messages": List[Dict[str, Any]],  # optional
    "variables": Dict[str, Any],  # optional
}

{
  id: string;  // required
  invokeEvaluatorRequest: {  // required
    context?: {
      messages?: Record<string, unknown>[];
      input?: {
        systemInstructions?: string;
        userQuery?: string;
        retrievals?: string[];
        expectedOutput?: string;
      };
      output?: {
        response?: string;
        toolsCalled?: {
          name?: string;
          arguments?: string;
          output?: string;
        }[];
      };
      variables?: Record<string, any>;
    };
    model?: string;
    query?: string;
    output?: string;
    reference?: string;
    retrievals?: string[];
    messages?: Record<string, unknown>[];
    variables?: Record<string, any>;
  };
}

Show Response

{
    "result": {  # optional
        "type": Optional[str],
        "value": Optional[Any],
        "trace_id": Optional[str],
        "span_id": Optional[str],
        "evaluator_id": Optional[str],
        "status": Optional[str],
        "passed": Optional[bool],
        "explanation": Optional[str],
        "categories": List[str],  # optional
        "confidence": Optional[float],
    },
}

{
  result?: {
    type?: string;
    value?: any;
    traceId?: string;
    spanId?: string;
    evaluatorId?: string;
    status?: string;
    passed?: boolean;
    explanation?: string;
    categories?: string[];
    confidence?: number;
  };
}

List Evaluator Versions

Returns version history for a specific evaluator

from orq_ai_sdk import Orq
import os

with Orq(
    api_key=os.getenv("ORQ_API_KEY", ""),
) as orq:

    res = orq.evals.list_versions(id="<id>", limit=10)

    # Handle response
    print(res)

import { Orq } from "@orq-ai/node";

const orq = new Orq({
  apiKey: process.env["ORQ_API_KEY"] ?? "",
});

async function run() {
  const result = await orq.evals.listVersions({
    id: "<id>",
  });

  console.log(result);
}

run();

Show Parameters

{
    "id": str,  # required
    "limit": Optional[int],
    "starting_after": Optional[str],
    "ending_before": Optional[str],
}

{
  id: string;  // required
  limit?: number;
  startingAfter?: string;
  endingBefore?: string;
}

Show Response

{
    "object": Literal["list"],
    "data": [{
        "id": str,
        "created_at": str,
        "updated_at": str,
        "created_by_id": Optional[str],
        "updated_by_id": Optional[str],
        "version": str,
        "description": Optional[str],
        "checksum": str,
        "entity_type": str,
        "entity_id": str,
        "data": Dict[str, Any],
        "workspace_id": str,
    }],
    "has_more": bool,
}

{
  object: "list";
  data: {
    id: string;
    createdAt: string;
    updatedAt: string;
    createdById?: string;
    updatedById?: string;
    version: string;
    description?: string;
    checksum: string;
    entityType: string;
    entityId: string;
    data: Record<string, any>;
    workspaceId: string;
  }[];
  hasMore: boolean;
}

Orq SDK

Terminal & Agents

Terraform

Platform APIs

Management APIs

Evals

List Evals

Create an Eval

Retrieve an Eval

Update an Eval

Delete an Eval

Invoke an Eval

List Evaluator Versions

​Evals

​List Evals

​Create an Eval

​Retrieve an Eval

​Update an Eval

​Delete an Eval

​Invoke an Eval

​List Evaluator Versions

Evals

List Evals

Create an Eval

Retrieve an Eval

Update an Eval

Delete an Eval

Invoke an Eval

List Evaluator Versions