Build Experiments

from evaluatorq import DatasetIdInput

dataset_id = "01ARZ3NDEKTSV4RRFFQ69G5FAV"
# Pass DatasetIdInput directly to evaluatorq in the Run step

import { DatasetIdInput } from "@orq-ai/evaluatorq";

const datasetId = "01ARZ3NDEKTSV4RRFFQ69G5FAV";
// Pass DatasetIdInput directly to evaluatorq in the Run step

import csv, json
from evaluatorq import DataPoint

with open("test_data.csv", "r") as f:
    test_data = [DataPoint(inputs=row) for row in csv.DictReader(f)]

# or from JSON
with open("test_data.json", "r") as f:
    test_data = [DataPoint(inputs=item) for item in json.load(f)]

import { DataPoint } from "@orq-ai/evaluatorq";
import * as fs from "fs";
import csv from "csv-parser";

const data = JSON.parse(fs.readFileSync("test_data.json", "utf-8"));
const testData = data.map((item: any) => ({ inputs: item }));

from evaluatorq import DataPoint

test_data = [
    DataPoint(inputs={"text": "Cinderella tells the story of a kind young woman..."}),
    DataPoint(inputs={"text": "Little Red Riding Hood follows a girl traveling..."}),
]

import { DataPoint } from "@orq-ai/evaluatorq";

const testData: DataPoint[] = [
    { inputs: { text: "Cinderella tells the story of a kind young woman..." } },
    { inputs: { text: "Little Red Riding Hood follows a girl traveling..." } },
];

from evaluatorq import EvaluationResult

async def word_count_scorer(params):
    word_count = len(params["output"].get("summary", "").split())
    if word_count >= 10:
        return EvaluationResult(value=1.0, explanation=f"Sufficient ({word_count} words)")
    elif word_count >= 5:
        return EvaluationResult(value=0.5, explanation=f"Partial ({word_count} words)")
    else:
        return EvaluationResult(value=0.0, explanation=f"Too short ({word_count} words)")

const wordCountScorer = async (params: any) => {
    const wordCount = (params?.output?.summary || "").split(" ").filter((w: string) => w.length > 0).length;
    return {
        value: wordCount >= 10 ? 1.0 : wordCount >= 5 ? 0.5 : 0.0,
        explanation: `Word count: ${wordCount}`,
    };
};

import asyncio, os
from evaluatorq import EvaluationResult

EVAL_ID = os.environ.get("ORQ_EVALUATOR_ID", "your-evaluator-id")

async def summarization_quality_scorer(params):
    data, output = params["data"], params["output"]
    source_text = (data.inputs.get("text") or "").strip()
    summary = (output.get("summary") or "").strip()
    if not summary or not source_text:
        return EvaluationResult(value=0.0, explanation="Missing source or summary")
    evaluation = await asyncio.to_thread(
        orq_client.evals.invoke,
        id=EVAL_ID, query=source_text, output=summary,
        reference=None, messages=[], retrievals=[],
    )
    return EvaluationResult(value=float(evaluation.value.value), explanation=str(evaluation.value.explanation or ""))

const EVAL_ID = process.env.ORQ_EVALUATOR_ID || "your-evaluator-id";

const summarizationQualityScorer = async (params: any) => {
    const sourceText = (params.data?.inputs?.text || "").trim();
    const summary = (params.output?.summary || "").trim();
    if (!summary || !sourceText) return { value: 0.0, explanation: "Missing source or summary" };
    const evaluation = await orqClient.evals.invoke({
        id: EVAL_ID,
        invokeEvaluatorRequest: {
            query: sourceText, output: summary,
            reference: undefined, messages: [], retrievals: [],
        },
    });
    return { value: parseFloat(evaluation.value.value), explanation: evaluation.value.explanation || "" };
};

from evaluatorq import EvaluationResult

async def deepeval_relevancy_scorer(params):
    from deepeval.metrics import AnswerRelevancyMetric
    from deepeval.test_case import LLMTestCase
    source_text = (params["data"].inputs.get("text") or "").strip()
    summary = (params["output"].get("summary") or "").strip()
    if not summary or not source_text:
        return EvaluationResult(value=0.0, explanation="Missing source or summary")
    metric = AnswerRelevancyMetric(threshold=0.5)
    test_case = LLMTestCase(input=source_text, actual_output=summary)
    result = await asyncio.to_thread(metric.measure, test_case)
    return EvaluationResult(value=float(result.score), explanation=f"DeepEval relevancy: {result.score:.2f}")