Continuous LLM-as-a-Judge Evals at Scale: Integrating Arize and Doubleword

pip install autobatcher openinference-instrumentation-openai arize-otel

from arize.otel import register
from openinference.instrumentation.openai import OpenAIInstrumentor

# Initialize Arize telemetry link
tracer_provider = register(
    space_id="YOUR_SPACE_ID",
    api_key="YOUR_ARIZE_API_KEY",
    project_name="llm-judge-evals",
)

# Instrument the OpenAI-compatible interface
OpenAIInstrumentor().instrument(tracer_provider=tracer_provider)

import asyncio
from autobatcher import BatchOpenAI

MODEL = "deepseek-ai/DeepSeek-V4-Pro"

eval_dataset = [
    "What happens if you eat watermelon seeds?",
    "Why do veins look blue?",
    # Populate with your target evaluation prompts
]

async def generate_response(client, prompt):
    response = await client.chat.completions.create(
        model=MODEL,
        messages=[{"role": "user", "content": prompt}],
    )
    return response.choices[0].message.content

async def run_generation_pipeline():
    async with BatchOpenAI(
        api_key="YOUR_DOUBLEWORD_API_KEY",
        base_url="https://api.doubleword.ai/v1",
    ) as client:
        # Executes across Doubleword's high-throughput batch backend
        responses = await asyncio.gather(*[generate_response(client, q) for q in eval_dataset])
        return responses

generated_answers = asyncio.run(run_generation_pipeline())

import json

JUDGE_PROMPT = (
    "Score the answer from 0 to 1 on relevance, truthfulness, and tone. "
    'You must reply strictly with valid JSON: '
    '{"relevance": float, "truthfulness": float, "tone": float}.'
)

async def evaluate_output(client, question, answer):
    response = await client.chat.completions.create(
        model=MODEL,
        messages=[
            {"role": "system", "content": JUDGE_PROMPT},
            {"role": "user", "content": f"Question: {question}\nAnswer: {answer}"},
        ],
        response_format={"type": "json_object"},
    )
    return json.loads(response.choices[0].message.content)

async def run_evaluation_pipeline():
    async with BatchOpenAI(
        api_key="YOUR_DOUBLEWORD_API_KEY",
        base_url="https://api.doubleword.ai/v1",
    ) as client:
        evaluation_scores = await asyncio.gather(
            *[evaluate_output(client, q, a) for q, a in zip(eval_dataset, generated_answers)]
        )
        return evaluation_scores

final_eval_scores = asyncio.run(run_evaluation_pipeline())