run_semantic_tests

Parameters schema

{
  "type": "object",
  "required": [
    "cases"
  ],
  "properties": {
    "mode": {
      "enum": [
        "tfidf",
        "embeddings"
      ],
      "type": "string",
      "description": "tfidf (default): fast, free, lexical. embeddings: OpenAI text-embedding-3-small, true semantic similarity, requires api_key."
    },
    "cases": {
      "type": "array",
      "items": {
        "type": "object",
        "required": [
          "actual",
          "expected"
        ],
        "properties": {
          "id": {
            "type": "string",
            "description": "Optional identifier for this case."
          },
          "actual": {
            "type": "string",
            "description": "The text produced by your LLM/system."
          },
          "expected": {
            "type": "string",
            "description": "The reference/ground-truth text."
          }
        }
      },
      "maxItems": 50,
      "description": "Array of (actual, expected) pairs to evaluate."
    },
    "api_key": {
      "type": "string",
      "description": "OpenAI API key — required only when mode is embeddings."
    },
    "thresholds": {
      "type": "object",
      "properties": {
        "cosine": {
          "type": "number",
          "maximum": 1,
          "minimum": 0,
          "description": "Minimum cosine similarity to pass (default: 0.75)."
        },
        "rouge_l": {
          "type": "number",
          "maximum": 1,
          "minimum": 0,
          "description": "Minimum ROUGE-L F1 to pass (default: 0.5)."
        }
      },
      "description": "Pass/fail thresholds (defaults: cosine 0.75, rouge_l 0.5)."
    },
    "require_all": {
      "type": "boolean",
      "description": "If true (default), all cases must pass for overall PASS. If false, at least one case passing returns PASS."
    }
  }
}

No endpoints wrapped at confidence ≥ 0.70.

Parent server