{"data":{"name":"io.github.JcJamet/ia-qa-eval","title":"IA-QA Eval & RAG — 38 tools","description":"LLM & RAG evals: hallucination, eval contracts, semantic tests, tokens & model info. Free, no key.","version":"1.0.1","websiteUrl":null,"repositoryUrl":"https://github.com/ia-qa-com/mcp","remotes":[{"type":"streamable-http","url":"https://www.ia-qa.com/mcp/eval"}],"packages":[],"status":"active","isLatest":true,"publishedAt":"2026-10-26T21:30:39.179Z","updatedAt":"2026-10-26T21:30:39.179Z","source":"official-registry","verification":"official","confidence":100,"stars":null,"sources":[{"source":"official-registry","sourceId":"io.github.JcJamet/ia-qa-eval","verification":"official","confidence":100,"stars":null,"updatedAt":"2026-10-26T21:30:39.179Z","title":"IA-QA Eval & RAG — 38 tools","description":"LLM & RAG evals: hallucination, eval contracts, semantic tests, tokens & model info. Free, no key.","websiteUrl":null,"repositoryUrl":"https://github.com/ia-qa-com/mcp"}],"toolIndex":{"tools":[{"serverName":"io.github.JcJamet/ia-qa-eval","endpointFingerprint":"7fcc39b1e7937f1ee4cf3244acb7b0ac5b5cd90b622b49e0100cc523f18f7de7","name":"ab_test_report","title":null,"description":"Generate an A/B test report comparing two prompts or model configurations. Accepts arrays of scores and returns mean, median, std deviation, improvement percentage, and a Welch t-test at 95 %: `winner` names a variant only when the difference is significant — otherwise \"no_significant_difference\", or \"insufficient_samples\" under 2 scores per variant.","inputSchema":{"type":"object","properties":{"variant_a":{"type":"object","description":"First variant configuration with name and score array","properties":{"name":{"type":"string","description":"Name/label for variant A"},"scores":{"type":"array","items":{"type":"number"},"description":"Array of scores (0-100)"}}},"variant_b":{"type":"object","description":"Second variant configuration with name and score array","properties":{"name":{"type":"string","description":"Name/label for variant B"},"scores":{"type":"array","items":{"type":"number"},"description":"Array of scores (0-100)"}}}},"required":["variant_a","variant_b"],"examples":[{"variant_a":{"name":"A","scores":[70,71,69]},"variant_b":{"name":"B","scores":[70,71,69]}}]},"annotations":{"readOnlyHint":true,"destructiveHint":false,"idempotentHint":true,"openWorldHint":false},"definitionHash":"c0dd6f225cbbde97d914915dd7532fb732234e36b46c492ee043298f71a19810","schemaStatus":"valid","origin":"official-registry","observedAt":"2026-11-19T20:46:12.132Z","metadataTrust":"untrusted"},{"serverName":"io.github.JcJamet/ia-qa-eval","endpointFingerprint":"7fcc39b1e7937f1ee4cf3244acb7b0ac5b5cd90b622b49e0100cc523f18f7de7","name":"analyze_responses","title":null,"description":"Semantically analyze N already-produced model outputs for the SAME task (the MCP counterpart to the LLM Sandbox). Without a reference: computes consensus — pairwise cosine agreement, the most-representative output, and the outlier. With a `reference` (ground truth): also ranks every output by closeness (token cosine + ROUGE-L composite) and names the closest. No LLM, no key, runs locally. The reference mode is gate-able in CI; the consensus/outlier mode is a signal (no reference to fail against). You bring the outputs (2+). A 2-way head-to-head with structural JSON diff is compare_responses.","inputSchema":{"type":"object","properties":{"responses":{"type":"array","minItems":2,"description":"Outputs to analyze (same task, N models/prompts/versions): plain strings or { \"label\": \"GPT-4o\", \"text\": \"...\" }. At least 2.","items":{"type":"object","required":["text"],"properties":{"label":{"type":"string","description":"Human name for this candidate (e.g. model id)"},"text":{"type":"string","description":"The produced output"}}}},"reference":{"type":"string","description":"Optional ground-truth answer. If set, each output is also ranked by closeness to it and the closest one is named."}},"required":["responses"],"examples":[{"responses":[{"label":"A","text":"Lyon is the capital."},{"label":"B","text":"Paris is the capital of France."}],"reference":"Paris is the capital of France."}]},"annotations":{"readOnlyHint":true,"destructiveHint":false,"idempotentHint":true,"openWorldHint":false},"definitionHash":"ddae544cf7662e53c7e493082e2e52e74a2788912388d257c0bad440ad5145c8","schemaStatus":"valid","origin":"official-registry","observedAt":"2026-11-19T20:46:12.132Z","metadataTrust":"untrusted"},{"serverName":"io.github.JcJamet/ia-qa-eval","endpointFingerprint":"7fcc39b1e7937f1ee4cf3244acb7b0ac5b5cd90b622b49e0100cc523f18f7de7","name":"bm25_score","title":null,"description":"Compute BM25 relevance score between a query and one or more documents. BM25 is the industry-standard keyword-based ranking algorithm used in Elasticsearch, OpenSearch, and Weaviate hybrid search. Returns ranked results with normalized scores.","inputSchema":{"type":"object","properties":{"query":{"type":"string","description":"The search query"},"documents":{"type":"array","items":{"type":"string"},"description":"Array of documents to rank"},"k1":{"type":"number","description":"Term frequency saturation (default: 1.5)"},"b":{"type":"number","description":"Length normalization factor (default: 0.75)"},"top_k":{"type":"number","description":"Return top K results (default: all)"}},"required":["query","documents"],"examples":[{"query":"refund damaged","documents":["shipping times","refund for damaged items","office hours"]}]},"annotations":{"readOnlyHint":true,"destructiveHint":false,"idempotentHint":true,"openWorldHint":false},"definitionHash":"d6bcd65fdcb4c5c20a33cbdae74c7f5961d3ec604765e54c0277485591d059ca","schemaStatus":"valid","origin":"official-registry","observedAt":"2026-11-19T20:46:12.132Z","metadataTrust":"untrusted"},{"serverName":"io.github.JcJamet/ia-qa-eval","endpointFingerprint":"7fcc39b1e7937f1ee4cf3244acb7b0ac5b5cd90b622b49e0100cc523f18f7de7","name":"build_rag_prompt","title":null,"description":"Assemble a complete RAG (Retrieval-Augmented Generation) prompt from retrieved context chunks and a user query. Handles token budgeting, citation numbering, system instruction injection, and source attribution.","inputSchema":{"type":"object","properties":{"query":{"type":"string","description":"The user question to answer"},"chunks":{"type":"array","items":{"type":"object","properties":{"text":{"type":"string"},"source":{"type":"string"},"score":{"type":"number"}}},"description":"Retrieved context chunks with .text (required), .source (optional), .score (optional)"},"system_instruction":{"type":"string","description":"Custom system instruction (default: standard RAG grounding instruction)"},"max_context_tokens":{"type":"number","description":"Max tokens for context section (default: 2000)"},"cite_sources":{"type":"boolean","description":"Add [1], [2] citation numbers (default: true)"},"language":{"type":"string","description":"Response language instruction (e.g. \"French\", \"Spanish\")"}},"required":["query","chunks"],"examples":[{"query":"q?","chunks":[{"text":"c1 text","source":"a.md"}]}]},"annotations":{"readOnlyHint":true,"destructiveHint":false,"idempotentHint":true,"openWorldHint":false},"definitionHash":"0b82b531ceedcdfc5d5e794bd92960938b55d2bea466587d1d42ea5a1d647148","schemaStatus":"valid","origin":"official-registry","observedAt":"2026-11-19T20:46:12.132Z","metadataTrust":"untrusted"},{"serverName":"io.github.JcJamet/ia-qa-eval","endpointFingerprint":"7fcc39b1e7937f1ee4cf3244acb7b0ac5b5cd90b622b49e0100cc523f18f7de7","name":"compare_models","title":null,"description":"Put 2-5 AI models side by side: price per 1k tokens, context window, multimodal, reasoning, provider, pricing page. Facts only: names the cheapest and the largest context, never a \"best model\" — run your cases on each to choose.","inputSchema":{"type":"object","properties":{"models":{"type":"array","items":{"type":"string"},"description":"Array of 2-5 model names (e.g. [\"gpt-5\",\"claude-sonnet-4-6\",\"gemini-3.5-flash\"])"}},"required":["models"],"examples":[{"models":["gpt-6.1-sol","claude-sonnet-5-5"]}]},"annotations":{"readOnlyHint":true,"destructiveHint":false,"idempotentHint":true,"openWorldHint":false},"definitionHash":"5b8540d334778aa55e682ca84ebe6ef69d4a8a1626da9fd5b5e1ffb31b0da70e","schemaStatus":"valid","origin":"official-registry","observedAt":"2026-11-19T20:46:12.132Z","metadataTrust":"untrusted"},{"serverName":"io.github.JcJamet/ia-qa-eval","endpointFingerprint":"7fcc39b1e7937f1ee4cf3244acb7b0ac5b5cd90b622b49e0100cc523f18f7de7","name":"compare_responses","title":null,"description":"Compare two ALREADY-PRODUCED outputs (e.g. model A vs model B on the same task) side by side. Returns deterministic metrics (token cosine, ROUGE-L, Jaccard, length/structure deltas, JSON diff) and a verdict. If a `reference` (ground truth) is given, scores each output against it and picks the closer one. If `model` + `api_key` are given, an LLM judge also picks a qualitative winner for the task. No re-execution — you bring the outputs.","inputSchema":{"type":"object","properties":{"response_a":{"type":"string","description":"First output (e.g. model A's answer)"},"response_b":{"type":"string","description":"Second output (e.g. model B's answer)"},"label_a":{"type":"string","description":"Label for output A (e.g. \"GPT-4o\", \"v1.0\")"},"label_b":{"type":"string","description":"Label for output B (e.g. \"GPT-5-nano\", \"v1.1\")"},"task":{"type":"string","description":"The task/prompt both outputs were answering — used by the LLM judge for context"},"reference":{"type":"string","description":"Optional ground-truth / expected answer. If set, each output is scored against it and the closer one wins (deterministic)."},"check_json":{"type":"boolean","description":"Try to parse as JSON and compare structurally (keys, types, values)"},"model":{"type":"string","description":"Optional judge model id (BYOK). When set with api_key, an LLM judge picks a qualitative winner."},"api_key":{"type":"string","description":"Optional API key for the judge model (BYOK). Used only for the judge call; never stored."}},"required":["response_a","response_b"],"examples":[{"response_a":"Paris is the capital.","response_b":"Paris is the capital."}]},"annotations":{"readOnlyHint":true,"destructiveHint":false,"idempotentHint":true,"openWorldHint":false},"definitionHash":"ce8918eefb44600306374e261e056dc95f35e65a54dcfa92287dd0458d0bc042","schemaStatus":"valid","origin":"official-registry","observedAt":"2026-11-19T20:46:12.132Z","metadataTrust":"untrusted"},{"serverName":"io.github.JcJamet/ia-qa-eval","endpointFingerprint":"7fcc39b1e7937f1ee4cf3244acb7b0ac5b5cd90b622b49e0100cc523f18f7de7","name":"consistency_check","title":null,"description":"Compare multiple LLM responses to the same prompt and detect inconsistencies using Jaccard word-overlap similarity and fact drift (number comparison). Fast, deterministic, no API key needed. Limitations: relies on surface-level word matching — \"Paris is the capital of France\" vs \"Paris is the French capital\" may score low despite semantic equivalence. True semantic consistency: run_semantic_tests in embedding mode. Essential for determinism testing.","inputSchema":{"type":"object","properties":{"responses":{"type":"array","items":{"type":"string"},"description":"Array of 2+ LLM responses to compare (same prompt, different runs)"},"check_facts":{"type":"boolean","description":"Check for contradictory numbers/facts across responses (default: true)"}},"required":["responses"],"examples":[{"responses":["The capital of France is Paris.","Paris is the capital of France."]}]},"annotations":{"readOnlyHint":true,"destructiveHint":false,"idempotentHint":true,"openWorldHint":false},"definitionHash":"1a0bc72bbb908590106d5c3b82f4b0fafa02fe880f948f3ee7aeb53a4ae190d4","schemaStatus":"valid","origin":"official-registry","observedAt":"2026-11-19T20:46:12.132Z","metadataTrust":"untrusted"},{"serverName":"io.github.JcJamet/ia-qa-eval","endpointFingerprint":"7fcc39b1e7937f1ee4cf3244acb7b0ac5b5cd90b622b49e0100cc523f18f7de7","name":"context_window_check","title":null,"description":"Given an array of message objects [{role, content}], estimate total token usage and check if it fits in the target model's context window. Warns about truncation risk.","inputSchema":{"type":"object","properties":{"messages":{"type":"array","items":{"type":"object","properties":{"role":{"type":"string"},"content":{"type":"string"}}},"description":"Array of messages (system/user/assistant)"},"model":{"type":"string","description":"Model id from list_llm_models (e.g. gpt-6.1-sol, claude-sonnet-5-5)"},"max_output_tokens":{"type":"number","description":"Reserved tokens for output (default: 4096)"}},"required":["messages","model"],"examples":[{"messages":[{"role":"user","content":"hi"}],"model":"gpt-4o"}]},"annotations":{"readOnlyHint":true,"destructiveHint":false,"idempotentHint":true,"openWorldHint":false},"definitionHash":"c789b83eb9373b741679cc36097d331f4eeb555cbad1d9213c9d7e940fbf6f51","schemaStatus":"valid","origin":"official-registry","observedAt":"2026-11-19T20:46:12.132Z","metadataTrust":"untrusted"},{"serverName":"io.github.JcJamet/ia-qa-eval","endpointFingerprint":"7fcc39b1e7937f1ee4cf3244acb7b0ac5b5cd90b622b49e0100cc523f18f7de7","name":"conversation_analyze","title":null,"description":"Analyze a multi-turn conversation for context retention, topic drift, instruction following, and repetition. Accepts messages array [{role, content}]. Essential for chatbot QA.","inputSchema":{"type":"object","properties":{"messages":{"type":"array","items":{"type":"object","properties":{"role":{"type":"string","enum":["system","user","assistant"]},"content":{"type":"string"}}},"description":"Conversation messages in order"}},"required":["messages"],"examples":[{"messages":[{"role":"user","content":"What is your refund policy?"},{"role":"assistant","content":"Refunds are available within 30 days."}]}]},"annotations":{"readOnlyHint":true,"destructiveHint":false,"idempotentHint":true,"openWorldHint":false},"definitionHash":"d27706d0447bcf4d9df5feed77b927ad5cf34db72c2a16861a11dcdf956e3443","schemaStatus":"valid","origin":"official-registry","observedAt":"2026-11-19T20:46:12.132Z","metadataTrust":"untrusted"},{"serverName":"io.github.JcJamet/ia-qa-eval","endpointFingerprint":"7fcc39b1e7937f1ee4cf3244acb7b0ac5b5cd90b622b49e0100cc523f18f7de7","name":"cot_analyzer","title":null,"description":"Analyze a Chain-of-Thought (CoT) or reasoning trace from an LLM. Detects step count, logical flow, conclusion presence, backtracking, and estimates reasoning depth. Useful for o1/o3/DeepSeek-R1 evaluation.","inputSchema":{"type":"object","properties":{"reasoning":{"type":"string","description":"The CoT / reasoning trace text (e.g. from <think> tags or step-by-step output)"},"expected_conclusion":{"type":"string","description":"Expected final answer to check against (optional)"}},"required":["reasoning"],"examples":[{"reasoning":"Step 1: 12 apples minus 5 eaten.\nStep 2: 12 - 5 = 7.\nTherefore, the answer is 7.","expected_conclusion":"7"}]},"annotations":{"readOnlyHint":true,"destructiveHint":false,"idempotentHint":true,"openWorldHint":false},"definitionHash":"513d50575582ed8fcab6ca7f30f73befc6a1dc67966c94642974aa17d0d08772","schemaStatus":"valid","origin":"official-registry","observedAt":"2026-11-19T20:46:12.132Z","metadataTrust":"untrusted"},{"serverName":"io.github.JcJamet/ia-qa-eval","endpointFingerprint":"7fcc39b1e7937f1ee4cf3244acb7b0ac5b5cd90b622b49e0100cc523f18f7de7","name":"count_tokens","title":null,"description":"Estimate the token count of a text string using the cl100k_base approximation (~4 chars/token). Shows whether a text fits the context window of a model and what it costs. Returns token estimate, character count, and word count.","inputSchema":{"type":"object","properties":{"input":{"type":"string","description":"Text to count tokens for"}},"required":["input"],"examples":[{"input":"Hello world, this is a test."}]},"annotations":{"readOnlyHint":true,"destructiveHint":false,"idempotentHint":true,"openWorldHint":false},"definitionHash":"c6ae8e81a61cf941b5152dc71267d1e34f95ede567bc74a3146943b3830187ad","schemaStatus":"valid","origin":"official-registry","observedAt":"2026-11-19T20:46:12.132Z","metadataTrust":"untrusted"},{"serverName":"io.github.JcJamet/ia-qa-eval","endpointFingerprint":"7fcc39b1e7937f1ee4cf3244acb7b0ac5b5cd90b622b49e0100cc523f18f7de7","name":"embedding_similarity","title":null,"description":"Compute text similarity using local algorithms (Bag of Words, TF-IDF, Character N-grams). No API key needed — runs entirely in-process. NOT real embeddings: true vector embeddings are run_semantic_tests with mode=\"embeddings\" and an OpenAI key. Supports single pair or batch mode with pipe-separated pairs. Useful for RAG retrieval testing, semantic search evaluation, and text deduplication.","inputSchema":{"type":"object","properties":{"text_a":{"type":"string","description":"First text to compare (single-pair mode)"},"text_b":{"type":"string","description":"Second text to compare (single-pair mode)"},"batch":{"type":"array","description":"Batch mode: array of { text_a, text_b } pairs. Overrides text_a/text_b if provided.","items":{"type":"object","required":["text_a","text_b"],"properties":{"text_a":{"type":"string"},"text_b":{"type":"string"}}}},"methods":{"type":"array","description":"Algorithms to use (default: all three). Options: \"bow\", \"tfidf\", \"ngram\"","items":{"type":"string","enum":["bow","tfidf","ngram"]}}},"examples":[{"text_a":"the cat sat","text_b":"quantum field theory"}]},"annotations":{"readOnlyHint":true,"destructiveHint":false,"idempotentHint":true,"openWorldHint":false},"definitionHash":"30e3ab7c8448b7f80d7b193c1078dce1ce5f024ebcfc18b6cb7f4a76755b065d","schemaStatus":"valid","origin":"official-registry","observedAt":"2026-11-19T20:46:12.132Z","metadataTrust":"untrusted"},{"serverName":"io.github.JcJamet/ia-qa-eval","endpointFingerprint":"7fcc39b1e7937f1ee4cf3244acb7b0ac5b5cd90b622b49e0100cc523f18f7de7","name":"estimate_llm_cost","title":null,"description":"Estimate the API cost in USD for a given model and token counts, at the price read daily from the provider's own pricing page (prices_as_of, price_status). Any model from list_llm_models.","inputSchema":{"type":"object","properties":{"model":{"type":"string","description":"Model name, e.g. \"gpt-4o\", \"claude-3.5-sonnet\", \"deepseek-v3\""},"input_tokens":{"type":"number","description":"Number of input/prompt tokens"},"output_tokens":{"type":"number","description":"Number of output/completion tokens (default: 0)"}},"required":["model","input_tokens"],"examples":[{"model":"gpt-4o","input_tokens":1000000,"output_tokens":1000000}]},"annotations":{"readOnlyHint":true,"destructiveHint":false,"idempotentHint":true,"openWorldHint":false},"definitionHash":"4fb280136913c20c866bdfd8d2f8f67571ba719a16cddf9a57baa0bbfe5b48b8","schemaStatus":"valid","origin":"official-registry","observedAt":"2026-11-19T20:46:12.132Z","metadataTrust":"untrusted"},{"serverName":"io.github.JcJamet/ia-qa-eval","endpointFingerprint":"7fcc39b1e7937f1ee4cf3244acb7b0ac5b5cd90b622b49e0100cc523f18f7de7","name":"extract_json_from_text","title":null,"description":"Extract the first valid JSON object or array embedded in chaotic LLM output (surrounded by markdown fences, prose, or explanatory text). Handles ```json blocks and inline JSON.","inputSchema":{"type":"object","properties":{"input":{"type":"string","description":"Raw text (e.g., LLM output) that may contain a JSON object or array"}},"required":["input"],"examples":[{"input":"Here:\n```json\n{\"a\":[1,2]}\n```\nthanks"}]},"annotations":{"readOnlyHint":true,"destructiveHint":false,"idempotentHint":true,"openWorldHint":false},"definitionHash":"c9100a62fc488d4ea2a721e0847e566562e41a8ecc87697681dd7fa2bb867d7b","schemaStatus":"valid","origin":"official-registry","observedAt":"2026-11-19T20:46:12.132Z","metadataTrust":"untrusted"},{"serverName":"io.github.JcJamet/ia-qa-eval","endpointFingerprint":"7fcc39b1e7937f1ee4cf3244acb7b0ac5b5cd90b622b49e0100cc523f18f7de7","name":"find_tool","title":null,"description":"Search available MCP tools by keyword or category before calling them. Returns names, descriptions and, on request, inputSchemas. Queries are logged 90 days, screened for credentials and personal data. Categories: data, dev, encoding, eval, integrations, llm, meta, qa, rag, security, text, web.","inputSchema":{"type":"object","properties":{"query":{"type":"string","description":"Keywords, e.g. \"cors\", \"token\", \"vector\", \"json\""},"category":{"type":"string","description":"Optional filter, one of the categories above. Any other value is refused."},"with_schema":{"type":"boolean","description":"Set true to include inputSchema in results (default: false)"},"max_results":{"type":"number","description":"Maximum tools to return (default 10, max 50). Best match first."}},"required":["query"],"examples":[{"query":"leaked api key"}]},"annotations":{"readOnlyHint":true,"destructiveHint":false,"idempotentHint":true,"openWorldHint":false},"definitionHash":"54122085a43b655ab57c8386ad35bab84d39d0329f1af0670a57864b1f1e96d4","schemaStatus":"valid","origin":"official-registry","observedAt":"2026-11-19T20:46:12.132Z","metadataTrust":"untrusted"},{"serverName":"io.github.JcJamet/ia-qa-eval","endpointFingerprint":"7fcc39b1e7937f1ee4cf3244acb7b0ac5b5cd90b622b49e0100cc523f18f7de7","name":"generate_ci_workflow","title":null,"description":"Generate a ready-to-commit GitHub Actions workflow that gates a build on IA-QA: \"eval_contract\" runs a .ia-eval.yaml through ia-qa-com/eval-action@v1 (provider key as a repo secret), \"cli_checks\" runs deterministic tools via npx @ia-qa/cli and fails on their verdict, \"selector_drift\" gates E2E selectors with @ia-qa/self-healing. Deterministic template, no LLM; returns the YAML, the secrets and the next steps. Pair with generate_eval_yaml.","inputSchema":{"type":"object","properties":{"gate":{"type":"string","enum":["eval_contract","cli_checks","both","selector_drift","all"],"description":"Default eval_contract. both = CLI checks + eval; all = CLI checks, drift, eval."},"contract_path":{"type":"string","description":"Default evals/smoke.ia-eval.yaml."},"provider":{"type":"string","enum":["groq","openai","anthropic","google"],"description":"Decides which repo secret is wired (default groq)."},"cli_tools":{"type":"array","items":{"type":"string"},"description":"Tools to gate on, e.g. [\"secret_scan\",\"prompt_injection_scan\"]."},"triggers":{"type":"array","items":{"type":"string","enum":["push","pull_request","workflow_dispatch","schedule"]},"description":"Default push + pull_request."},"cron":{"type":"string","description":"For schedule (default '0 6 * * 1')."},"min_score":{"type":"number","description":"Override the contract min_score (0-100)."},"fail_on_fail":{"type":"boolean","description":"Default true; false reports without gating."},"workflow_name":{"type":"string"},"node_version":{"type":"string","description":"Default \"20\"."},"app_start_command":{"type":"string","description":"[selector_drift] Boots the app, e.g. \"npm run start:ci\". Never guessed: omitted, the step is a TODO."},"app_base_url":{"type":"string","description":"[selector_drift] Default http://127.0.0.1:3000; must match .ia-qa/config.json."}},"examples":[{}]},"annotations":{"readOnlyHint":true,"destructiveHint":false,"idempotentHint":true,"openWorldHint":false},"definitionHash":"4ab25a4ebf35935d0ce73419eec13092928a0b3e79f01c12c43912ba0c2b1faa","schemaStatus":"valid","origin":"official-registry","observedAt":"2026-11-19T20:46:12.132Z","metadataTrust":"untrusted"},{"serverName":"io.github.JcJamet/ia-qa-eval","endpointFingerprint":"7fcc39b1e7937f1ee4cf3244acb7b0ac5b5cd90b622b49e0100cc523f18f7de7","name":"generate_eval_yaml","title":null,"description":"Generate a complete .ia-eval.yaml evaluation contract from a plain-language description of what your LLM should do. Uses Groq openai/gpt-oss-20b (server-side, no API key needed). Returns ready-to-run YAML for the LLM Test Runner (run_eval_contract). Picks appropriate evaluators (cosine_similarity, contains_check, hallucination_check, etc.) based on the task type.","inputSchema":{"type":"object","properties":{"description":{"type":"string","description":"Plain-language description of what the LLM under test should do. Be specific: describe inputs, expected behaviour, and constraints."},"system_prompt":{"type":"string","description":"Optional system prompt of the LLM under test. Helps generate more accurate test cases."},"scenario_count":{"type":"number","enum":[3,5,8],"description":"Number of scenarios to generate (default: 5). Covers happy path + edge cases + adversarial."},"task_type":{"type":"string","enum":["rag","summarization","classification","safety","customer_support","code_gen"],"description":"Optional task type hint to guide evaluator selection."}},"required":["description"],"examples":[{"description":"Summarize a customer support ticket in one sentence."}]},"annotations":{"readOnlyHint":true,"destructiveHint":false,"idempotentHint":false,"openWorldHint":true},"definitionHash":"9aa4b7adf2d864a80bf6dcd24234618bba6246c6be408bde89d71b85ef74300e","schemaStatus":"valid","origin":"official-registry","observedAt":"2026-11-19T20:46:12.132Z","metadataTrust":"untrusted"},{"serverName":"io.github.JcJamet/ia-qa-eval","endpointFingerprint":"7fcc39b1e7937f1ee4cf3244acb7b0ac5b5cd90b622b49e0100cc523f18f7de7","name":"get_testing_guidelines","title":null,"description":"Query the IA-QA methodology knowledge base. Returns structured testing guidelines, assertion strategies, thresholds, best practices, and relevant MCP tools for a given topic. Call without a topic to list all available topics. Topics: llm-unit-testing, rag-pipeline, prompt-stability, prompt-ab-testing, embedding-quality, eval-framework, semantic-testing, auto-testing, security, api-testing, ci-cd, multimodal, llm-data-security, agent-observability, pro-tips, learning-paths, golden-dataset, selector-drift, qa-recipes, playbooks. Not sure where to start testing an LLM, RAG pipeline or agent? Call without a topic (or with \"start-here\"): it maps what you are testing to the tools to call and the output field to gate CI on. A plain question such as \"how do I test my RAG\" also resolves to the right topic.","inputSchema":{"type":"object","properties":{"topic":{"type":"string","enum":["start-here","llm-unit-testing","rag-pipeline","prompt-stability","prompt-ab-testing","embedding-quality","eval-framework","semantic-testing","auto-testing","security","api-testing","ci-cd","multimodal","llm-data-security","agent-observability","pro-tips","learning-paths","golden-dataset","selector-drift","qa-recipes","playbooks"],"description":"The testing topic to retrieve guidelines for. Omit to get the start-here map and the full list of available topics."}},"examples":[{"topic":"rag-pipeline"}]},"annotations":{"readOnlyHint":true,"destructiveHint":false,"idempotentHint":true,"openWorldHint":false},"definitionHash":"7fa9775dabd7a8681609420a8c0c102780c06d5c69a7b373ff3a1b264cbbfbd3","schemaStatus":"valid","origin":"official-registry","observedAt":"2026-11-19T20:46:12.132Z","metadataTrust":"untrusted"},{"serverName":"io.github.JcJamet/ia-qa-eval","endpointFingerprint":"7fcc39b1e7937f1ee4cf3244acb7b0ac5b5cd90b622b49e0100cc523f18f7de7","name":"hallucination_check","title":null,"description":"Lexical hallucination check: an LLM answer's words, numbers and polarity against the source. Deterministic, no API key. Each answer sentence is aligned to its best source sentence, so a number is support only on the SAME statement (\"founded in 1998\" is not grounded by \"sold 1998 units\"), and a negation or antonym flip against it returns verdict \"contradicted\". Lexical only: no paraphrase, synonym or multi-sentence reasoning, so \"well_grounded\" means nothing lexical was found, never \"verified true\". Entailment: run_semantic_tests (NLI/embedding) or a calibrated judge.","inputSchema":{"type":"object","properties":{"answer":{"type":"string","description":"The LLM answer to verify (with context)."},"context":{"type":"string","description":"The source text that should ground it."},"cases":{"type":"array","maxItems":50,"items":{"type":"object","required":["answer","context"],"properties":{"id":{"type":"string"},"answer":{"type":"string"},"context":{"type":"string"}}},"description":"Batch instead of answer + context: one result per case, and the worst gate overall."},"strict":{"type":"boolean","description":"Every sentence must be supported (default false)."}},"examples":[{"context":"Le remboursement prend plus de 7 jours ouvrés.","answer":"Le remboursement prend plus de 7 jours ouvrés."}]},"annotations":{"readOnlyHint":true,"destructiveHint":false,"idempotentHint":true,"openWorldHint":false},"definitionHash":"501c6b82096e65035794550ef9e0ed22eb1d0b1965ea756f64868f8e171ab756","schemaStatus":"valid","origin":"official-registry","observedAt":"2026-11-19T20:46:12.132Z","metadataTrust":"untrusted"},{"serverName":"io.github.JcJamet/ia-qa-eval","endpointFingerprint":"7fcc39b1e7937f1ee4cf3244acb7b0ac5b5cd90b622b49e0100cc523f18f7de7","name":"list_llm_models","title":null,"description":"List all LLM models available on ia-qa.com with their provider, API endpoint, and capabilities. Filter by provider name (e.g. \"Groq\", \"HuggingFace\", \"OpenAI\") or return the full catalog. Use this to discover which models are available before calling an LLM API, or to compare providers.","inputSchema":{"type":"object","properties":{"provider":{"type":"string","description":"Provider name, case-insensitive (e.g. \"Groq\", \"OpenAI\", \"Anthropic\"). Omit for the full catalogue."}},"examples":[{"provider":"groq"}]},"annotations":{"readOnlyHint":true,"destructiveHint":false,"idempotentHint":true,"openWorldHint":false},"definitionHash":"7b94852daca0e5ea42c9eaccf336b0fb62fe0ef32919dd04909e6344180d08a0","schemaStatus":"valid","origin":"official-registry","observedAt":"2026-11-19T20:46:12.132Z","metadataTrust":"untrusted"},{"serverName":"io.github.JcJamet/ia-qa-eval","endpointFingerprint":"7fcc39b1e7937f1ee4cf3244acb7b0ac5b5cd90b622b49e0100cc523f18f7de7","name":"llm_format_check","title":null,"description":"Validate that an LLM output matches an expected format: JSON, Markdown, code block, bullet list, numbered list, table, YAML, XML, or custom regex. Essential for structured output testing.","inputSchema":{"type":"object","properties":{"output":{"type":"string","description":"The LLM output to validate"},"expected_format":{"type":"string","description":"Expected format","enum":["json","markdown_heading","code_block","bullet_list","numbered_list","table","yaml","xml","regex"]},"regex_pattern":{"type":"string","description":"Custom regex pattern (only when expected_format is \"regex\")"}},"required":["output","expected_format"],"examples":[{"output":"a: 1\nb: [1, 2]","expected_format":"yaml"}]},"annotations":{"readOnlyHint":true,"destructiveHint":false,"idempotentHint":true,"openWorldHint":false},"definitionHash":"ea28df3c40add41ad2a2607dc78f09950d8e87515331092098008806e0e0e084","schemaStatus":"valid","origin":"official-registry","observedAt":"2026-11-19T20:46:12.132Z","metadataTrust":"untrusted"},{"serverName":"io.github.JcJamet/ia-qa-eval","endpointFingerprint":"7fcc39b1e7937f1ee4cf3244acb7b0ac5b5cd90b622b49e0100cc523f18f7de7","name":"llm_generate","title":null,"description":"Generate text with an open-source model hosted on Groq, on the server's key — no API key needed. Models: openai/gpt-oss-20b, openai/gpt-oss-120b, qwen/qwen3.8-27b (default \"openai/gpt-oss-20b\"). HuggingFace models are not served on the server key. A reasoning model that spends max_tokens thinking is retried once at 4096. Rate-limited.","inputSchema":{"type":"object","properties":{"prompt":{"type":"string","description":"The user prompt / instruction to send to the model"},"model":{"type":"string","description":"Model ID (default: \"openai/gpt-oss-20b\"). Server-keyed whitelist only — Groq: openai/gpt-oss-20b, openai/gpt-oss-120b, qwen/qwen3.8-27b. Other ids from list_llm_models, HuggingFace ones included, are BYOK-only and will be rejected."},"system":{"type":"string","description":"Optional system prompt to set context or persona"},"temperature":{"type":"number","description":"Sampling temperature 0.0–1.5 (default: 0.7)"},"max_tokens":{"type":"number","description":"Maximum tokens to generate (default: 2048, max: 4096)"}},"required":["prompt"],"examples":[{"prompt":"Reply with the single word OK."}]},"annotations":{"readOnlyHint":true,"destructiveHint":false,"idempotentHint":false,"openWorldHint":true},"definitionHash":"2178e4ced23f1a97b5355ec54567b991c09841838a546b1de6f962552784097f","schemaStatus":"valid","origin":"official-registry","observedAt":"2026-11-19T20:46:12.132Z","metadataTrust":"untrusted"},{"serverName":"io.github.JcJamet/ia-qa-eval","endpointFingerprint":"7fcc39b1e7937f1ee4cf3244acb7b0ac5b5cd90b622b49e0100cc523f18f7de7","name":"llm_json_schema_check","title":null,"description":"Validate that an LLM JSON output matches a JSON Schema definition. Tests required fields, types, enums, nested objects, and arrays. Critical for function-calling and structured output testing.","inputSchema":{"type":"object","properties":{"output":{"type":"string","description":"The LLM JSON output (raw string, will be parsed)"},"schema":{"type":"object","additionalProperties":true,"description":"JSON Schema (draft-07 subset) to validate against"}},"required":["output","schema"],"examples":[{"output":"{\"name\":\"a\",\"age\":3}","schema":{"type":"object","required":["name","age"],"properties":{"name":{"type":"string"},"age":{"type":"integer","minimum":0},"role":{"enum":["admin","user"]}}}}]},"annotations":{"readOnlyHint":true,"destructiveHint":false,"idempotentHint":true,"openWorldHint":false},"definitionHash":"a8668e1451e41b65cfc6615fd4179faf8a41f93ee74814026c6fd6a54d80a17f","schemaStatus":"valid","origin":"official-registry","observedAt":"2026-11-19T20:46:12.132Z","metadataTrust":"untrusted"},{"serverName":"io.github.JcJamet/ia-qa-eval","endpointFingerprint":"7fcc39b1e7937f1ee4cf3244acb7b0ac5b5cd90b622b49e0100cc523f18f7de7","name":"llm_output_validator","title":null,"description":"Validate an LLM response against QA criteria: format checks (JSON, code, markdown), content rules (must-include, must-not-include), length constraints, language detection, and safety patterns. Essential for QA testing LLM-powered features.","inputSchema":{"type":"object","properties":{"output":{"type":"string","description":"The LLM output text to validate"},"expected_format":{"type":"string","description":"Expected output format","enum":["json","markdown","code","plain","any"]},"must_include":{"type":"string","description":"Comma-separated strings that MUST appear in the output"},"must_not_include":{"type":"string","description":"Comma-separated strings that must NOT appear (e.g. \"TODO, FIXME, undefined, NaN\")"},"max_length":{"type":"number","description":"Maximum character length for the output"},"min_length":{"type":"number","description":"Minimum character length for the output"},"expected_language":{"type":"string","description":"Expected language of the output (en, fr, es, de…). Checks for common words."},"check_safety":{"type":"boolean","description":"Check for PII patterns (emails, phones, SSN), profanity signals, and prompt leakage"},"check_json_schema":{"type":"string","description":"If expected_format is JSON, provide required keys as comma-separated list to validate the structure"}},"required":["output"],"examples":[{"output":"{\"a\":1}","expected_format":"json","must_include":"a","must_not_include":"TODO"}]},"annotations":{"readOnlyHint":true,"destructiveHint":false,"idempotentHint":true,"openWorldHint":false},"definitionHash":"63ce39a46ebc47dd09236e260a802e533380206ab946472556d688e5ac040165","schemaStatus":"valid","origin":"official-registry","observedAt":"2026-11-19T20:46:12.132Z","metadataTrust":"untrusted"},{"serverName":"io.github.JcJamet/ia-qa-eval","endpointFingerprint":"7fcc39b1e7937f1ee4cf3244acb7b0ac5b5cd90b622b49e0100cc523f18f7de7","name":"metamorphic_check","title":null,"description":"Reference-free stability check: does an assistant's answer stay the same when the QUESTION is transformed (case, typo, paraphrase, reorder, translation)? You bring the outputs; no model is called (tfidf mode is deterministic and free). Default thresholds: case .95, typo .90, paraphrase .80, reorder .80, translation .75 (embeddings only), specialization .60 (advisory, never gated). Returns PASS / FAIL / INVALID — INVALID when the BASE answer is a refusal or too short, so a model that refuses everything never scores 1.0. Pair with run_semantic_tests for correctness.","inputSchema":{"type":"object","properties":{"base":{"type":"object","description":"The original run.","properties":{"question":{"type":"string","description":"Optional, echoed back."},"output":{"type":"string","description":"The answer to the original question."}}},"variants":{"type":"array","maxItems":20,"description":"Answers to transformed versions of the question.","items":{"type":"object","required":["relation","output"],"properties":{"id":{"type":"string"},"relation":{"type":"string","enum":["case","typo","paraphrase","reorder","translation","specialization"],"description":"Transformation applied to the question; sets the metric and threshold."},"question":{"type":"string","description":"Optional, echoed back."},"output":{"type":"string","description":"The answer to the transformed question."}}}},"mode":{"type":"string","enum":["tfidf","embeddings"],"description":"tfidf (default, lexical: gate on case/typo, read paraphrase as a trend) or embeddings (OpenAI, needs api_key; required for translation)."},"api_key":{"type":"string","description":"OpenAI key, embeddings mode only."},"thresholds":{"type":"object","description":"Per-relation overrides, 0-1. Calibrate on your own data before gating.","properties":{"case":{"type":"number","minimum":0,"maximum":1},"typo":{"type":"number","minimum":0,"maximum":1},"paraphrase":{"type":"number","minimum":0,"maximum":1},"reorder":{"type":"number","minimum":0,"maximum":1},"translation":{"type":"number","minimum":0,"maximum":1},"specialization":{"type":"number","minimum":0,"maximum":1}}},"baseline_guard":{"type":"object","description":"Floor on the BASE answer; failing it returns INVALID.","properties":{"min_length":{"type":"number","description":"Default 40 chars."},"must_not_match":{"type":"array","items":{"type":"string"},"description":"Refusal patterns, checked in the first 200 chars. Replaces the default FR+EN list; [] disables."}}},"require_all":{"type":"boolean","description":"Default true: every gated variant must pass. Keep it for a gate — false lets one easy typo row hold the run at PASS while a paraphrase fails."}},"required":["base","variants"],"examples":[{"base":{"output":"Paris is the capital and largest city of France."},"variants":[{"relation":"case","output":"Paris is the capital and largest city of France."}]}]},"annotations":{"readOnlyHint":true,"destructiveHint":false,"idempotentHint":true,"openWorldHint":false},"definitionHash":"50559389ba8ae0574bb49cca06605f0da8937088351d7090495f72395a590d00","schemaStatus":"valid","origin":"official-registry","observedAt":"2026-11-19T20:46:12.132Z","metadataTrust":"untrusted"}],"endpoints":[{"endpointFingerprint":"7fcc39b1e7937f1ee4cf3244acb7b0ac5b5cd90b622b49e0100cc523f18f7de7","protocol":"streamable-http","status":"indexed","toolCount":39,"observedAt":"2026-11-19T20:46:12.132Z"}],"authentication":"unknown"},"connectionConfigurations":{"status":"ready","authentication":"not embedded","clients":{"Claude Desktop":{"mcpServers":{"io.github.JcJamet/ia-qa-eval":{"url":"https://www.ia-qa.com/mcp/eval"}}},"Cursor":{"mcpServers":{"io.github.JcJamet/ia-qa-eval":{"url":"https://www.ia-qa.com/mcp/eval"}}},"VS Code":{"servers":{"io.github.JcJamet/ia-qa-eval":{"type":"http","url":"https://www.ia-qa.com/mcp/eval"}}}}},"resourceType":"mcp-server","callable":true,"runtime":{"callable":true,"protocols":["mcp"]},"interfaces":[{"protocol":"mcp","transport":"streamable-http","url":"https://www.ia-qa.com/mcp/eval"}],"profile":{"apiVersion":"1","schemaVersion":"1","id":"mcp-server:io.github.JcJamet%2Fia-qa-eval","resourceType":"mcp-server","canonicalUrl":"https://www.agentsjunction.com/api/v1/servers/io.github.JcJamet%2Fia-qa-eval","name":"io.github.JcJamet/ia-qa-eval","description":"LLM & RAG evals: hallucination, eval contracts, semantic tests, tokens & model info. Free, no key.","interfaces":[{"protocol":"mcp","transport":"streamable-http","url":"https://www.ia-qa.com/mcp/eval"}],"capabilities":[],"identity":{"source":"official-registry","websiteUrl":null,"repositoryUrl":"https://github.com/ia-qa-com/mcp","version":"1.0.1"},"verification":{"state":"official","confidence":100},"operations":{"resourceType":"mcp-server","name":"io.github.JcJamet/ia-qa-eval","status":"online","lastCheckedAt":"2026-11-20T08:17:18.437Z","lastSuccessfulCheck":"2026-11-20T08:17:18.437Z","latencyMs":218,"consecutiveFailures":0,"isStale":false,"checks":{"endpointReachable":true,"protocolValid":true,"agentCardValid":null,"mcpHandshakeValid":true},"lastSeen":null,"lastSuccessfulInteraction":null,"lastHealthCheck":"2026-11-20T08:17:18.437Z","source":{"type":"health-check","url":null},"observedAt":"2026-11-20T08:17:18.437Z"},"security":null,"timestamps":{"createdAt":"2026-10-26T21:30:39.179Z","updatedAt":"2026-10-26T21:30:39.179Z","observedAt":"2026-11-20T08:17:18.437Z"},"compatibility":{"serverVersion":"1.0.1","protocolVersions":[{"protocol":"mcp","version":null}]}}}}