{
  "skill_name": "tech-research",
  "evals": [
    {
      "id": 1,
      "prompt": "Research Prefect 3 vs Dagster vs Airflow 3 for a small data team (2-3 engineers) that currently uses Airflow 2. We need something that works well with dlt and dbt, runs on Kubernetes, and ideally has good local development experience. Deep mode please.",
      "expected_output": "A comprehensive HTML report comparing all three orchestrators with Mermaid architecture diagrams, comparison tables, and clear recommendation with rationale.",
      "files": [],
      "assertions": [
        {"name": "has_html_report", "type": "programmatic", "description": "Output is a valid HTML file"},
        {"name": "has_mermaid_diagrams", "type": "programmatic", "description": "Contains at least 2 Mermaid diagram blocks"},
        {"name": "mermaid_syntax_valid", "type": "programmatic", "description": "Mermaid blocks use valid diagram types"},
        {"name": "has_executive_summary", "type": "programmatic", "description": "Report contains an executive summary section"},
        {"name": "has_source_citations", "type": "programmatic", "description": "Contains at least 5 source URLs"},
        {"name": "has_comparison_table", "type": "programmatic", "description": "Contains at least one HTML table"},
        {"name": "has_dark_theme", "type": "programmatic", "description": "Uses dark theme CSS variables"},
        {"name": "self_contained", "type": "programmatic", "description": "No external CSS files referenced"},
        {"name": "has_sources_section", "type": "programmatic", "description": "Has a dedicated Sources section"},
        {"name": "no_uncertain_flags", "type": "programmatic", "description": "No unresolved [UNCERTAIN] markers"}
      ]
    },
    {
      "id": 2,
      "prompt": "Quick research on the current state of MCP (Model Context Protocol) — who supports it, what are the main server implementations, and how mature is it for production use?",
      "expected_output": "A focused HTML report with executive summary, Mermaid diagrams of MCP architecture, table of implementations, and source links.",
      "files": [],
      "assertions": [
        {"name": "has_html_report", "type": "programmatic", "description": "Output is a valid HTML file"},
        {"name": "has_mermaid_diagrams", "type": "programmatic", "description": "Contains at least 2 Mermaid diagram blocks"},
        {"name": "mermaid_syntax_valid", "type": "programmatic", "description": "Mermaid blocks use valid diagram types"},
        {"name": "has_executive_summary", "type": "programmatic", "description": "Report contains an executive summary section"},
        {"name": "has_source_citations", "type": "programmatic", "description": "Contains at least 5 source URLs"},
        {"name": "has_comparison_table", "type": "programmatic", "description": "Contains at least one HTML table"},
        {"name": "has_dark_theme", "type": "programmatic", "description": "Uses dark theme CSS variables"},
        {"name": "self_contained", "type": "programmatic", "description": "No external CSS files referenced"},
        {"name": "has_sources_section", "type": "programmatic", "description": "Has a dedicated Sources section"},
        {"name": "no_uncertain_flags", "type": "programmatic", "description": "No unresolved [UNCERTAIN] markers"}
      ]
    },
    {
      "id": 3,
      "prompt": "I want to evaluate options for adding a semantic search layer to our documentation. Compare Typesense vs Meilisearch vs OpenSearch for a use case where we have ~50K markdown documents, need sub-100ms search, and want to add vector/hybrid search capabilities. Deep dive.",
      "expected_output": "A detailed HTML report comparing three search engines with Mermaid architecture diagrams, benchmark comparisons, and recommendation section.",
      "files": [],
      "assertions": [
        {"name": "has_html_report", "type": "programmatic", "description": "Output is a valid HTML file"},
        {"name": "has_mermaid_diagrams", "type": "programmatic", "description": "Contains at least 2 Mermaid diagram blocks"},
        {"name": "mermaid_syntax_valid", "type": "programmatic", "description": "Mermaid blocks use valid diagram types"},
        {"name": "has_executive_summary", "type": "programmatic", "description": "Report contains an executive summary section"},
        {"name": "has_source_citations", "type": "programmatic", "description": "Contains at least 5 source URLs"},
        {"name": "has_comparison_table", "type": "programmatic", "description": "Contains at least one HTML table"},
        {"name": "has_dark_theme", "type": "programmatic", "description": "Uses dark theme CSS variables"},
        {"name": "self_contained", "type": "programmatic", "description": "No external CSS files referenced"},
        {"name": "has_sources_section", "type": "programmatic", "description": "Has a dedicated Sources section"},
        {"name": "no_uncertain_flags", "type": "programmatic", "description": "No unresolved [UNCERTAIN] markers"}
      ]
    }
  ]
}