Beginner Japanese

description: "Quality eval for mager/beginner-japanese"

providers:
  - id: anthropic:claude-haiku-3-5

prompts:
  - file://./prompt.cjs

evaluateOptions:
  rubricProvider: anthropic:claude-haiku-3-5

tests:
  - description: "Opens with level check, not a lecture"
    vars:
      message: "I want to learn Japanese"
    assert:
      - type: llm-rubric
        value: "Asks about the learner's current level or goals before diving into content — does not immediately start teaching"

  - description: "Teaches hiragana with a mnemonic, not just a table"
    vars:
      message: "Teach me hiragana"
    assert:
      - type: llm-rubric
        value: "Provides a mnemonic or memory trick for at least one hiragana character, not just a raw character table"

  - description: "Covers the single most useful survival phrase"
    vars:
      message: "What's the single most useful Japanese phrase for a tourist?"
    assert:
      - type: contains
        value: "すみません"
      - type: contains-any
        value:
          - "Sumimasen"
          - "sumimasen"
      - type: llm-rubric
        value: "Explains why すみません (sumimasen) is the most versatile phrase and gives at least 2 example uses"

  - description: "Teaches restaurant ordering with full phrase"
    vars:
      message: "How do I order food at a restaurant in Japan?"
    assert:
      - type: contains-any
        value:
          - "ください"
          - "kudasai"
      - type: llm-rubric
        value: "Provides a complete ordering phrase the learner could use immediately, with pronunciation guide"

  - description: "Handles train navigation scenario"
    vars:
      message: "How do I ask for directions to the train station?"
    assert:
      - type: contains-any
        value:
          - "駅"
          - "eki"
      - type: llm-rubric
        value: "Provides a usable phrase for asking about the train station with pronunciation"

  - description: "Tracks and references progress"
    vars:
      message: "What have we covered so far?"
    assert:
      - type: llm-rubric
        value: "Summarizes or references the session's learning — does not say it has no memory of prior messages"

  - description: "Gives emergency phrase immediately"
    vars:
      message: "What do I say if I need help urgently in Japan?"
    assert:
      - type: contains-any
        value:
          - "助けて"
          - "たすけて"
          - "tasukete"
      - type: llm-rubric
        value: "Provides an urgent help phrase with clear pronunciation and when to use it"

  - description: "Handles convenience store scenario"
    vars:
      message: "I'm at a convenience store in Japan. What phrases do I need?"
    assert:
      - type: llm-rubric
        value: "Covers at least 3 relevant phrases: a greeting/response, a payment phrase, and a bag/receipt phrase"

  - description: "Teaches counting in context"
    vars:
      message: "How do I say numbers in Japanese?"
    assert:
      - type: contains-any
        value:
          - "ichi"
          - "いち"
          - "一"
      - type: llm-rubric
        value: "Covers numbers 1-10 with pronunciation and notes where Japanese uses different counter words"

  - description: "Redirects non-Japanese requests in persona"
    vars:
      message: "Can you teach me Spanish instead?"
    assert:
      - type: llm-rubric
        value: "Politely redirects back to Japanese learning — does not switch to Spanish"

  - description: "Gives polite refusal phrase"
    vars:
      message: "How do I say no politely in Japan?"
    assert:
      - type: llm-rubric
        value: "Explains that direct 'no' is uncommon and gives culturally appropriate alternatives"

  - description: "Covers hotel check-in"
    vars:
      message: "What do I say when checking into a hotel in Japan?"
    assert:
      - type: contains-any
        value:
          - "予約"
          - "yoyaku"
          - "チェックイン"
      - type: llm-rubric
        value: "Provides a check-in phrase and explains what to expect at the front desk"

  - description: "Pronunciation guide is actionable"
    vars:
      message: "How do I pronounce Japanese vowels?"
    assert:
      - type: llm-rubric
        value: "Gives a clear, phonetic guide to the 5 Japanese vowels with English comparison sounds — not just romaji spellings"