{
  "permissions": {
    "allow": [
      "WebFetch"
    ],
    "deny": []
  }
}

name: Bug Report
description: File a bug report to help us improve OpenBench
title: "[Bug]: "
labels: ["bug"]
body:
  - type: markdown
    attributes:
      value: |
        Thanks for taking the time to fill out this bug report!

  - type: input
    id: version
    attributes:
      label: OpenBench Version
      description: What version of OpenBench are you running?
      placeholder: "0.3.0"
    validations:
      required: true

  - type: textarea
    id: what-happened
    attributes:
      label: What happened?
      description: A clear and concise description of what the bug is.
      placeholder: Tell us what you see!
    validations:
      required: true

  - type: textarea
    id: expected
    attributes:
      label: Expected behavior
      description: A clear and concise description of what you expected to happen.
    validations:
      required: true

  - type: textarea
    id: reproduce
    attributes:
      label: Steps to reproduce
      description: Steps to reproduce the behavior
      placeholder: |
        1. Run command '...'
        2. With arguments '...'
        3. See error
    validations:
      required: true

  - type: input
    id: command
    attributes:
      label: Command that failed
      description: The exact command you ran that caused the issue
      placeholder: "bench eval mmlu --model groq/llama-3.1-70b --limit 10"

  - type: textarea
    id: logs
    attributes:
      label: Error logs
      description: Please copy and paste any relevant log output or error messages
      render: shell

  - type: dropdown
    id: os
    attributes:
      label: Operating System
      options:
        - macOS
        - Linux
        - Windows
        - Other
    validations:
      required: true

  - type: input
    id: python-version
    attributes:
      label: Python Version
      placeholder: "3.11.5"
    validations:
      required: true

  - type: textarea
    id: additional-context
    attributes:
      label: Additional context
      description: Add any other context about the problem here (environment, model provider, etc.)

name: Feature Request
description: Suggest an idea for OpenBench
title: "[Feature]: "
labels: ["enhancement"]
body:
  - type: markdown
    attributes:
      value: |
        Thanks for suggesting a new feature for OpenBench!

  - type: textarea
    id: problem
    attributes:
      label: Is your feature request related to a problem?
      description: A clear and concise description of what the problem is.
      placeholder: "I'm always frustrated when..."

  - type: textarea
    id: solution
    attributes:
      label: Describe the solution you'd like
      description: A clear and concise description of what you want to happen.
    validations:
      required: true

  - type: dropdown
    id: feature-type
    attributes:
      label: What are you requesting?
      options:
        - New benchmark/evaluation
        - New model provider
        - CLI enhancement
        - Performance improvement
        - Documentation
        - API/SDK feature
        - Integration (CI/CD, tools)
        - Export/import functionality
        - Other
    validations:
      required: true

  - type: textarea
    id: alternatives
    attributes:
      label: Describe alternatives you've considered
      description: A clear and concise description of any alternative solutions or features you've considered.

  - type: textarea
    id: use-case
    attributes:
      label: Use case
      description: Describe your specific use case and how this feature would help you or others.
    validations:
      required: true

  - type: textarea
    id: additional-context
    attributes:
      label: Additional context
      description: Add any other context, screenshots, or examples about the feature request here.

version: 2
updates:
  # Enable version updates for GitHub Actions
  - package-ecosystem: "github-actions"
    directory: "/"
    schedule:
      interval: "weekly"
      day: "monday"
      time: "03:00"
    labels:
      - "dependencies"
      - "github-actions"
    open-pull-requests-limit: 10
    groups:
      actions:
        patterns:
          - "*"

  # Enable version updates for Python dependencies via pip
  - package-ecosystem: "pip"
    directory: "/"
    schedule:
      interval: "weekly"
      day: "monday"
      time: "03:00"
    labels:
      - "dependencies"
      - "python"
    open-pull-requests-limit: 10
    # Group all Python dependency updates together
    groups:
      python-dependencies:
        patterns:
          - "*"

name: CI

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]

# Cancel in-progress runs when a new run is queued on the same branch
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

# Set environment variables for consistency
env:
  UV_CACHE_TTL_SECONDS: 604800  # 1 week
  UV_HTTP_TIMEOUT: 600  # 10 minutes

jobs:
  # Single job for linting and type checking to reduce overhead
  quality-checks:
    name: Quality Checks (Python ${{ matrix.python-version }})
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]  # Use latest for quality checks
    steps:
      - uses: actions/checkout@v4

      - name: Install uv
        uses: astral-sh/setup-uv@v5
        with:
          version: "latest"
          enable-cache: true
          cache-dependency-glob: |
            **/pyproject.toml
            **/uv.lock

      - name: Set up Python ${{ matrix.python-version }}
        run: uv python install ${{ matrix.python-version }}

      - name: Install dependencies
        run: uv sync --group dev

      - name: Run ruff format check
        run: uv run ruff format --check .

      - name: Run ruff lint
        run: uv run ruff check .

      - name: Run type checking
        run: uv run mypy .

  test:
    name: Tests (Python ${{ matrix.python-version }})
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.10", "3.11", "3.12", "3.13"]
    steps:
      - uses: actions/checkout@v4

      - name: Install uv
        uses: astral-sh/setup-uv@v5
        with:
          version: "latest"
          enable-cache: true
          cache-dependency-glob: |
            **/pyproject.toml
            **/uv.lock

      - name: Set up Python ${{ matrix.python-version }}
        run: uv python install ${{ matrix.python-version }}

      - name: Install dependencies
        run: uv sync --group dev

      - name: Run unit tests with coverage
        run: |
          uv run pytest -m "not integration" --cov=openbench --cov-report=term-missing

  integration-test:
    name: Integration Tests
    runs-on: ubuntu-latest
    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository
    steps:
      - uses: actions/checkout@v4

      - name: Install uv
        uses: astral-sh/setup-uv@v5
        with:
          version: "latest"
          enable-cache: true
          cache-dependency-glob: |
            **/pyproject.toml
            **/uv.lock

      - name: Set up Python
        run: uv python install 3.12

      - name: Install dependencies
        run: uv sync --group dev

      - name: Run integration tests
        env:
          GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
        run: |
          uv run pytest -m integration -v
        if: env.GROQ_API_KEY != ''

  # Security scanning with pip-audit
  security:
    name: Security Scan
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Install uv
        uses: astral-sh/setup-uv@v5
        with:
          version: "latest"
          enable-cache: true

      - name: Set up Python
        run: uv python install 3.12

      - name: Install dependencies
        run: uv sync --group dev

      - name: Run security audit
        run: |
          uv pip install pip-audit
          uv run pip-audit --desc
        continue-on-error: true  # Don't fail the build on vulnerabilities

  # All checks must pass
  all-checks:
    name: All Checks Pass
    runs-on: ubuntu-latest
    needs: [quality-checks, test, integration-test, security]
    if: always()
    steps:
      - name: Verify all checks passed
        run: |
          # Quality checks and test must always pass
          if [[ "${{ needs.quality-checks.result }}" != "success" || 
                "${{ needs.test.result }}" != "success" ]]; then
            echo "One or more required checks failed"
            exit 1
          fi
          
          # Integration test can be skipped (for external PRs) or must succeed
          if [[ "${{ needs.integration-test.result }}" != "success" && 
                "${{ needs.integration-test.result }}" != "skipped" ]]; then
            echo "Integration tests failed"
            exit 1
          fi
          
          # Security is continue-on-error, so we don't check it
          echo "All required checks passed!"

name: Claude Code Review

on:
  pull_request:
    types: [opened, synchronize]
    # Optional: Only run on specific file changes
    # paths:
    #   - "src/**/*.ts"
    #   - "src/**/*.tsx"
    #   - "src/**/*.js"
    #   - "src/**/*.jsx"

jobs:
  claude-review:
    # Optional: Filter by PR author
    # if: |
    #   github.event.pull_request.user.login == 'external-contributor' ||
    #   github.event.pull_request.user.login == 'new-developer' ||
    #   github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR'
    
    runs-on: ubuntu-latest
    continue-on-error: true
    permissions:
      contents: read
      pull-requests: read
      issues: read
      id-token: write
    
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 1

      - name: Run Claude Code Review
        id: claude-review
        uses: anthropics/claude-code-action@beta
        with:
          anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
          
          # Optional: Specify model (defaults to Claude Sonnet 4, uncomment for Claude Opus 4.1)
          model: "claude-opus-4-1-20250805"
          
          # Direct prompt for automated review (no @claude mention needed)
          direct_prompt: |
            Review this PR like a chill staff engineer. Focus on NEW changes only and think DRY.

            ONLY bring up things if they're actually important - you don't need to comment on every category or find something to say. Quality over quantity.

            PRIORITY ORDER:
            - Critical bugs, security issues, performance problems  
            - Code quality: proper separation of concerns, clear naming, best practices
            - Missing tests for new functionality
            - Suggestions for cleaner patterns (only if they meaningfully improve the code)

            Be constructive and EXTREMELY concise. Think "would a 10x engineer actually care about this?" If it's not blocking or genuinely helpful, skip it entirely. Respect separation of concerns.

            Stay chill: flag real issues, suggest improvements where they add value, but don't nitpick or over-engineer. Sometimes the best review is a LGTM. ultrathink.
          
          # Optional: Customize review based on file types
          # direct_prompt: |
          #   Review this PR focusing on:
          #   - For TypeScript files: Type safety and proper interface usage
          #   - For API endpoints: Security, input validation, and error handling
          #   - For React components: Performance, accessibility, and best practices
          #   - For tests: Coverage, edge cases, and test quality
          
          # Optional: Different prompts for different authors
          # direct_prompt: |
          #   ${{ github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR' && 
          #   'Welcome! Please review this PR from a first-time contributor. Be encouraging and provide detailed explanations for any suggestions.' ||
          #   'Please provide a thorough code review focusing on our coding standards and best practices.' }}
          
          # Optional: Add specific tools for running tests or linting
          # allowed_tools: "Bash(npm run test),Bash(npm run lint),Bash(npm run typecheck)"
          
          # Optional: Skip review for certain conditions
          # if: |
          #   !contains(github.event.pull_request.title, '[skip-review]') &&
          #   !contains(github.event.pull_request.title, '[WIP]')

name: Claude Code

on:
  issue_comment:
    types: [created]
  pull_request_review_comment:
    types: [created]
  issues:
    types: [opened, assigned]
  pull_request_review:
    types: [submitted]

jobs:
  claude:
    if: |
      (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||
      (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
      (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
      (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
    runs-on: ubuntu-latest
    permissions:
      contents: read
      pull-requests: read
      issues: read
      id-token: write
      actions: read # Required for Claude to read CI results on PRs
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 1

      - name: Run Claude Code
        id: claude
        uses: anthropics/claude-code-action@beta
        with:
          anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}

          # This is an optional setting that allows Claude to read CI results on PRs
          additional_permissions: |
            actions: read
          
          # Optional: Specify model (defaults to Claude Sonnet 4, uncomment for Claude Opus 4.1)
          # model: "claude-opus-4-1-20250805"
          
          # Optional: Customize the trigger phrase (default: @claude)
          # trigger_phrase: "/claude"
          
          # Optional: Trigger when specific user is assigned to an issue
          # assignee_trigger: "claude-bot"
          
          # Optional: Allow Claude to run specific commands
          # allowed_tools: "Bash(npm install),Bash(npm run build),Bash(npm run test:*),Bash(npm run lint:*)"
          
          # Optional: Add custom instructions for Claude to customize its behavior for your project
          # custom_instructions: |
          #   Follow our coding standards
          #   Ensure all new code has tests
          #   Use TypeScript for new files
          
          # Optional: Custom environment variables for Claude
          # claude_env: |
          #   NODE_ENV: test

name: Dependency Check

on:
  schedule:
    # Run every Monday at 9 AM PST
    - cron: '0 17 * * 1'
  workflow_dispatch:  # Allow manual trigger

jobs:
  check-dependencies:
    name: Check for Outdated Dependencies
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Install uv
        uses: astral-sh/setup-uv@v5
        with:
          version: "latest"
          enable-cache: true

      - name: Set up Python
        run: uv python install 3.12

      - name: Check for outdated packages
        run: |
          echo "## Checking for outdated packages..."
          uv pip list --outdated || true
          
      - name: Create issue if dependencies are outdated
        if: failure()
        uses: actions/github-script@v7
        with:
          script: |
            github.rest.issues.create({
              owner: context.repo.owner,
              repo: context.repo.repo,
              title: 'Dependencies are outdated',
              body: 'The dependency check workflow found outdated packages. Please review and update.',
              labels: ['dependencies', 'maintenance']
            })

name: Release Please

on:
  push:
    branches: [ main ]
  pull_request:
      branches: [ main ]

permissions:
  contents: write   # allow tagging and releases
  pull-requests: write   # allow managing release PRs
  issues: write   # allow creating and managing labels

concurrency:
  group: release-please
  cancel-in-progress: true

jobs:
  release:
    runs-on: ubuntu-latest
    outputs:
      release_created: ${{ steps.release.outputs.release_created }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 0   # fetch full history for accurate changelog generation

      - name: Run Release Please
        uses: googleapis/release-please-action@v4
        id: release
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          config-file: release-please-config.json
          manifest-file: .release-please-manifest.json

  publish:
    runs-on: ubuntu-latest
    needs: release
    if: ${{ needs.release.outputs.release_created }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Install UV
        uses: astral-sh/setup-uv@v4

      - name: Set up environment
        run: |
          uv venv
          source .venv/bin/activate

      - name: Build package
        run: |
          source .venv/bin/activate
          uv build

      - name: Publish to PyPI
        run: |
          source .venv/bin/activate
          uv publish --token ${{ secrets.PYPI_API_TOKEN }}

#####################################
#       DO NOT EDIT DIRECTLY.       #
# This file is managed by Terraform #
#####################################

name: "Close stale PRs"
on:
  schedule:
    - cron: "30 1 * * *"

jobs:
  stale:
    runs-on: ubuntu-latest
    # Read repo and write to PRs
    permissions:
      contents: read
      pull-requests: write
      issues: write
    steps:
      - uses: actions/stale@v9
        with:
          stale-pr-message: "This PR is stale because it has been open for 30 days with no activity. Remove stale label or comment or this will be closed in 7 days."
          close-pr-message: "This PR was closed because it has been stalled for 7 days with no activity."
          days-before-pr-stale: 30
          days-before-pr-close: 7
          exempt-pr-labels: "dependencies,security"
          operations-per-run: 60 # Default is 30

tmp
tmp*
*.pyc
__pycache__
temp
*.eval
.venv/
logs/
*.log
.mypy_cache/
*.egg-info/
**/CLAUDE.local.md
.DS_Store
.*cache/
.*cache
.coverage*
.claude/settings.local.json
local-prompts.md
.env

repos:
  - repo: https://github.com/astral-sh/ruff-pre-commit
    rev: v0.3.0
    hooks:
      - id: ruff
        name: ruff check
        args: [--fix]
      - id: ruff-format
        name: ruff format
  - repo: local
    hooks:
      - id: mypy
        name: mypy
        entry: .venv/bin/mypy
        language: system
        types: [python]
        args: [.]
        pass_filenames: false

3.13.5

{
    ".": "0.3.0"
}  

{
    "python.testing.pytestArgs": [
        "tests"
    ],
    "python.testing.unittestEnabled": false,
    "python.testing.pytestEnabled": true
}

cff-version: 1.2.0
message: "If you use this software, please cite it as below."
type: software
authors:
- family-names: "Sah"
  given-names: "Aarush"
  orcid: "https://orcid.org/0009-0004-6429-8982"
title: "OpenBench: Provider-agnostic, open-source evaluation infrastructure for language models"
version: 0.3.0
date-released: 2025-07-31
url: "https://openbench.dev"
repository-code: "https://github.com/groq/openbench"
keywords:
  - "machine learning"
  - "evaluation"
  - "benchmarking"
  - "language models"
  - "LLM"
  - "artificial intelligence"
license: MIT

* @AarushSah

include README.md
include LICENSE.md
include pyproject.toml
recursive-include src/openbench *
include src/openbench/py.typed

[build-system]
requires = ["setuptools>=61.0"]
build-backend = "setuptools.build_meta"

[project]
name = "openbench"
version = "0.3.0"
requires-python = ">=3.10"
description = "OpenBench - open source, replicable, and standardized evaluation infrastructure"
readme = "README.md"
license = {text = "MIT"}
authors = [
    { name="Aarush Sah" },
    { name="Groq" }
]
dependencies = [
    "datasets>=3.6.0",
    "groq>=0.30.0",
    "inspect-ai==0.3.115",
    "jsonschema>=4.23.0",
    "openai>=1.97.1",
    "pydantic-settings>=2.9.1",
    "scicode",
    "scipy>=1.15.3",
    "tiktoken>=0.11.0",
    "typer>=0.15.3",
]

[project.urls]
Homepage = "https://github.com/groq/openbench"
Repository = "https://github.com/groq/openbench"

[project.scripts]
bench = "openbench._cli:main"
openbench = "openbench._cli:main"

[project.entry-points.inspect_ai]
openbench = "openbench._registry"

[tool.setuptools]
package-dir = {"" = "src"}

[tool.setuptools.packages.find]
where = ["src"]
include = ["openbench*"]

[tool.pytest.ini_options]
minversion = "8.0"
addopts = "-ra -q --strict-markers"
testpaths = ["tests"]
pythonpath = ["src", "."]
asyncio_mode = "auto"
asyncio_default_fixture_loop_scope = "function"
markers = [
    "integration: marks tests that require external services (deselect with '-m \"not integration\"')",
]

[tool.coverage.run]
source = ["src/openbench"]
omit = ["tests/*", "**/__init__.py", "**/conftest.py"]

[tool.coverage.report]
exclude_lines = [
    "pragma: no cover",
    "def __repr__",
    "raise AssertionError",
    "raise NotImplementedError",
    "if __name__ == .__main__.:",
    "if TYPE_CHECKING:",
]
precision = 2
show_missing = true

[tool.uv.sources]
scicode = { git = "https://github.com/TheFloatingString/SciCode-fork.git", rev = "4f20d721ba3e2227196262083b9b7a70484d54f7" }

[dependency-groups]
dev = [
    "mypy>=1.15.0",
    "pre-commit>=4.2.0",
    "pytest>=8.3.5",
    "pytest-asyncio==0.24.0",
    "pytest-cov>=4.1.0",
    "ruff>=0.11.9",
    "types-jsonschema>=4.0.0",
    "types-pyyaml>=6.0.12.20250809",
]

{
    "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json",
    "packages": {
      ".": {
        "release-type": "python",
        "package-name": "openbench",
        "include-v-in-tag": true,
        "include-component-in-tag": false,
        "changelog-path": "CHANGELOG.md",
        "extra-files": [
          {
            "type": "toml",
            "path": "uv.lock",
            "jsonpath": "$.package[?(@.name==\"openbench\")].version"
          }
        ],
        "changelog-sections": [
          { "type": "feat", "section": "Features" },
          { "type": "fix", "section": "Bug Fixes" },
          { "type": "docs", "section": "Documentation" },
          { "type": "chore", "section": "Chores" },
          { "type": "refactor", "section": "Refactor" },
          { "type": "test", "section": "Tests" },
          { "type": "ci", "section": "CI" }
        ]
      }
    }
  }
  

import typer
from openbench._cli.eval_command import run_eval
from openbench._cli.eval_retry_command import run_eval_retry
from openbench._cli.list_command import list_evals
from openbench._cli.describe_command import describe_eval
from openbench._cli.view_command import run_view

app = typer.Typer(rich_markup_mode="rich")

app.command("list")(list_evals)
app.command("describe")(describe_eval)
app.command("eval")(run_eval)
app.command("eval-retry")(run_eval_retry)
app.command("view")(run_view)


def main() -> None:
    app()


if __name__ == "__main__":
    main()

"""
Describe command for benchmark evaluations.
"""

from rich.console import Console
from rich.panel import Panel
from rich.table import Table
from rich.box import ROUNDED

from openbench.eval_config import get_eval_config
from openbench.config import get_all_benchmarks
from openbench._cli.utils import get_category_display_name


def describe_eval(name: str) -> None:
    """Show detailed information about a specific evaluation."""
    console = Console()

    if name not in get_all_benchmarks():
        console.print(f"\n[red]Unknown evaluation: {name}[/red]")

        # Suggest similar names
        all_names = list(get_all_benchmarks().keys())
        similar = [n for n in all_names if name.lower() in n.lower()]
        if similar:
            console.print("\nDid you mean one of these?")
            for s in similar[:5]:
                console.print(f"   • {s}")
        console.print()
        return

    # Get both static and dynamic config
    config = get_eval_config(name, load_dynamic=True)
    if not config:
        console.print(f"\n[red]Failed to load configuration for {name}[/red]\n")
        return

    # Header
    console.print()
    console.print(
        Panel(f"[bold blue]{config.name}[/bold blue]", expand=False, box=ROUNDED)
    )

    # Static metadata section
    console.print("\n[bold yellow]Metadata[/bold yellow]")
    console.print("─" * 40)

    static_table = Table(show_header=False, show_lines=False, padding=(0, 2), box=None)
    static_table.add_column("Property", style="cyan", width=15)
    static_table.add_column("Value", style="white")

    static_table.add_row("Description", config.description)
    static_table.add_row("Category", get_category_display_name(config.category))
    static_table.add_row("Command", f"[bold]bench eval {name}[/bold]")

    if config.tags:
        tag_str = " ".join(f"[blue]#{tag}[/blue]" for tag in config.tags)
        static_table.add_row("Tags", tag_str)

    console.print(static_table)

    # Dynamic configuration section
    if config._dynamic_loaded:
        console.print("\n[bold yellow]Configuration[/bold yellow]")
        console.print("─" * 40)

        config_table = Table(
            show_header=False, show_lines=False, padding=(0, 2), box=None
        )
        config_table.add_column("Property", style="cyan", width=15)
        config_table.add_column("Value", style="white")

        # Show key configuration values
        if config.epochs is not None:
            config_table.add_row("Epochs", str(config.epochs))

        # Show all GenerateConfig fields that have values
        config_dict = config.__dict__

        # Get all fields from config dict, excluding private/internal fields
        for field, value in sorted(config_dict.items()):
            # Skip private fields, None values, and non-GenerateConfig fields
            if (
                field.startswith("_")
                or value is None
                or field
                in [
                    "name",
                    "description",
                    "category",
                    "tags",
                    "epochs",
                    "sandbox",
                    "dataset_size",
                    "task_args",
                ]
            ):
                continue

            # Format the value nicely
            if isinstance(value, float):
                formatted_value = f"{value:.2f}" if value < 10 else f"{value:,.0f}"
            elif isinstance(value, int):
                formatted_value = f"{value:,}" if value > 999 else str(value)
            else:
                formatted_value = str(value)

            # Pretty field name
            field_name = field.replace("_", " ").title()
            config_table.add_row(field_name, formatted_value)

        if config.sandbox:
            config_table.add_row("Sandbox", config.sandbox)

        if config.dataset_size:
            config_table.add_row("Dataset Size", f"{config.dataset_size:,}")

        console.print(config_table)

        # Show task-specific arguments if any
        if config.task_args:
            console.print("\n[bold yellow]Task Arguments[/bold yellow]")
            console.print("─" * 40)

            args_table = Table(
                show_header=False, show_lines=False, padding=(0, 2), box=None
            )
            args_table.add_column("Argument", style="cyan", width=15)
            args_table.add_column("Default Value", style="white")

            for arg_name, arg_value in sorted(config.task_args.items()):
                # Pretty argument name
                display_name = arg_name.replace("_", " ").title()

                # Format the value
                if isinstance(arg_value, float):
                    formatted_value = (
                        f"{arg_value:.2f}" if arg_value < 10 else f"{arg_value:,.0f}"
                    )
                elif isinstance(arg_value, int):
                    formatted_value = (
                        f"{arg_value:,}" if arg_value > 999 else str(arg_value)
                    )
                elif isinstance(arg_value, bool):
                    formatted_value = "Yes" if arg_value else "No"
                else:
                    formatted_value = str(arg_value)

                args_table.add_row(display_name, formatted_value)

            console.print(args_table)
    else:
        console.print("\n[red]Failed to load dynamic configuration[/red]")

    # Footer
    console.print()
    console.print(f"[dim]Run with: [bold]bench eval {name}[/bold][/dim]")
    console.print()

from typing import Optional, List, Dict, Annotated, Tuple, Union
from enum import Enum
import sys
import time
import typer
from inspect_ai import eval
from inspect_ai.model import Model

from openbench.config import load_task
from openbench.monkeypatch.display_results_patch import patch_display_results
from openbench._cli.utils import parse_cli_args


class SandboxType(str, Enum):
    """Type of environment to run evaluations in."""

    LOCAL = "local"
    DOCKER = "docker"


class DisplayType(str, Enum):
    """Display type for evaluation progress."""

    FULL = "full"
    CONVERSATION = "conversation"
    RICH = "rich"
    PLAIN = "plain"
    NONE = "none"


class ReasoningEffortLevel(str, Enum):
    """Reasoning effort level."""

    LOW = "low"
    MEDIUM = "medium"
    HIGH = "high"


def parse_limit(value: Optional[str]) -> Optional[Union[int, Tuple[int, int]]]:
    """Parse the limit parameter which can be an int or a tuple of ints.

    Args:
        value: The value passed to the --limit option.

    Returns:
        Parsed limit value: int, tuple of (start, end), or None.

    Raises:
        typer.BadParameter: If the input format is incorrect.
    """
    if value is None:
        return None

    try:
        if "," in value:
            start, end = map(int, value.split(","))
            return (start, end)
        return int(value)
    except ValueError:
        raise typer.BadParameter(
            "Limit must be an integer or two integers separated by a comma"
        )


def validate_model_name(model: str, context: str = "") -> None:
    """Validate a model name format.

    Args:
        model: Model name to validate
        context: Additional context for error message

    Raises:
        typer.BadParameter: If model name format is invalid
    """
    if not model or "/" not in model:
        raise typer.BadParameter(
            f"Invalid model name format{context}: {model}. Expected format: provider/model-name"
        )


def validate_model_role(model_role: Optional[str]) -> Dict[str, str | Model]:
    """Validate and parse model role string.

    Args:
        model_role: Optional string in format 'role=model'

    Returns:
        Dictionary mapping role to model name

    Raises:
        typer.BadParameter: If model_role format is invalid
    """
    if not model_role:
        return {}

    try:
        role, model = model_role.split("=")
        if not role or not model:
            raise ValueError("Model role must be in format 'role=model'")
        validate_model_name(model, f" for role '{role}'")
        return {role: model}
    except ValueError as e:
        raise typer.BadParameter(str(e))


def run_eval(
    benchmarks: Annotated[
        List[str],
        typer.Argument(
            help="Benchmark(s) to run. Can be a built-in name (e.g. mmlu) or a path to a local eval directory/file containing __metadata__",
            envvar="BENCH_BENCHMARKS",
        ),
    ],
    model: Annotated[
        List[str],
        typer.Option(
            help="Model(s) to evaluate. Equivalent to --model-role candidate=<model>",
            envvar="BENCH_MODEL",
        ),
    ] = ["groq/meta-llama/llama-4-scout-17b-16e-instruct"],
    max_connections: Annotated[
        Optional[int],
        typer.Option(
            help="Maximum number of parallel requests to the model",
            envvar="BENCH_MAX_CONNECTIONS",
        ),
    ] = 10,
    model_base_url: Annotated[
        Optional[str],
        typer.Option(help="Base URL for model(s)", envvar="BENCH_MODEL_BASE_URL"),
    ] = None,
    model_role: Annotated[
        List[str],
        typer.Option(
            help="Model role(s). For example, --model-role grader=groq/meta-llama/llama-4-scout-17b-16e-instruct. Can be specified multiple times.",
            envvar="BENCH_MODEL_ROLE",
        ),
    ] = [],
    m: Annotated[
        List[str],
        typer.Option(
            "-M",
            help="One or more native model arguments (e.g. -M arg=value)",
            envvar="BENCH_MODEL_ARGS",
        ),
    ] = [],
    t: Annotated[
        List[str],
        typer.Option(
            "-T",
            help="One or more task arguments (e.g. -T arg=value)",
            envvar="BENCH_TASK_ARGS",
        ),
    ] = [],
    logfile: Annotated[
        Optional[str],
        typer.Option(help="Output file for results", envvar="BENCH_OUTPUT"),
    ] = None,
    sandbox: Annotated[
        Optional[SandboxType],
        typer.Option(
            help="Environment to run the evaluation in (local or docker)",
            case_sensitive=False,
            envvar="BENCH_SANDBOX",
        ),
    ] = None,
    epochs: Annotated[
        int,
        typer.Option(
            help="Number of epochs to run each evaluation", envvar="BENCH_EPOCHS"
        ),
    ] = 1,
    limit: Annotated[
        Optional[str],
        typer.Option(
            help="Limit evaluated samples (single number or start,end)",
            envvar="BENCH_LIMIT",
        ),
    ] = None,
    fail_on_error: Annotated[
        Optional[float],
        typer.Option(
            help="Failure threshold for sample errors. If between 0 and 1, it is interpreted as a percentage of samples that can fail. If greater than 1, it is interpreted as a fixed number of samples that can fail",
            envvar="BENCH_FAIL_ON_ERROR",
        ),
    ] = None,
    message_limit: Annotated[
        Optional[int],
        typer.Option(
            help="Maximum number of messages one sample can run",
            envvar="BENCH_MESSAGE_LIMIT",
        ),
    ] = None,
    max_subprocesses: Annotated[
        Optional[int],
        typer.Option(
            help="Maximum number of parallel subprocesses",
            envvar="BENCH_MAX_SUBPROCESSES",
        ),
    ] = None,
    log_samples: Annotated[
        Optional[bool],
        typer.Option(
            help="Log detailed samples and scores",
            envvar="BENCH_LOG_SAMPLES",
        ),
    ] = None,
    log_images: Annotated[
        Optional[bool],
        typer.Option(
            help="Log base64 encoded images",
            envvar="BENCH_LOG_IMAGES",
        ),
    ] = None,
    log_buffer: Annotated[
        Optional[int],
        typer.Option(
            help="Number of samples to buffer before writing to log",
            envvar="BENCH_LOG_BUFFER",
        ),
    ] = 10,
    score: Annotated[
        bool,
        typer.Option(
            help="Grade the benchmark, or leave unscored",
            envvar="BENCH_SCORE",
        ),
    ] = True,
    temperature: Annotated[
        Optional[float],
        typer.Option(
            help="Model temperature",
            envvar="BENCH_TEMPERATURE",
        ),
    ] = None,
    top_p: Annotated[
        Optional[float],
        typer.Option(
            help="Model top-p",
            envvar="BENCH_TOP_P",
        ),
    ] = None,
    max_tokens: Annotated[
        Optional[int],
        typer.Option(
            help="Maximum tokens for model response",
            envvar="BENCH_MAX_TOKENS",
        ),
    ] = None,
    seed: Annotated[
        Optional[int],
        typer.Option(
            help="Seed for deterministic generation",
            envvar="BENCH_SEED",
        ),
    ] = None,
    display: Annotated[
        Optional[DisplayType],
        typer.Option(
            help="Display type for evaluation progress",
            envvar="BENCH_DISPLAY",
            case_sensitive=False,
        ),
    ] = None,
    timeout: Annotated[
        Optional[int],
        typer.Option(
            help="Timeout for each request to the model API in seconds",
            envvar="BENCH_TIMEOUT",
        ),
    ] = 10000,
    reasoning_effort: Annotated[
        Optional[ReasoningEffortLevel],
        typer.Option(
            help="Reasoning effort level. used for reasoning models like openai/o3",
            envvar="BENCH_REASONING_EFFORT",
            case_sensitive=False,
        ),
    ] = None,
    debug: Annotated[
        bool,
        typer.Option(
            "--debug",
            help="Enable debug mode with full stack traces",
            envvar="BENCH_DEBUG",
        ),
    ] = False,
    hub_repo: Annotated[
        Optional[str],
        typer.Option(
            help=(
                "Target Hub dataset repo (e.g. username/openbench-logs). "
                "If provided, logs will be exported to this dataset"
            ),
            envvar="BENCH_HUB_REPO",
        ),
    ] = None,
    hub_private: Annotated[
        Optional[bool],
        typer.Option(
            help="Create/update the Hub dataset as private",
            envvar="BENCH_HUB_PRIVATE",
        ),
    ] = False,
    alpha: Annotated[
        bool,
        typer.Option(
            "--alpha",
            help="Allow running experimental/alpha benchmarks",
            envvar="BENCH_ALPHA",
        ),
    ] = False,
) -> None:
    """
    Run a benchmark on a model.
    """
    # Parse model and task arguments
    model_args = parse_cli_args(m) if m else {}
    task_args = parse_cli_args(t) if t else {}

    # Validate and aggregate model_role(s) into a dict
    role_models = {}
    for mr in model_role:
        parsed = validate_model_role(mr)
        for k, v in parsed.items():
            if k in role_models:
                raise typer.BadParameter(f"Duplicate model role: {k}")
            role_models[k] = v

    # Check for mutual exclusivity between --model and --model-role candidate
    if model and "candidate" in role_models:
        raise typer.BadParameter(
            "Cannot specify both --model and --model-role candidate=<model>"
        )

    # Validate model names
    for model_name in model:
        validate_model_name(model_name)

    # Load tasks from registry
    tasks = []
    for benchmark in benchmarks:
        try:
            task = load_task(benchmark, allow_alpha=alpha)
            tasks.append(task)
        except (ValueError, ImportError, AttributeError) as e:
            raise typer.BadParameter(str(e))

    # Monkey patch FileRecorder log file name if logfile is provided
    if logfile:
        from openbench.monkeypatch.file_recorder_logfile_patch import (
            patch_file_recorder_logfile,
        )

        patch_file_recorder_logfile(logfile)

    # Parse limit string to int or tuple
    parsed_limit = parse_limit(limit)

    # Apply display patch
    patch_display_results()

    # Capture start time to locate logs created by this run
    start_time = time.time()

    try:
        eval(
            tasks=tasks,
            model=model,
            max_connections=max_connections,
            model_base_url=model_base_url,
            model_args=model_args,
            model_roles=role_models if role_models else None,
            task_args=task_args,
            epochs=epochs,
            limit=parsed_limit,
            fail_on_error=fail_on_error,
            message_limit=message_limit,
            max_subprocesses=max_subprocesses,
            log_samples=log_samples,
            log_images=log_images,
            log_buffer=log_buffer,
            score=score,
            temperature=temperature,
            top_p=top_p,
            max_tokens=max_tokens,
            seed=seed,
            display=display.value if display else None,
            timeout=timeout,
            reasoning_effort=reasoning_effort.value if reasoning_effort else None,
            sandbox=sandbox,
        )

        typer.echo("Evaluation complete!")

        if hub_repo:
            from openbench._cli.export import export_logs_to_hub

            export_logs_to_hub(
                logfile=logfile,
                start_time=start_time,
                hub_repo=hub_repo,
                hub_private=hub_private,
            )
    except Exception as e:
        if debug:
            # In debug mode, let the full stack trace show
            raise
        else:
            # In normal mode, show clean error message
            error_msg = str(e)
            typer.secho(f"\n❌ Error: {error_msg}", fg=typer.colors.RED, err=True)
            typer.secho(
                "\nFor full stack trace, run with --debug flag",
                fg=typer.colors.CYAN,
                err=True,
            )
            sys.exit(1)

from typing import List, Optional, Annotated
import sys
import typer
from inspect_ai import eval_retry
from inspect_ai.log._file import log_file_info
from inspect_ai._util.file import filesystem
from openbench.monkeypatch.display_results_patch import patch_display_results


def run_eval_retry(
    log_files: Annotated[
        List[str],
        typer.Argument(help="Log file(s) to retry failed evaluations from"),
    ],
    max_connections: Annotated[
        Optional[int],
        typer.Option(
            help="Maximum number of parallel requests to the model",
            envvar="BENCH_MAX_CONNECTIONS",
        ),
    ] = None,
    max_subprocesses: Annotated[
        Optional[int],
        typer.Option(
            help="Maximum number of parallel subprocesses",
            envvar="BENCH_MAX_SUBPROCESSES",
        ),
    ] = None,
    fail_on_error: Annotated[
        Optional[float],
        typer.Option(
            help="Failure threshold for sample errors. If between 0 and 1, it is interpreted as a percentage of samples that can fail. If greater than 1, it is interpreted as a fixed number of samples that can fail",
            envvar="BENCH_FAIL_ON_ERROR",
        ),
    ] = None,
    log_samples: Annotated[
        Optional[bool],
        typer.Option(
            help="Log detailed samples and scores",
            envvar="BENCH_LOG_SAMPLES",
        ),
    ] = None,
    log_images: Annotated[
        Optional[bool],
        typer.Option(
            help="Log base64 encoded images",
            envvar="BENCH_LOG_IMAGES",
        ),
    ] = None,
    log_buffer: Annotated[
        Optional[int],
        typer.Option(
            help="Number of samples to buffer before writing to log",
            envvar="BENCH_LOG_BUFFER",
        ),
    ] = 10,
    score: Annotated[
        bool,
        typer.Option(
            help="Grade the benchmark, or leave unscored",
            envvar="BENCH_SCORE",
        ),
    ] = True,
    timeout: Annotated[
        Optional[int],
        typer.Option(
            help="Timeout for each request to the model API in seconds",
            envvar="BENCH_TIMEOUT",
        ),
    ] = None,
    max_retries: Annotated[
        Optional[int],
        typer.Option(
            help="Maximum number of times to retry model API requests (defaults to unlimited)",
            envvar="BENCH_MAX_RETRIES",
        ),
    ] = None,
    retry_on_error: Annotated[
        Optional[int],
        typer.Option(
            help="Retry samples if they encounter errors (by default, no retries occur). Specify --retry-on-error to retry a single time, or specify e.g. --retry-on-error=3 to retry multiple times.",
            envvar="BENCH_RETRY_ON_ERROR",
        ),
    ] = None,
    no_fail_on_error: Annotated[
        bool,
        typer.Option(
            "--no-fail-on-error",
            help="Do not fail the eval if errors occur within samples (instead, continue running other samples)",
            envvar="BENCH_NO_FAIL_ON_ERROR",
        ),
    ] = False,
    no_log_samples: Annotated[
        bool,
        typer.Option(
            "--no-log-samples",
            help="Do not include samples in the log file",
            envvar="BENCH_NO_LOG_SAMPLES",
        ),
    ] = False,
    no_log_images: Annotated[
        bool,
        typer.Option(
            "--no-log-images",
            help="Do not include base64 encoded images in the log file",
            envvar="BENCH_NO_LOG_IMAGES",
        ),
    ] = False,
    no_score: Annotated[
        bool,
        typer.Option(
            "--no-score",
            help="Do not score model output (use the inspect score command to score output later)",
            envvar="BENCH_NO_SCORE",
        ),
    ] = False,
    sandbox_cleanup: Annotated[
        Optional[bool],
        typer.Option(
            help="Cleanup sandbox environments after task completes",
            envvar="BENCH_SANDBOX_CLEANUP",
        ),
    ] = None,
    no_sandbox_cleanup: Annotated[
        bool,
        typer.Option(
            "--no-sandbox-cleanup",
            help="Do not cleanup sandbox environments after task completes",
            envvar="BENCH_NO_SANDBOX_CLEANUP",
        ),
    ] = False,
    trace: Annotated[
        bool,
        typer.Option(
            "--trace",
            help="Trace message interactions with evaluated model to terminal",
            envvar="BENCH_TRACE",
        ),
    ] = False,
    log_dir: Annotated[
        str,
        typer.Option(
            help="Directory for log files",
            envvar="BENCH_LOG_DIR",
        ),
    ] = "./logs",
    debug_errors: Annotated[
        bool,
        typer.Option(
            "--debug-errors",
            help="Enable debug mode for errors",
            envvar="BENCH_DEBUG_ERRORS",
        ),
    ] = False,
    debug: Annotated[
        bool,
        typer.Option(
            "--debug",
            help="Enable debug mode with full stack traces",
            envvar="BENCH_DEBUG",
        ),
    ] = False,
) -> None:
    """Retry failed evaluation(s) from log files."""

    # Process negating options
    if no_log_samples:
        log_samples = False
    if no_log_images:
        log_images = False
    if no_score:
        score = False
    if no_sandbox_cleanup:
        sandbox_cleanup = False

    # Process fail_on_error
    if no_fail_on_error:
        fail_on_error = False
    elif fail_on_error == 0.0:
        fail_on_error = True

    # Process retry_on_error
    if retry_on_error == 0:
        retry_on_error = None

    # Resolve log files
    retry_log_files = [
        log_file_info(filesystem(log_file).info(log_file)) for log_file in log_files
    ]

    # Set defaults
    log_level = "info"
    log_level_transcript = "info"

    # Apply display patch
    patch_display_results()

    try:
        # Retry
        eval_retry(
            retry_log_files,
            log_level=log_level,
            log_level_transcript=log_level_transcript,
            log_dir=log_dir,
            max_connections=max_connections,
            max_subprocesses=max_subprocesses,
            fail_on_error=fail_on_error,
            retry_on_error=retry_on_error,
            debug_errors=debug_errors,
            log_samples=log_samples,
            log_images=log_images,
            log_buffer=log_buffer,
            score=score,
            timeout=timeout,
            max_retries=max_retries,
            sandbox_cleanup=sandbox_cleanup,
            trace=trace,
            # These are additional retry-specific parameters
            max_samples=None,
            max_tasks=None,
            max_sandboxes=None,
            log_shared=None,
            score_display=None,
        )

        typer.echo("Retry evaluation complete!")
    except Exception as e:
        if debug:
            # In debug mode, let the full stack trace show
            raise
        else:
            # In normal mode, show clean error message
            error_msg = str(e)
            typer.secho(f"\n❌ Error: {error_msg}", fg=typer.colors.RED, err=True)
            typer.secho(
                "\nFor full stack trace, run with --debug flag",
                fg=typer.colors.CYAN,
                err=True,
            )
            sys.exit(1)