siemens_ragas/run_eval.ps1

# run_eval.ps1 - Siemens RAGAS Evaluation Runner
# Usage:
#   .\run_eval.ps1                         # online eval (default)
#   .\run_eval.ps1 offline                 # offline smoke
#   .\run_eval.ps1 path\to\scenario.yaml   # custom scenario
#   .\run_eval.ps1 online DEBUG            # second arg = log level (DEBUG/INFO/WARNING)
# Or: powershell -ExecutionPolicy Bypass -File run_eval.ps1 [scenario] [log-level]

param(
    [string]$Scenario = "online",
    [string]$LogLevel = "INFO"
)

$ErrorActionPreference = "Stop"
Set-Location $PSScriptRoot

Write-Host ""
Write-Host "============================================================" -ForegroundColor Cyan
Write-Host "  Siemens RAGAS  -  Evaluation Runner" -ForegroundColor Cyan
Write-Host "============================================================" -ForegroundColor Cyan
Write-Host ""

# ----------------------------------------------------------------
# 1. Resolve scenario path
# ----------------------------------------------------------------
$scenarioMap = @{
    "online"  = "scenarios\online\siemens-pdf-question-bank-online.yaml"
    "offline" = "scenarios\offline\siemens-pdf-offline-smoke.yaml"
}
if ($scenarioMap.ContainsKey($Scenario.ToLower())) {
    $Scenario = $scenarioMap[$Scenario.ToLower()]
}
if (-not (Test-Path $Scenario)) {
    Write-Host "[ERROR] Scenario file not found: $Scenario" -ForegroundColor Red
    Write-Host ""
    Write-Host "Usage examples:"
    Write-Host "  .\run_eval.ps1                    - online eval (default)"
    Write-Host "  .\run_eval.ps1 offline            - offline smoke"
    Write-Host "  .\run_eval.ps1 path\to\file.yaml  - custom scenario"
    Read-Host "Press Enter to exit"
    exit 1
}
Write-Host "[OK] Scenario : $Scenario" -ForegroundColor Green

# ----------------------------------------------------------------
# 2. Validate log level
# ----------------------------------------------------------------
$validLevels = @("DEBUG", "INFO", "WARNING", "ERROR")
if ($validLevels -notcontains $LogLevel.ToUpper()) {
    Write-Host "[WARN] Unknown log level '$LogLevel', defaulting to INFO" -ForegroundColor Yellow
    $LogLevel = "INFO"
}
Write-Host "[OK] Log level: $LogLevel" -ForegroundColor Green

# ----------------------------------------------------------------
# 3. Create logs dir with timestamped filename
# ----------------------------------------------------------------
if (-not (Test-Path "logs")) { New-Item -ItemType Directory "logs" | Out-Null }
$timestamp = Get-Date -Format "yyyy-MM-dd_HHmmss"
$logFile = "logs\eval_$timestamp.log"
Write-Host "[OK] Log file : $logFile" -ForegroundColor Green

Write-Host ""
Write-Host "============================================================" -ForegroundColor Cyan
Write-Host "  Starting evaluation..." -ForegroundColor Cyan
Write-Host "  Logs also written to: $logFile" -ForegroundColor Cyan
Write-Host "  Press Ctrl+C to abort" -ForegroundColor Yellow
Write-Host "============================================================" -ForegroundColor Cyan
Write-Host ""

# ----------------------------------------------------------------
# 4. Run evaluation
# ----------------------------------------------------------------
$env:PYTHONIOENCODING = "utf-8"
$env:PYTHONPATH = "."

& python main.py `
    --scenario $Scenario `
    --log-file $logFile `
    --log-level $LogLevel.ToUpper()

if ($LASTEXITCODE -ne 0) {
    Write-Host ""
    Write-Host "[ERROR] Evaluation failed. Check log: $logFile" -ForegroundColor Red
    Read-Host "Press Enter to exit"
    exit 1
}

Write-Host ""
Write-Host "============================================================" -ForegroundColor Green
Write-Host "  Evaluation complete!" -ForegroundColor Green
Write-Host "  Log saved to: $logFile" -ForegroundColor Green
Write-Host "  Open the web console to view results: start.bat" -ForegroundColor Cyan
Write-Host "============================================================" -ForegroundColor Green
Write-Host ""
Read-Host "Press Enter to exit"