siemens_ragas/run_eval.bat

@echo off
setlocal enabledelayedexpansion

:: ============================================================
::  run_eval.bat  -  Run a RAGAS evaluation scenario with logs
::
::  Usage:
::    run_eval.bat                          (uses default online scenario)
::    run_eval.bat offline                  (runs offline smoke scenario)
::    run_eval.bat path\to\scenario.yaml    (any custom scenario)
::    run_eval.bat offline DEBUG            (second arg = log level)
:: ============================================================

cd /d "%~dp0"

echo.
echo ============================================================
echo   Siemens RAGAS  -  Evaluation Runner
echo ============================================================
echo.

:: ----------------------------------------------------------------
:: 1. Resolve scenario path  (arg1)
:: ----------------------------------------------------------------
set "SCENARIO=%~1"
if "%SCENARIO%"=="" set "SCENARIO=online"

if /i "%SCENARIO%"=="online" (
    set "SCENARIO=scenarios\online\siemens-pdf-question-bank-online.yaml"
)
if /i "%SCENARIO%"=="offline" (
    set "SCENARIO=scenarios\offline\siemens-pdf-offline-smoke.yaml"
)

if not exist "%SCENARIO%" (
    echo [ERROR] Scenario file not found: %SCENARIO%
    echo.
    echo Usage examples:
    echo   run_eval.bat                    - online eval (default)
    echo   run_eval.bat offline            - offline smoke
    echo   run_eval.bat path\to\file.yaml  - custom scenario
    goto :error
)
echo [OK] Scenario : %SCENARIO%

:: ----------------------------------------------------------------
:: 2. Resolve log level  (arg2, default INFO)
:: ----------------------------------------------------------------
set "LOG_LEVEL=%~2"
if "%LOG_LEVEL%"=="" set "LOG_LEVEL=INFO"
echo [OK] Log level: %LOG_LEVEL%

:: ----------------------------------------------------------------
:: 3. Create logs dir and build timestamped log filename
:: ----------------------------------------------------------------
if not exist "logs" mkdir logs
for /f "tokens=1-3 delims=/-" %%a in ("%DATE%") do (
    set "YMD=%%c-%%a-%%b"
)
for /f "tokens=1-3 delims=:." %%a in ("%TIME: =0%") do (
    set "HMS=%%a%%b%%c"
)
set "LOG_FILE=logs\eval_%YMD%_%HMS%.log"
echo [OK] Log file : %LOG_FILE%

echo.
echo ============================================================
echo   Starting evaluation...
echo   (Logs also written to %LOG_FILE%)
echo   Press Ctrl+C to abort
echo ============================================================
echo.

:: ----------------------------------------------------------------
:: 4. Run evaluation with UTF-8 and logging
:: ----------------------------------------------------------------
set PYTHONIOENCODING=utf-8
set PYTHONPATH=.

python main.py ^
    --scenario "%SCENARIO%" ^
    --log-file "%LOG_FILE%" ^
    --log-level %LOG_LEVEL%

if errorlevel 1 (
    echo.
    echo [ERROR] Evaluation failed. Check log: %LOG_FILE%
    goto :error
)

echo.
echo ============================================================
echo   Evaluation complete!
echo   Log saved to: %LOG_FILE%
echo   Open the web console to view results: start.bat
echo ============================================================
echo.
pause
exit /b 0

:error
echo.
echo ============================================================
echo   Evaluation failed. See error above or check log file.
echo ============================================================
pause
exit /b 1