wangwei
4173a40d93
Both scripts support: - Shortcut args: online (default), offline, or any custom .yaml path - Second arg: log level (DEBUG/INFO/WARNING/ERROR), default INFO - Auto-timestamped log file saved to logs\eval_<date>_<time>.log - Sets PYTHONIOENCODING=utf-8 and PYTHONPATH=. automatically - Friendly error/success banners with log file path Usage: run_eval.bat # online eval run_eval.bat offline DEBUG # offline eval with DEBUG logs .\run_eval.ps1 online DEBUG # PowerShell equivalent Co-Authored-By: Claude <noreply@anthropic.com>
108 lines
3.4 KiB
Batchfile
108 lines
3.4 KiB
Batchfile
@echo off
|
|
setlocal enabledelayedexpansion
|
|
|
|
:: ============================================================
|
|
:: run_eval.bat - Run a RAGAS evaluation scenario with logs
|
|
::
|
|
:: Usage:
|
|
:: run_eval.bat (uses default online scenario)
|
|
:: run_eval.bat offline (runs offline smoke scenario)
|
|
:: run_eval.bat path\to\scenario.yaml (any custom scenario)
|
|
:: run_eval.bat offline DEBUG (second arg = log level)
|
|
:: ============================================================
|
|
|
|
cd /d "%~dp0"
|
|
|
|
echo.
|
|
echo ============================================================
|
|
echo Siemens RAGAS - Evaluation Runner
|
|
echo ============================================================
|
|
echo.
|
|
|
|
:: ----------------------------------------------------------------
|
|
:: 1. Resolve scenario path (arg1)
|
|
:: ----------------------------------------------------------------
|
|
set "SCENARIO=%~1"
|
|
if "%SCENARIO%"=="" set "SCENARIO=online"
|
|
|
|
if /i "%SCENARIO%"=="online" (
|
|
set "SCENARIO=scenarios\online\siemens-pdf-question-bank-online.yaml"
|
|
)
|
|
if /i "%SCENARIO%"=="offline" (
|
|
set "SCENARIO=scenarios\offline\siemens-pdf-offline-smoke.yaml"
|
|
)
|
|
|
|
if not exist "%SCENARIO%" (
|
|
echo [ERROR] Scenario file not found: %SCENARIO%
|
|
echo.
|
|
echo Usage examples:
|
|
echo run_eval.bat - online eval (default)
|
|
echo run_eval.bat offline - offline smoke
|
|
echo run_eval.bat path\to\file.yaml - custom scenario
|
|
goto :error
|
|
)
|
|
echo [OK] Scenario : %SCENARIO%
|
|
|
|
:: ----------------------------------------------------------------
|
|
:: 2. Resolve log level (arg2, default INFO)
|
|
:: ----------------------------------------------------------------
|
|
set "LOG_LEVEL=%~2"
|
|
if "%LOG_LEVEL%"=="" set "LOG_LEVEL=INFO"
|
|
echo [OK] Log level: %LOG_LEVEL%
|
|
|
|
:: ----------------------------------------------------------------
|
|
:: 3. Create logs dir and build timestamped log filename
|
|
:: ----------------------------------------------------------------
|
|
if not exist "logs" mkdir logs
|
|
for /f "tokens=1-3 delims=/-" %%a in ("%DATE%") do (
|
|
set "YMD=%%c-%%a-%%b"
|
|
)
|
|
for /f "tokens=1-3 delims=:." %%a in ("%TIME: =0%") do (
|
|
set "HMS=%%a%%b%%c"
|
|
)
|
|
set "LOG_FILE=logs\eval_%YMD%_%HMS%.log"
|
|
echo [OK] Log file : %LOG_FILE%
|
|
|
|
echo.
|
|
echo ============================================================
|
|
echo Starting evaluation...
|
|
echo (Logs also written to %LOG_FILE%)
|
|
echo Press Ctrl+C to abort
|
|
echo ============================================================
|
|
echo.
|
|
|
|
:: ----------------------------------------------------------------
|
|
:: 4. Run evaluation with UTF-8 and logging
|
|
:: ----------------------------------------------------------------
|
|
set PYTHONIOENCODING=utf-8
|
|
set PYTHONPATH=.
|
|
|
|
python main.py ^
|
|
--scenario "%SCENARIO%" ^
|
|
--log-file "%LOG_FILE%" ^
|
|
--log-level %LOG_LEVEL%
|
|
|
|
if errorlevel 1 (
|
|
echo.
|
|
echo [ERROR] Evaluation failed. Check log: %LOG_FILE%
|
|
goto :error
|
|
)
|
|
|
|
echo.
|
|
echo ============================================================
|
|
echo Evaluation complete!
|
|
echo Log saved to: %LOG_FILE%
|
|
echo Open the web console to view results: start.bat
|
|
echo ============================================================
|
|
echo.
|
|
pause
|
|
exit /b 0
|
|
|
|
:error
|
|
echo.
|
|
echo ============================================================
|
|
echo Evaluation failed. See error above or check log file.
|
|
echo ============================================================
|
|
pause
|
|
exit /b 1
|