From d0a83f10323fa381c30304b0f3cfb9d8d14b326b Mon Sep 17 00:00:00 2001 From: Geoff McLane Date: Wed, 13 May 2015 14:12:03 +0200 Subject: [PATCH] add another test set that includes a compare to testbase --- test/alltest2.bat | 89 +++++++++++++++++++++++++++++ test/alltestc.bat | 27 +++++++++ test/onetest2.bat | 140 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 256 insertions(+) create mode 100644 test/alltest2.bat create mode 100644 test/alltestc.bat create mode 100644 test/onetest2.bat diff --git a/test/alltest2.bat b/test/alltest2.bat new file mode 100644 index 0000000..e265e17 --- /dev/null +++ b/test/alltest2.bat @@ -0,0 +1,89 @@ +@setlocal +@echo off + +REM alltest2.bat - execute all test cases +REM + +@if "%1." == "." goto USE +@if "%2." == "." goto USE + +REM check for input file +@if NOT EXIST testcases.txt goto Err0 +@if NOT EXIST onetest2.bat goto Err3 +@if NOT EXIST input\nul goto Err4 +@if NOT EXIST testbase\nul goto Err5 +@diff --version >nul +@if ERRORLEVEL 1 goto Err6 + +REM set the runtime exe file +set TIDY=%1 +@if NOT EXIST %TIDY% goto ERR1 + +REM set the OUTPUT folder +set TIDYOUT=%2 +@if EXIST %TIDYOUT%\nul goto GOTDIR +@echo Folder '%TIDYOUT%' does not exist ... it will be created? ... Ctrl+C to EXIT! +@pause +@md %TIDYOUT% +@if NOT EXIST %TIDYOUT%\nul goto Err2 +:GOTDIR +@set TMPTEST=temptest.txt +@if EXIST %TMPTEST% @del %TMPTEST% + +@echo Processing input test case list from testcases.txt +@echo Each test will be passed to onetest1.bat for processing... + +@set ERRTESTS= +for /F "tokens=1*" %%i in (testcases.txt) do call onetest2.bat %%i %%j +@if "%ERRTESTS%." == "." goto DONE +@echo ERROR TESTS [%ERRTESTS%] ... +:DONE +@echo Completed all test... full output written to %TMPTEST% +goto END + +:ERR0 +echo ERROR: Can not locate 'testcases.txt' ... check name, and location ... +goto END +:ERR1 +echo ERROR: Can not locate %TIDY% ... check name, and location ... +goto END +:ERR2 +echo ERROR: Can not create %TIDYOUT% folder ... check name, and location ... +goto END +:ERR3 +echo ERROR: Can not locate 'onetest2.bat' ... check name, and location ... +goto END +:ERR4 +echo ERROR: Can not locate 'input' folder ... check name, and location ... +goto END +:ERR5 +echo ERROR: Can not locate 'testbase' folder ... check name, and location ... +goto END +:ERR5 +echo ERROR: Seem unable to run 'diff'! Either intall 'diff' in your path, +@echo or *** FIX ME *** to use a some other file compare untility. +goto END + +:USE +@echo Usage of ALLTEST2.BAT ......................................... +@echo AllTest2 tidy.exe Out_Folder +@echo tidy.exe - This is the Tidy.exe you want to use for the test. +@echo Out_Folder - This is the FOLDER where you want the results put. +@echo This folder will be created if it does not already exist. +@echo ================================== +@echo ALLTEST2.BAT will run a battery of test files in the input folder +@echo Each test name, has an expected result, given in testcases.txt +@echo There will be a warning if any test file fails to give this result. +@echo ================================== +@echo But the main purpose is to compare the 'results' of two version of +@echo any two Tidy runtime exe's. Thus the NEW sets of results, +@echo will be compareed these two folders. +@echo Probably any file compare utility can be used, but here 'diff' +@echo is used. +@echo ================================== +@echo Essentially this batch file should be run using alltestc.bat, +@echo which establishes some of the variable needed. +@echo ................................................................ +@goto END + +:END diff --git a/test/alltestc.bat b/test/alltestc.bat new file mode 100644 index 0000000..da30ed1 --- /dev/null +++ b/test/alltestc.bat @@ -0,0 +1,27 @@ +@setlocal + +@REM alltestc.bat - execute all test cases +@REM + +@REM ######################################################## +@REM ### *** SET LOCATION OF TIDY EXE TO USE FOR TEST *** ### +@REM ### (for MS compiler users) ### +@REM ######################################################## +@REM This is the location of the cmake build output using the MSVC Generator +@set TMPEXE=..\build\cmake\Release\tidy5.exe +@if NOT EXIST %TMPEXE% goto NOEXE + +@set TMPTEST=temptests.txt +@if EXIST %TMPTEST% @del %TMPTEST% + +@echo Doing: 'alltest2 %TMPEXE% temp-5' + +@call alltest2 %TMPEXE% temp-5 + +@goto END + +:NOEXE +@echo. +@echo Error: Can NOT locate %TMPEXE%! Has it been built? +@echo *** FIX ME *** setting the location of the EXE to use for the test +:END diff --git a/test/onetest2.bat b/test/onetest2.bat new file mode 100644 index 0000000..401477e --- /dev/null +++ b/test/onetest2.bat @@ -0,0 +1,140 @@ +@setlocal +@echo off +@set TMPTEST=temptest.txt +REM onetest2.bat - execute a single test case +REM + +@if "%TIDY%." == "." goto Err1 +@if NOT EXIST %TIDY% goto Err2 +@if "%TIDYOUT%." == "." goto Err3 +@if NOT EXIST %TIDYOUT%\nul goto Err4 +@if NOT EXIST input\nul goto Err5 +@if NOT EXIST testbase\nul goto Err8 + +set TESTNO=%1 +set EXPECTED=%2 + +set INFILES=input\in_%1.*ml +set CFGFILE=input\cfg_%1.txt + +set TIDYFILE=%TIDYOUT%\out_%1.html +set MSGFILE=%TIDYOUT%\msg_%1.txt + +@rem Setup the BASE file +@set TIDYBASE=testbase\out_%1.html + +set HTML_TIDY= + +REM If no test specific config file, use default. +if NOT exist %CFGFILE% set CFGFILE=input\cfg_default.txt + +REM Get specific input file name +@set INFILE= +for %%F in ( %INFILES% ) do set INFILE=%%F +@if "%INFILE%." == "." goto Err6 +@if NOT EXIST %INFILE% goto Err7 + +REM Remove any pre-exising test outputs +if exist %MSGFILE% del %MSGFILE% +if exist %TIDYFILE% del %TIDYFILE% + +@REM Noisy output, or quiet +@REM echo Testing %1 input %INFILE% config %CFGFILE% ... +@echo Doing: '%TIDY% -f %MSGFILE% -config %CFGFILE% %3 %4 %5 %6 %7 %8 %9 --tidy-mark no -o %TIDYFILE% %INFILE% >> %TMPTEST% + +@%TIDY% -f %MSGFILE% -config %CFGFILE% %3 %4 %5 %6 %7 %8 %9 --tidy-mark no -o %TIDYFILE% %INFILE% +@set STATUS=%ERRORLEVEL% +@echo Testing %1, expect %EXPECTED%, got %STATUS% +@echo Testing %1, expect %EXPECTED%, got %STATUS% >> %TMPTEST% + +@if %STATUS% EQU %EXPECTED% goto EXITOK +@set ERRTESTS=%ERRTESTS% %TESTNO% +@echo *** Failed - got %STATUS%, expected %EXPECTED% *** +@type %MSGFILE% +@echo *** Failed - got %STATUS%, expected %EXPECTED% *** >> %TMPTEST% +@type %MSGFILE% >> %TMPTEST% + +:EXITOK +@if NOT EXIST %TIDYBASE% goto NOBASE +@if NOT EXIST %TIDYFILE% goto NOOUT +@echo Doing: 'diff -u %TIDYBASE% %TIDYFILE%' >> %TMPTEST% +@diff -u %TIDYBASE% %TIDYFILE% >> %TMPTEST% +@if ERRORLEVEL 1 goto GOTDIFF +@goto done + +:GOTDIFF +@echo Got a DIFFERENCE between %TIDYBASE% and %TIDYFILE% +@echo Got a DIFFERENCE between %TIDYBASE% and %TIDYFILE% >> %TMPTEST% +@goto done + +:NOBASE +@REM If errorlevel 2 then normally no output generated, to this is OK +@if "%STATUS%" == "2" goto done +@REM If no output generated this time, then this is probably OK +@if NOT EXIST %TIDYFILE% goto done +@echo Can NOT locate %TIDYBASE%. This may be ok if not file generated +@goto done +:NOOUT +@echo *** FAILED: A base file exists, but none geneated this time! +@echo *** FAILED: A base file exists, but none geneated this time! >> %TMPTEST% +@goto done + +:Err1 +@echo. +@echo ERROR: runtime exe not set in TIDY environment variable ... +@echo. +@goto TRYAT + +:Err2 +@echo. +@echo ERROR: runtime exe %TIDY% not found ... check name, location ... +@echo. +@goto TRYAT + +:Err3 +@echo. +@echo ERROR: output folder TIDYOUT not set in environment ... +@echo. +@goto TRYAT + +:Err4 +@echo. +@echo ERROR: output folder %TIDYOUT% does not exist ... +@echo. +@goto TRYAT + +:Err5 +@echo. +@echo ERROR: input folder 'input' does not exist ... check name, location .. +@echo. +@goto TRYAT + +:TRYAT +@echo You could try running alltest1.cmd ..\build\cmake\Release\Tidy5.exe tmp +@echo but essentially this file should be run using the alltestc.bat batch file. +@echo. +@pause +@goto done + +:Err6 +@echo. +@echo ERROR: Failed to find input matching '%INFILES%'!!! +@echo. +@pause +@goto done + +:Err7 +@echo. +@echo ERROR: Failed to find input file '%INFILE%'!!! +@echo. +@pause +@goto done + +:Err8 +@echo. +@echo ERROR: Failed to find 'testbase' folder!!! +@echo. +@pause +@goto done + +:done