Codechange: move regression outside of bin and make it work via CMake script
The tst_stationlist savegame had to be changed to start the correct AI. In the old setup, all regression AIs had the same name, which made it impossible to run both regressions in parallel. With the new setup this is possible. Although all files are available to run the regression, it won't really work till CMake is introduced (which will happen in a few commits from here)desync-debugging
Patric Stout
5 years ago
committed by
glx22
parent
84b71f7962
commit
203a77c1dc
@ -1,67 +0,0 @@
|
||||
#!/bin/sh
|
||||
|
||||
if ! [ -f ai/regression/completeness.sh ]; then
|
||||
echo "Make sure you are in the root of OpenTTD before starting this script."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cat ai/regression/tst_*/main.nut | tr ';' '\n' | awk '
|
||||
/^function/ {
|
||||
for (local in locals) {
|
||||
delete locals[local]
|
||||
}
|
||||
if (match($0, "function Regression::Start") || match($0, "function Regression::Stop")) next
|
||||
locals["this"] = "AIControllerSquirrel"
|
||||
}
|
||||
|
||||
/local/ {
|
||||
gsub(".*local", "local")
|
||||
if (match($4, "^AI")) {
|
||||
sub("\\(.*", "", $4)
|
||||
locals[$2] = $4
|
||||
}
|
||||
}
|
||||
|
||||
/Valuate/ {
|
||||
gsub(".*Valuate\\(", "")
|
||||
gsub("\\).*", "")
|
||||
gsub(",.*", "")
|
||||
gsub("\\.", "::")
|
||||
print $0
|
||||
}
|
||||
|
||||
/\./ {
|
||||
for (local in locals) {
|
||||
if (match($0, local ".")) {
|
||||
fname = substr($0, index($0, local "."))
|
||||
sub("\\(.*", "", fname)
|
||||
sub("\\.", "::", fname)
|
||||
sub(local, locals[local], fname)
|
||||
print fname
|
||||
if (match(locals[local], "List")) {
|
||||
sub(locals[local], "AIAbstractList", fname)
|
||||
print fname
|
||||
}
|
||||
}
|
||||
}
|
||||
# We want to remove everything before the FIRST occurrence of AI.
|
||||
# If we do not remove any other occurrences of AI from the string
|
||||
# we will remove everything before the LAST occurrence of AI, so
|
||||
# do some little magic to make it work the way we want.
|
||||
sub("AI", "AXXXXY")
|
||||
gsub("AI", "AXXXXX")
|
||||
sub(".*AXXXXY", "AI")
|
||||
if (match($0, "^AI") && match($0, ".")) {
|
||||
sub("\\(.*", "", $0)
|
||||
sub("\\.", "::", $0)
|
||||
print $0
|
||||
}
|
||||
}
|
||||
' | sed 's/ //g' | sort | uniq > tmp.in_regression
|
||||
|
||||
grep 'DefSQ.*Method' ../src/script/api/ai/*.hpp.sq | grep -v 'AIError::' | grep -v 'AIAbstractList::Valuate' | grep -v '::GetClassName' | sed 's/^[^,]*, &//g;s/,[^,]*//g' | sort > tmp.in_api
|
||||
|
||||
diff -u tmp.in_regression tmp.in_api | grep -v '^+++' | grep '^+' | sed 's/^+//'
|
||||
|
||||
rm -f tmp.in_regression tmp.in_api
|
||||
|
||||
@ -1,69 +0,0 @@
|
||||
#!/bin/sh
|
||||
|
||||
if ! [ -f ai/regression/run.sh ]; then
|
||||
echo "Make sure you are in the root of OpenTTD before starting this script."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -f scripts/game_start.scr ]; then
|
||||
mv scripts/game_start.scr scripts/game_start.scr.regression
|
||||
fi
|
||||
|
||||
params=""
|
||||
gdb=""
|
||||
if [ "$1" != "-r" ]; then
|
||||
params="-snull -mnull -vnull:ticks=30000"
|
||||
fi
|
||||
if [ "$1" = "-g" ]; then
|
||||
gdb="gdb --ex run --args "
|
||||
fi
|
||||
|
||||
if [ -d "ai/regression/tst_$1" ]; then
|
||||
tests="ai/regression/tst_$1"
|
||||
elif [ -d "ai/regression/tst_$2" ]; then
|
||||
tests="ai/regression/tst_$2"
|
||||
else
|
||||
tests=ai/regression/tst_*
|
||||
fi
|
||||
|
||||
ret=0
|
||||
for tst in $tests; do
|
||||
echo -n "Running $tst... "
|
||||
|
||||
# Make sure that only one info.nut is present for each test run. Otherwise openttd gets confused.
|
||||
cp ai/regression/regression_info.nut $tst/info.nut
|
||||
|
||||
sav=$tst/test.sav
|
||||
if ! [ -f $sav ]; then
|
||||
sav=ai/regression/empty.sav
|
||||
fi
|
||||
|
||||
if [ -n "$gdb" ]; then
|
||||
$gdb ./openttd -x -c ai/regression/regression.cfg $params -g $sav
|
||||
else
|
||||
./openttd -x -c ai/regression/regression.cfg $params -g $sav -d script=2 -d misc=9 2>&1 | awk '{ gsub("0x(\\(nil\\)|0+)(x0)?", "0x00000000", $0); gsub("^dbg: \\[script\\]", "", $0); gsub("^ ", "ERROR: ", $0); gsub("ERROR: \\[1\\] ", "", $0); gsub("\\[P\\] ", "", $0); print $0; }' | grep -v '^dbg: \[.*\]' > $tst/tmp.regression
|
||||
fi
|
||||
|
||||
if [ -z "$gdb" ]; then
|
||||
res="`diff -ub $tst/result.txt $tst/tmp.regression`"
|
||||
if [ -z "$res" ]; then
|
||||
echo "passed!"
|
||||
else
|
||||
echo "failed! Difference:"
|
||||
echo "$res"
|
||||
ret=1
|
||||
fi
|
||||
fi
|
||||
|
||||
rm $tst/info.nut
|
||||
|
||||
if [ "$1" != "-k" ]; then
|
||||
rm -f $tst/tmp.regression
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -f scripts/game_start.scr.regression ]; then
|
||||
mv scripts/game_start.scr.regression scripts/game_start.scr
|
||||
fi
|
||||
|
||||
exit $ret
|
||||
@ -1,152 +0,0 @@
|
||||
Option Explicit
|
||||
|
||||
' This file is part of OpenTTD.
|
||||
' OpenTTD is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2.
|
||||
' OpenTTD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
|
||||
' See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with OpenTTD. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Dim FSO
|
||||
Set FSO = CreateObject("Scripting.FileSystemObject")
|
||||
|
||||
Function GetTestList()
|
||||
Dim retests, i, tests, dir
|
||||
Set retests = New RegExp
|
||||
Set GetTestList = CreateObject("Scripting.Dictionary")
|
||||
|
||||
retests.Pattern = "ai/regression/tst_*"
|
||||
retests.Global = True
|
||||
For i = 0 To WScript.Arguments.Count - 1
|
||||
Dim test
|
||||
test = "ai/regression/tst_" & WScript.Arguments.Item(i)
|
||||
If FSO.FolderExists(test) Then
|
||||
retests.Pattern = test
|
||||
Exit For
|
||||
End If
|
||||
Next
|
||||
|
||||
For Each dir In FSO.GetFolder("ai/regression/").SubFolders
|
||||
Dim name
|
||||
name = "ai/regression/" & dir.Name
|
||||
If retests.Test(name) Then
|
||||
GetTestList.Add name, name
|
||||
End If
|
||||
Next
|
||||
End Function
|
||||
|
||||
Function GetParams()
|
||||
GetParams = "-snull -mnull -vnull:ticks=30000"
|
||||
If WScript.Arguments.Count = 0 Then Exit Function
|
||||
If WScript.Arguments.Item(0) <> "-r" Then Exit Function
|
||||
GetParams = ""
|
||||
End Function
|
||||
|
||||
Sub FilterFile(filename)
|
||||
Dim lines, filter, file
|
||||
|
||||
Set file = FSO.OpenTextFile(filename, 1)
|
||||
If Not file.AtEndOfStream Then
|
||||
lines = file.ReadAll
|
||||
End If
|
||||
file.Close
|
||||
|
||||
Set filter = New RegExp
|
||||
filter.Global = True
|
||||
filter.Multiline = True
|
||||
filter.Pattern = "0x(\(nil\)|0+)(x0)?"
|
||||
lines = filter.Replace(lines, "0x00000000")
|
||||
filter.Pattern = "^dbg: \[script\]"
|
||||
lines = filter.Replace(lines, "")
|
||||
filter.Pattern = "^ "
|
||||
lines = filter.Replace(lines, "ERROR: ")
|
||||
filter.Pattern = "ERROR: \[1\] \[P\] "
|
||||
lines = filter.Replace(lines, "")
|
||||
filter.Pattern = "^dbg: .*\r\n"
|
||||
lines = filter.Replace(lines, "")
|
||||
|
||||
Set file = FSO.OpenTextFile(filename, 2)
|
||||
file.Write lines
|
||||
file.Close
|
||||
End Sub
|
||||
|
||||
Function CompareFiles(filename1, filename2)
|
||||
Dim file, lines1, lines2
|
||||
Set file = FSO.OpenTextFile(filename1, 1)
|
||||
If Not file.AtEndOfStream Then
|
||||
lines1 = file.ReadAll
|
||||
End IF
|
||||
file.Close
|
||||
Set file = FSO.OpenTextFile(filename2, 1)
|
||||
If Not file.AtEndOfStream Then
|
||||
lines2 = file.ReadAll
|
||||
End IF
|
||||
file.Close
|
||||
CompareFiles = (lines1 = lines2)
|
||||
End Function
|
||||
|
||||
Function RunTest(test, params, ret)
|
||||
Dim WshShell, oExec, sav, command
|
||||
Set WshShell = CreateObject("WScript.Shell")
|
||||
|
||||
' Make sure that only one info.nut is present for each test run. Otherwise openttd gets confused.
|
||||
FSO.CopyFile "ai/regression/regression_info.nut", test & "/info.nut"
|
||||
|
||||
sav = test & "/test.sav"
|
||||
If Not FSO.FileExists(sav) Then
|
||||
sav = "ai/regression/empty.sav"
|
||||
End If
|
||||
|
||||
command = ".\openttd -x -c ai/regression/regression.cfg " & params & " -g " & sav & " -d script=2 -d misc=9"
|
||||
' 2>&1 must be after >tmp.regression, else stderr is not redirected to the file
|
||||
WshShell.Run "cmd /c " & command & " >"& test & "/tmp.regression 2>&1", 0, True
|
||||
|
||||
FilterFile test & "/tmp.regression"
|
||||
|
||||
If CompareFiles(test & "/result.txt", test & "/tmp.regression") Then
|
||||
RunTest = "passed!"
|
||||
Else
|
||||
RunTest = "failed!"
|
||||
ret = 1
|
||||
End If
|
||||
|
||||
FSO.DeleteFile test & "/info.nut"
|
||||
|
||||
If WScript.Arguments.Count > 0 Then
|
||||
If WScript.Arguments.Item(0) = "-k" Then
|
||||
Exit Function
|
||||
End If
|
||||
End If
|
||||
|
||||
FSO.DeleteFile test & "/tmp.regression"
|
||||
End Function
|
||||
|
||||
On Error Resume Next
|
||||
WScript.StdOut.WriteLine ""
|
||||
If Err.Number <> 0 Then
|
||||
WScript.Echo "This script must be started with cscript."
|
||||
WScript.Quit 1
|
||||
End If
|
||||
On Error Goto 0
|
||||
|
||||
If Not FSO.FileExists("ai/regression/run.vbs") Then
|
||||
WScript.Echo "Make sure you are in the root of OpenTTD before starting this script."
|
||||
WScript.Quit 1
|
||||
End If
|
||||
|
||||
If FSO.FileExists("scripts/game_start.scr") Then
|
||||
FSO.MoveFile "scripts/game_start.scr", "scripts/game_start.scr.regression"
|
||||
End If
|
||||
|
||||
Dim params, test, ret
|
||||
params = GetParams()
|
||||
ret = 0
|
||||
|
||||
For Each test in GetTestList()
|
||||
WScript.StdOut.Write "Running " & test & "... "
|
||||
WScript.StdOut.WriteLine RunTest(test, params, ret)
|
||||
Next
|
||||
|
||||
If FSO.FileExists("scripts/game_start.scr.regression") Then
|
||||
FSO.MoveFile "scripts/game_start.scr.regression", "scripts/game_start.scr"
|
||||
End If
|
||||
|
||||
WScript.Quit ret
|
||||
Binary file not shown.
@ -0,0 +1,97 @@
|
||||
cmake_minimum_required(VERSION 3.5)
|
||||
|
||||
#
|
||||
# Runs a single regressoion test
|
||||
#
|
||||
|
||||
if (NOT REGRESSION_TEST)
|
||||
message(FATAL_ERROR "Script needs REGRESSION_TEST defined (tip: use -DREGRESSION_TEST=..)")
|
||||
endif (NOT REGRESSION_TEST)
|
||||
if (NOT OPENTTD_EXECUTABLE)
|
||||
message(FATAL_ERROR "Script needs OPENTTD_EXECUTABLE defined (tip: use -DOPENTTD_EXECUTABLE=..)")
|
||||
endif (NOT OPENTTD_EXECUTABLE)
|
||||
|
||||
if (NOT EXISTS ai/${REGRESSION_TEST}/test.sav)
|
||||
message(FATAL_ERROR "Regression test ${REGRESSION_TEST} does not exist (tip: check regression folder for the correct spelling)")
|
||||
endif ()
|
||||
|
||||
# If editbin is given, copy the executable to a new folder, and change the
|
||||
# subsystem to console. The copy is needed as multiple regressions can run
|
||||
# at the same time.
|
||||
if (EDITBIN_EXECUTABLE)
|
||||
execute_process(COMMAND ${CMAKE_COMMAND} -E copy ${OPENTTD_EXECUTABLE} regression/${REGRESSION_TEST}.exe)
|
||||
set(OPENTTD_EXECUTABLE "regression/${REGRESSION_TEST}.exe")
|
||||
|
||||
execute_process(COMMAND ${EDITBIN_EXECUTABLE} /nologo /subsystem:console ${OPENTTD_EXECUTABLE})
|
||||
endif (EDITBIN_EXECUTABLE)
|
||||
|
||||
# Run the regression test
|
||||
execute_process(COMMAND ${OPENTTD_EXECUTABLE}
|
||||
-x
|
||||
-c regression/regression.cfg
|
||||
-g ai/${REGRESSION_TEST}/test.sav
|
||||
-snull
|
||||
-mnull
|
||||
-vnull:ticks=30000
|
||||
-d script=2
|
||||
-d misc=9
|
||||
OUTPUT_VARIABLE REGRESSION_OUTPUT
|
||||
ERROR_VARIABLE REGRESSION_RESULT
|
||||
OUTPUT_STRIP_TRAILING_WHITESPACE
|
||||
ERROR_STRIP_TRAILING_WHITESPACE
|
||||
)
|
||||
|
||||
if (REGRESSION_OUTPUT)
|
||||
message(FATAL_ERROR "Unexpected output: ${REGRESSION_OUTPUT}")
|
||||
endif (REGRESSION_OUTPUT)
|
||||
|
||||
if (NOT REGRESSION_RESULT)
|
||||
message(FATAL_ERROR "Regression did not output anything; did the compilation fail?")
|
||||
endif (NOT REGRESSION_RESULT)
|
||||
|
||||
# For some reason pointer can be printed as '0x(nil)', '0x0000000000000000', or '0x0x0'
|
||||
string(REPLACE "0x(nil)" "0x00000000" REGRESSION_RESULT "${REGRESSION_RESULT}")
|
||||
string(REPLACE "0x0000000000000000" "0x00000000" REGRESSION_RESULT "${REGRESSION_RESULT}")
|
||||
string(REPLACE "0x0x0" "0x00000000" REGRESSION_RESULT "${REGRESSION_RESULT}")
|
||||
|
||||
# Convert the output to a format that is expected (and more readable) by result.txt
|
||||
string(REPLACE "\ndbg: [script]" "\n" REGRESSION_RESULT "${REGRESSION_RESULT}")
|
||||
string(REPLACE "\n " "\nERROR: " REGRESSION_RESULT "${REGRESSION_RESULT}")
|
||||
string(REPLACE "\nERROR: [1] " "\n" REGRESSION_RESULT "${REGRESSION_RESULT}")
|
||||
string(REPLACE "\n[P] " "\n" REGRESSION_RESULT "${REGRESSION_RESULT}")
|
||||
string(REGEX REPLACE "dbg: ([^\n]*)\n?" "" REGRESSION_RESULT "${REGRESSION_RESULT}")
|
||||
|
||||
# Read the expected result
|
||||
file(READ ai/${REGRESSION_TEST}/result.txt REGRESSION_EXPECTED)
|
||||
|
||||
# Convert the string to a list
|
||||
string(REPLACE "\n" ";" REGRESSION_RESULT "${REGRESSION_RESULT}")
|
||||
string(REPLACE "\n" ";" REGRESSION_EXPECTED "${REGRESSION_EXPECTED}")
|
||||
|
||||
set(ARGC 0)
|
||||
set(ERROR NO)
|
||||
|
||||
list(LENGTH REGRESSION_EXPECTED REGRESSION_EXPECTED_LENGTH)
|
||||
|
||||
# Compare the output
|
||||
foreach(RESULT IN LISTS REGRESSION_RESULT)
|
||||
list(GET REGRESSION_EXPECTED ${ARGC} EXPECTED)
|
||||
|
||||
if (NOT RESULT STREQUAL EXPECTED)
|
||||
message("${ARGC}: - ${EXPECTED}")
|
||||
message("${ARGC}: + ${RESULT}'")
|
||||
set(ERROR YES)
|
||||
endif (NOT RESULT STREQUAL EXPECTED)
|
||||
|
||||
math(EXPR ARGC "${ARGC} + 1")
|
||||
endforeach(RESULT)
|
||||
|
||||
if (NOT REGRESSION_EXPECTED_LENGTH EQUAL ARGC)
|
||||
math(EXPR MISSING "${REGRESSION_EXPECTED_LENGTH} - ${ARGC}")
|
||||
message("(${MISSING} more lines were expected than found)")
|
||||
set(ERROR YES)
|
||||
endif (NOT REGRESSION_EXPECTED_LENGTH EQUAL ARGC)
|
||||
|
||||
if (ERROR)
|
||||
message(FATAL_ERROR "Regression failed")
|
||||
endif (ERROR)
|
||||
@ -0,0 +1,13 @@
|
||||
class StationList extends AIInfo {
|
||||
function GetAuthor() { return "OpenTTD NoAI Developers Team"; }
|
||||
function GetName() { return "StationList"; }
|
||||
function GetShortName() { return "REGS"; }
|
||||
function GetDescription() { return "This runs stationlist-tests on some commands. On the same map the result should always be the same."; }
|
||||
function GetVersion() { return 1; }
|
||||
function GetAPIVersion() { return "1.11"; }
|
||||
function GetDate() { return "2007-03-18"; }
|
||||
function CreateInstance() { return "StationList"; }
|
||||
}
|
||||
|
||||
RegisterAI(StationList());
|
||||
|
||||
Binary file not shown.
Loading…
Reference in New Issue