Revision control
Copy as Markdown
#!/usr/bin/env python3
# Copyright (C) 2021 The Qt Company Ltd.
# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
# This is an artificial test, mimicking the Qt tests, for example tst_whatever.
# Its purpose is to assist in testing qt-testrunner.py.
#
# Mode A:
#
# If invoked with a test function argument, it runs that test function.
#
# Usage:
#
# $0 always_pass
# $0 always_fail
# $0 always_crash
# $0 fail_then_pass:N # where N is the number of failing runs before passing
#
# Needs environment variable:
# + QT_MOCK_TEST_STATE_FILE :: points to a unique filename, to be written
# for keeping the results of previous runs and controlling results of
# fail_then_pass:N.
#
# Mode B:
#
# If invoked without any argument, it runs the tests listed in the
# variable QT_MOCK_TEST_RUN_LIST. If variable is empty it just runs
# the always_pass test. It also understands qtestlib's `-o outfile.xml,xml`
# option for writing a mock testlog in a file. Requires environment variables:
# + QT_MOCK_TEST_STATE_FILE :: See above
# + QT_MOCK_TEST_XML_TEMPLATE_FILE :: may point to the template XML file
# located in the same source directory. Without this variable, the
# option `-o outfile.xml,xml` will be ignored.
# + QT_MOCK_TEST_RUN_LIST :: may contain a comma-separated list of test
# that should run.
# - Optionally, a test in that list can end in ~[PFC] in order to
# PASS,FAIL or CRASH respectively.
# + QT_MOCK_TEST_CRASH_CLEANLY :: if set to 1, then the executable will
# crash (exit with a high exit code)
# after successfully running the given tests and writing the XML logfile.
import sys
import os
import re
import traceback
from tst_qt_testrunner import write_xml_log
MY_NAME = os.path.basename(sys.argv[0])
STATE_FILE = None
XML_TEMPLATE = None
XML_OUTPUT_FILE = None
CRASH_CLEANLY = False
# Remember the test functions that were run in this invocation
TEST_FUNCTIONS_RUN = set()
def crash():
sys.exit(131)
# Write the result to the state file.
# @result can be any of the characters PFC, for Pass,Fail,Crash respectively.
# Format of each line in the state file:
# test_name <TAB> [PFC]
def put_result(test_name, result):
assert result in "PFC"
with open(STATE_FILE, "a") as f:
f.write(test_name + "\t" + result + "\n")
# Counts the previous results in the STATE file.
# If result=None then count any result of "test_name.*".
def get_previous_result_count(test_name, result):
n = 0
try:
with open(STATE_FILE) as f:
if result is None:
for line in f:
if line.split()[0].startswith(test_name):
n += 1
else:
for line in f:
if line.split() == [ test_name, result ]:
n += 1
except FileNotFoundError:
return 0
return n
# Only care about the XML log output file.
def parse_output_argument(a):
global XML_OUTPUT_FILE
if a.endswith(",xml"):
XML_OUTPUT_FILE = a[:-4]
# Strip qtestlib specific arguments.
# Only care about the "-o ...,xml" argument.
def clean_cmdline():
args = []
prev_arg = None
skip_next_arg = True # Skip argv[0]
for a in sys.argv:
if skip_next_arg:
if prev_arg == "-o":
parse_output_argument(a)
prev_arg = None
skip_next_arg = False
continue
if a in ("-o", "-maxwarnings"):
skip_next_arg = True
prev_arg = a
continue
if a in ("-v1", "-v2", "-vs"):
print("VERBOSE RUN")
if "QT_LOGGING_RULES" in os.environ:
print("Environment has QT_LOGGING_RULES:",
os.environ["QT_LOGGING_RULES"])
continue
args.append(a)
return args
def log_test(testcase, result,
testsuite=MY_NAME):
print("%-7s: %s::%s()" % (result, testsuite, testcase))
def log_xml(fail_list):
if XML_OUTPUT_FILE and XML_TEMPLATE is not None:
if XML_TEMPLATE == "":
# If the template is an empty file, then write an empty output file
with open(XML_OUTPUT_FILE, "w"):
pass
else:
write_xml_log(XML_OUTPUT_FILE, failure=fail_list)
clean_up_xml(XML_OUTPUT_FILE, TEST_FUNCTIONS_RUN)
# Remove unneeded XML and comments.
# Because the XML is rendered from a template, it contains a lot of garbage.
# Run this *after* you render the XML_OUTPUT_FILE from the XML template.
def clean_up_xml(filename, test_functions_run):
func_name_regex = re.compile(r' *<TestFunction .*name="([^"]+)">')
incident_regex = re.compile(r' *<Incident .*type="\{\{.*\}\}"')
cleaned_up_xml = ""
skipping_function = False
skipping_incident = False
skipping_comment = False
with open(filename) as f:
for line in f:
if skipping_function:
if "</TestFunction>" in line:
skipping_function = False
continue
if skipping_incident:
if "</Incident>" in line:
skipping_incident = False
continue
if skipping_comment:
if "-->" in line:
skipping_comment = False
continue
# Delete parts of the XML file that refer to non-executed test functions.
m1 = func_name_regex.match(line)
if m1:
func_name = m1.group(1)
if func_name not in test_functions_run:
skipping_function = True
continue
# Delete <Incident> tags that has a type that starts with '{{'.
# This means that this part has not been template-replaced by write_xml_log().
m2 = incident_regex.match(line)
if m2:
skipping_incident = True
continue
# Delete multi-line XML comments.
if line.strip().startswith("<!--") and "-->" not in line:
skipping_comment = True
continue
# Keep the rest.
cleaned_up_xml += line
with open(filename, "w") as f:
f.write(cleaned_up_xml)
def PFC_to_exitcode(pfc : str):
if pfc == "P":
return 0
elif pfc == "F":
return 1
elif pfc == "C":
return 131
assert False, "Unreachable! testname ended in ~[^PFC]!"
# Return the exit code.
# desired_result is only implemented for initTestCase.
def run_test(testname, desired_result=None):
assert not desired_result or desired_result in "PFC"
if testname == "initTestCase":
if not desired_result:
desired_result = "F" # by default test that initTestCase fails
exit_code = PFC_to_exitcode(desired_result)
elif testname == "always_pass":
exit_code = 0
elif testname == "always_fail":
exit_code = 1
elif testname == "always_crash":
exit_code = 131
elif testname == "fail_then_crash":
previous_fails = get_previous_result_count(testname, "F")
if previous_fails == 0:
exit_code = 1
else:
exit_code = 131
elif testname.startswith("fail_then_pass"):
wanted_fails = int(testname.partition(":")[2])
previous_fails = get_previous_result_count(testname, "F")
if previous_fails < wanted_fails:
exit_code = 1
else:
exit_code = 0
# This will fail ONCE after exactly 3 passes
# TODO accept a :N datatag to fail after N passes.
# (difficult because qt-testrunner never specifies any datatag in the MFF re-runs).
elif testname.startswith("pass_then_fail_once"):
fail_after = 3
previous_runs_count = get_previous_result_count(testname, None)
if previous_runs_count == fail_after:
exit_code = 1
else:
exit_code = 0
elif testname.startswith("pass_then_crash_once"):
fail_after = 3
previous_runs_count = get_previous_result_count(testname, None)
if previous_runs_count == fail_after:
exit_code = 131
else:
exit_code = 0
elif testname.startswith("pass_then_fail_then_crash_once"):
fail_after = 3
previous_runs_count = get_previous_result_count(testname, None)
if previous_runs_count == fail_after:
exit_code = 1
elif previous_runs_count == fail_after + 1:
exit_code = 131
else:
exit_code = 0
else:
assert False, "Unknown argument: %s" % testname
# Remember the test function name, it's needed in cleaned_up_xml().
TEST_FUNCTIONS_RUN.add(testname.partition(":")[0])
if exit_code == 0:
put_result(testname, "P")
log_test(testname, "PASS")
elif exit_code == 1:
put_result(testname, "F")
log_test(testname, "FAIL!")
else:
put_result(testname, "C")
log_test(testname, "CRASH!")
return exit_code
def no_args_run():
try:
run_list = os.environ["QT_MOCK_TEST_RUN_LIST"].split(",")
except KeyError:
run_list = ["always_pass"]
total_result = True
fail_list = []
for test in run_list:
# Optionally the test ends in ~[PFC] to set the result.
(test, _, desired_result) = test.partition("~")
test_exit_code = run_test(test, desired_result)
if test_exit_code not in (0, 1):
crash()
if test_exit_code != 0:
fail_list.append(test)
total_result = total_result and (test_exit_code == 0)
log_xml(fail_list)
if CRASH_CLEANLY:
# Crash despite all going well and writing all output files cleanly.
crash()
if total_result:
sys.exit(0)
else:
sys.exit(1)
def main():
global STATE_FILE
# Will fail if env var is not set.
STATE_FILE = os.environ["QT_MOCK_TEST_STATE_FILE"]
global XML_TEMPLATE
if "QT_MOCK_TEST_XML_TEMPLATE_FILE" in os.environ:
with open(os.environ["QT_MOCK_TEST_XML_TEMPLATE_FILE"]) as f:
XML_TEMPLATE = f.read()
global CRASH_CLEANLY
if ("QT_MOCK_TEST_CRASH_CLEANLY" in os.environ
and os.environ["QT_MOCK_TEST_CRASH_CLEANLY"] == "1"
):
CRASH_CLEANLY = True
args = clean_cmdline()
if len(args) == 0:
no_args_run()
assert False, "Unreachable!"
else: # run single test function
assert args[0] != "initTestCase", \
"QTest does not support executing initTestCase individually!"
exit_code = run_test(args[0])
# Write "fail" in the XML log only if the specific run has failed.
if exit_code != 0:
log_xml([args[0]])
else:
log_xml([])
sys.exit(exit_code)
# TODO write XPASS test that does exit(1)
if __name__ == "__main__":
try:
main()
except Exception as e:
traceback.print_exc()
exit(128) # Something went wrong with this script