Revision control
++ #!/usr/bin/env perl
++ use 5.010;
++ use strict;
++ use warnings;
++
++ =head1 NAME
++
++ 50-testplanner.t - basic test for testscheduler.pl
++
++ =cut
++
++ use English qw(-no_match_vars);
++ use File::Spec::Functions;
++ use File::Temp;
++ use File::chdir;
++ use FindBin;
++ use Readonly;
++ use ReleaseAction qw(on_release);
++ use Test::More;
++ use Text::Diff;
++ use Capture::Tiny qw(capture_merged);
++
++ use lib "$FindBin::Bin/../../lib/perl5";
++ use QtQA::Test::More qw(find_qmake);
++
++ Readonly my $TESTPLANNER => catfile( $FindBin::Bin, qw(.. testplanner.pl) );
++
++ Readonly my $TESTSCHEDULER => catfile( $FindBin::Bin, qw(.. testscheduler.pl) );
++
++ Readonly my $TESTDATA_DIR => catfile( $FindBin::Bin, qw(data test-projects) );
++
++ Readonly my $QMAKE => find_qmake( );
++
++ # Regular expression snippets
++ Readonly my %RE => make_regexes( );
++
++ # Returns various regex parts used in the test, mostly to do with timing.
++ # We do not generally test that the actual time values are correct.
++ sub make_regexes
++ {
++ my %out = (
++
++ # a positive time, less than 1 minute
++ time_seconds => qr{< 1 second|1 second|\d+ seconds},
++
++ # no time at all
++ no_time => qr{\(no time\)},
++
++ # Timing header line
++ timing_header => qr{
++ \Q=== Timing: =================== TEST RUN COMPLETED! ============================\E
++ }xms,
++
++ );
++
++ $out{ timing_section_begin } = qr|
++ $out{ timing_header } \n
++ [ ]+ Total: [ ]+ $out{ time_seconds } \n
++ |xms;
++
++ $out{ timing_section_j1 } = qr|
++ $out{ timing_section_begin }
++ [ ]+ \QEstimated time spent on insignificant tests:\E [ ]+ $out{ no_time } \n
++ |xms;
++
++ $out{ timing_section_j1_with_insignificant } = qr|
++ $out{ timing_section_begin }
++ [ ]+ \QEstimated time spent on insignificant tests:\E [ ]+ $out{ time_seconds } \n
++ |xms;
++
++ $out{ timing_section_j4_with_insignificant } = qr|
++ $out{ timing_section_begin }
++ [ ]+ \QSerial tests:\E [ ]+ $out{ time_seconds } \n
++ [ ]+ \QParallel tests:\E [ ]+ $out{ time_seconds } \n
++ [ ]+ \QEstimated time spent on insignificant tests:\E [ ]+ $out{ time_seconds } \n
++ [ ]+ \QEstimated time saved by -j4:\E [ ]+ $out{ time_seconds } \n
++ |xms;
++
++ return %out;
++ }
++
++ # Given a $directory, returns a filename pointing to a created testplan,
++ # and a handle for unlinking that filename.
++ # Any additional @args are passed to testplanner.
++ sub make_testplan_from_directory
++ {
++ my ($directory, @args) = @_;
++
++ my $testplan = File::Temp->new(
++ TEMPLATE => 'qtqa-testplan-XXXXXX',
++ TMPDIR => 1,
++ );
++ $testplan = "$testplan";
++
++ my @cmd = (
++ $EXECUTABLE_NAME,
++ $TESTPLANNER,
++ '--input',
++ $directory,
++ '--output',
++ "$testplan",
++ );
++
++ if ($OSNAME =~ m{win32}i) {
++ if (!system( 'where', "/Q", "nmake" )) {
++ push @cmd, (
++ '--make',
++ 'nmake',
++ );
++ } elsif (!system( 'where', "/Q", "mingw32-make" )) {
++ push @cmd, (
++ '--make',
++ 'mingw32-make',
++ );
++ } # else - use default
++ } # else - use default
++
++ push (@cmd, @args);
++
++ my $status = system( @cmd );
++ is( $status, 0, 'testplanner exit code OK' );
++
++ my $on_release = on_release { unlink $testplan };
++
++ return ($testplan, $on_release);
++ }
++
++ sub test_mixed
++ {
++ my ($testplan, $unlink) = make_testplan_from_directory $TESTDATA_DIR;
++
++ my $status;
++ my $output = capture_merged {
++ $status = system(
++ $EXECUTABLE_NAME,
++ $TESTSCHEDULER,
++ '--plan',
++ "$testplan",
++ '-j4',
++ '--sync-output',
++ '--verbose',
++ );
++ };
++ isnt( $status, 0, 'testscheduler fails if some tests fail' );
++ like( $output, qr|
++ \A
++ \QQtQA::App::TestRunner: begin failing_significant_test @ \E[^\n]+\Q
++ failing. 1 arg(s)
++ QtQA::App::TestRunner: end failing_significant_test: \E[^\n]+\Q
++ QtQA::App::TestScheduler: failing_significant_test failed; run concurrently with passing_insignificant_test
++ QtQA::App::TestRunner: begin passing_insignificant_test @ \E[^\n]+\Q
++ passing. 1 arg(s)
++ QtQA::App::TestRunner: end passing_insignificant_test: \E[^\n]+\Q
++ QtQA::App::TestScheduler: ran 2 parallel tests. Starting 6 serial tests.
++ QtQA::App::TestRunner: begin failing_custom_check_target @ \E[^\n]+\Q
++ Custom failing
++ QtQA::App::TestRunner: end failing_custom_check_target: \E[^\n]+\Q
++ QtQA::App::TestScheduler: failing_custom_check_target failed
++ QtQA::App::TestRunner: begin failing_insignificant_test @ \E[^\n]+\Q
++ failing. 1 arg(s)
++ QtQA::App::TestRunner: end failing_insignificant_test: \E[^\n]+\Q
++ QtQA::App::TestScheduler: failing_insignificant_test failed, but it is marked with insignificant_test
++ QtQA::App::TestScheduler: passing_custom_check_target: ignored invalid testcase.timeout value of "-120"
++ QtQA::App::TestRunner: begin passing_custom_check_target @ \E[^\n]+\Q
++ Custom passing
++ QtQA::App::TestRunner: end passing_custom_check_target: \E[^\n]+\Q
++ QtQA::App::TestRunner: begin passing_significant_test @ \E[^\n]+\Q
++ passing. 1 arg(s)
++ QtQA::App::TestRunner: end passing_significant_test: \E[^\n]+\Q
++ QtQA::App::TestScheduler: subtest (sub1): ignored invalid testcase.timeout value of "Some bogus timeout"
++ QtQA::App::TestRunner: begin subtest (sub1) @ \E[^\n]+\Q
++ failing. 1 arg(s)
++ QtQA::App::TestRunner: end subtest (sub1): \E[^\n]+\Q
++ QtQA::App::TestScheduler: subtest (sub1) failed
++ QtQA::App::TestRunner: begin subtest (sub2) @ \E[^\n]+\Q
++ failing. 1 arg(s)
++ QtQA::App::TestRunner: end subtest (sub2): \E[^\n]+\Q
++ QtQA::App::TestScheduler: subtest (sub2) failed
++ \E $RE{ timing_section_j4_with_insignificant }
++ \Q=== Failures: ==================================================================
++ failing_custom_check_target
++ failing_significant_test
++ subtest (sub1)
++ subtest (sub2)
++ failing_insignificant_test [insignificant]
++ === Totals: 8 tests, 3 passes, 4 fails, 1 insignificant fail ===================
++ \E
++ \z|xms, 'testscheduler output as expected' );
++
++ return;
++ }
++
++ sub test_mixed_parallel_stress
++ {
++ my ($testplan, $unlink) = make_testplan_from_directory $TESTDATA_DIR;
++
++ my $status;
++ my $output = capture_merged {
++ $status = system(
++ $EXECUTABLE_NAME,
++ $TESTSCHEDULER,
++ '--plan',
++ "$testplan",
++ '-j4',
++ '--sync-output',
++ '--parallel-stress'
++ );
++ };
++ isnt( $status, 0, '[parallel-stress] testscheduler fails if some tests fail' );
++
++
++ # We don't test the output from the tests, the order is unpredictable.
++ # We also don't test "Suggest removing ...", because we would have to write
++ # a test script which _stably_ fails if run concurrently with itself, and
++ # otherwise passes. This is a basic test.
++
++ like( $output, qr|\Q
++ === Parallel stress test: ======================================================
++ Suggest adding CONFIG+=parallel_test to these:
++ passing_custom_check_target
++ passing_significant_test
++ === 3 parallel-safe, 0 parallel-unsafe, 5 unknown, 2 to modify =================
++ === Failures: ==================================================================
++ failing_custom_check_target
++ failing_significant_test
++ subtest (sub1)
++ subtest (sub2)
++ failing_insignificant_test [insignificant]
++ === Totals: 8 tests, 3 passes, 4 fails, 1 insignificant fail ===================
++ \E
++ \z|xms, '[parallel-stress] testscheduler output as expected' );
++
++ return;
++ }
++
++ # Test what happens with a directory containing no tests
++ sub test_none
++ {
++ my ($testplan, $unlink) = make_testplan_from_directory "$TESTDATA_DIR/not_tests";
++
++ my $status;
++ my $output = capture_merged {
++ $status = system(
++ $EXECUTABLE_NAME,
++ $TESTSCHEDULER,
++ '--plan',
++ "$testplan",
++ );
++ };
++ is( $status, 0, 'testscheduler with no tests is a pass' );
++ like( $output, qr|
++ \A
++ $RE{ timing_section_j1 }
++ \Q=== Totals: no tests, no passes ================================================\E \n
++ \z
++ |xms, 'testscheduler output as expected' );
++
++ return;
++ }
++
++ sub test_single_fail
++ {
++ my ($testplan, $unlink) = make_testplan_from_directory "$TESTDATA_DIR/tests/failing_significant_test";
++
++ my $status;
++ my $output = capture_merged {
++ $status = system(
++ $EXECUTABLE_NAME,
++ $TESTSCHEDULER,
++ '--plan',
++ "$testplan",
++ '--', # test trailing -- is harmless
++ );
++ };
++ isnt( $status, 0, 'testscheduler with single fail is a fail' );
++ like( $output, qr|
++ \A
++ \Qfailing. 1 arg(s)
++ QtQA::App::TestScheduler: failing_significant_test failed
++ \E $RE{ timing_section_j1 }
++ \Q=== Failures: ==================================================================
++ failing_significant_test
++ === Totals: 1 test, no passes, 1 fail ==========================================
++ \E
++ \z|xms, 'testscheduler output as expected' );
++
++ return;
++ }
++
++ sub test_single_insignificant_fail
++ {
++ # Testing with some additional args
++ my ($testplan, $unlink) = make_testplan_from_directory(
++ "$TESTDATA_DIR/tests/failing_insignificant_test",
++ '--',
++ 'arg1',
++ 'arg2',
++ );
++
++ my $status;
++ my $output = capture_merged {
++ $status = system(
++ $EXECUTABLE_NAME,
++ $TESTSCHEDULER,
++ '--plan',
++ "$testplan",
++ );
++ };
++ is( $status, 0, 'testscheduler with single insignificant fail is a pass' );
++ like( $output, qr|
++ \A
++ \Qfailing. 3 arg(s)
++ QtQA::App::TestScheduler: failing_insignificant_test failed, but it is marked with insignificant_test
++ \E $RE{ timing_section_j1_with_insignificant }
++ \Q=== Failures: ==================================================================
++ failing_insignificant_test [insignificant]
++ === Totals: 1 test, no passes, 1 insignificant fail ============================
++ \E
++ \z|xms, 'testscheduler output as expected' );
++
++ return;
++ }
++
++ sub test_single_pass
++ {
++ my ($testplan, $unlink) = make_testplan_from_directory "$TESTDATA_DIR/tests/passing_significant_test";
++
++ my $status;
++ my $output = capture_merged {
++ $status = system(
++ $EXECUTABLE_NAME,
++ $TESTSCHEDULER,
++ '--plan',
++ "$testplan",
++ );
++ };
++ is( $status, 0, 'testscheduler with single pass is a pass' );
++ like( $output, qr|
++ \A
++ \Qpassing. 1 arg(s)
++ \E $RE{ timing_section_j1 }
++ \Q=== Totals: 1 test, 1 pass =====================================================
++ \E
++ \z|xms, 'testscheduler output as expected' );
++
++ return;
++ }
++
++ sub test_single_pass_no_summary
++ {
++ my ($testplan, $unlink) = make_testplan_from_directory "$TESTDATA_DIR/tests/passing_significant_test";
++
++ my $status;
++ my $output = capture_merged {
++ $status = system(
++ $EXECUTABLE_NAME,
++ $TESTSCHEDULER,
++ '--plan',
++ "$testplan",
++ '--no-summary',
++ );
++ };
++ is( $status, 0, 'testscheduler with single pass and --no-summary is a pass' );
++ is( $output, "passing. 1 arg(s)\n", 'testscheduler output as expected' );
++
++ return;
++ }
++
++ # test that the "run concurrently with" message is correctly truncated if too long
++ sub test_concurrently_limit
++ {
++ my ($testplan, $unlink) = make_testplan_from_directory( catfile( $TESTDATA_DIR, 'parallel_tests' ) );
++
++ my $status;
++ my $output = capture_merged {
++ $status = system(
++ $EXECUTABLE_NAME,
++ $TESTSCHEDULER,
++ '--plan',
++ "$testplan",
++ '-j4',
++ );
++ };
++ isnt( $status, 0, 'testscheduler fails if some tests fail' );
++ like( $output, qr|
++ \QQtQA::App::TestScheduler: fail1 failed; run concurrently with pass0, pass1, pass2, pass3, [3 other tests], pass7, pass8, pass9\E
++ |xms, 'testscheduler run concurrently message is correctly truncated' );
++
++ return;
++ }
++
++ sub run
++ {
++ if (!$QMAKE) {
++ plan skip_all => 'no qmake available for testing';
++ }
++
++ # qmake the testdata before doing anything else.
++ {
++ local $CWD = $TESTDATA_DIR;
++ # we use qmake -r so we can access the makefiles at any level,
++ # and we disable debug and release for predictable results
++ my $status = system( $QMAKE, '-r' );
++ is( $status, 0, 'qmake ran OK' );
++ }
++
++ test_none;
++ test_single_fail;
++ test_single_insignificant_fail;
++ test_single_pass;
++ test_single_pass_no_summary;
++ test_mixed;
++ test_mixed_parallel_stress;
++ test_concurrently_limit;
++ done_testing;
++
++ return;
++ }
++
++ run if (!caller);
++ 1;
++