Revision control
++ #!/usr/bin/env perl
++ use 5.010;
++ use strict;
++ use warnings;
++
++ =head1 NAME
++
++ 01-testrunner-testcocoon.t - test testrunner's 'testcocoon' plugin for code coverage analysis.
++
++ =head1 SYNOPSIS
++
++ perl ./01-testrunner-testcocoon.t
++
++ This test will run the testrunner.pl script with some fake testcocoon processes
++ and verify that the testcocoon plugin generates the expected output.
++
++ =cut
++
++ use Capture::Tiny qw( capture );
++ use Cwd;
++ use English qw( -no_match_vars );
++ use Env::Path;
++ use File::Path qw( mkpath );
++ use File::Spec::Functions;
++ use File::Temp qw( tempdir );
++ use FindBin;
++ use Readonly;
++ use Test::More;
++
++ use lib "$FindBin::Bin/../../lib/perl5";
++ use QtQA::Test::More qw( is_or_like create_mock_command );
++
++ # Directory separator, quoted for regex
++ Readonly my $DS => ($OSNAME eq 'MSWin32') ? q{\\\\} : q{/};
++
++ # Tempdir testcocoon for tests template
++ Readonly my $TEMPDIR_TOPLEVEL => catfile(File::Spec->tmpdir);
++
++ Readonly my $TEMPDIR_TEMPLATE =>
++ qr{\Q$TEMPDIR_TOPLEVEL\E${DS}testcocoon_plugin\..{6}}sm;
++
++ # Commands used multiple times
++
++ Readonly my $MYTEST_DIR => qr{${TEMPDIR_TEMPLATE}${DS}module${DS}mytest}sm;
++
++ Readonly my $GLOBAL_TEST =>
++ qr{$MYTEST_DIR${DS}mytest_global\.csmes}sm;
++
++ Readonly my $COVERAGERUNNER_CMMERGE_OTHERCSMES =>
++ qr{\+ cmmerge --append --output=$GLOBAL_TEST $MYTEST_DIR${DS}mytest\.csmes.*};
++ Readonly my $COVERAGERUNNER_CMMERGE_PLUGIN_WITH_TEST =>
++ qr{\+ cmmerge --append --output=$GLOBAL_TEST ${TEMPDIR_TEMPLATE}${DS}(?:module${DS}\.\.${DS})?qtbase${DS}plugins${DS}plugin1${DS}plugin1\.csmes.*};
++ Readonly my $COVERAGERUNNER_CMCSEXEIMPORT =>
++ qr{\+ cmcsexeimport --csmes=$GLOBAL_TEST --csexe=$MYTEST_DIR${DS}myother\.csexe --title=tc_mytest --policy=merge.*};
++ Readonly my $COVERAGERUNNER_CMCSEXEIMPORT_OTHER =>
++ qr{\+ cmcsexeimport --csmes=$GLOBAL_TEST --csexe=$MYTEST_DIR${DS}mytest\.csexe --title=tc_mytest --policy=merge.*};
++ Readonly my $COVERAGERUNNER_CMMERGE_TEST_WITH_GLOBAL =>
++ qr{\+ cmmerge --append --output=${TEMPDIR_TEMPLATE}${DS}tests\.csmes $GLOBAL_TEST.*};
++
++ # Missing --testcocoon-tests-output required option
++ Readonly my $MISSING_REQUIRED_TESTS_OUTPUT =>
++ qr{internal error: .*${DS}testcocoon\.pm loaded OK, but QtQA::App::TestRunner::Plugin::testcocoon could not be instantiated: Missing required '--testcocoon-tests-output' option at .*${DS}testcocoon\.pm line .*};
++
++ # Missing --testcocoon-qt-gitmodule-dir required option
++ Readonly my $MISSING_REQUIRED_GITDIR =>
++ qr{internal error: .*${DS}testcocoon\.pm loaded OK, but QtQA::App::TestRunner::Plugin::testcocoon could not be instantiated: Invalid or missing required '--testcocoon-qt-gitmodule-dir' option at .*${DS}testcocoon\.pm line .*};
++
++ # Missing --testcocoon-qt-gitmodule required option
++ Readonly my $MISSING_REQUIRED_MODULE =>
++ qr{internal error: .*${DS}testcocoon\.pm loaded OK, but QtQA::App::TestRunner::Plugin::testcocoon could not be instantiated: Missing required '--testcocoon-qt-gitmodule' option at .*${DS}testcocoon\.pm line .*};
++
++ # Cmmerge other csmes
++ Readonly my $CMMERGE_OTHERCSMES =>
++ qr{test success.*
++ $COVERAGERUNNER_CMMERGE_OTHERCSMES}sm;
++
++ # Cmmerge with a plugin
++ Readonly my $CMMERGE_PLUGIN_WITH_TEST =>
++ qr{$CMMERGE_OTHERCSMES
++ cmmerge success other csmes.*
++ $COVERAGERUNNER_CMMERGE_PLUGIN_WITH_TEST}sm;
++
++ # Cmcsexeimport
++ Readonly my $CMCSEXEIMPORT =>
++ qr{$CMMERGE_PLUGIN_WITH_TEST
++ cmmerge success plugin with test.*
++ $COVERAGERUNNER_CMCSEXEIMPORT}sm;
++
++ # Cmcsexeimport other
++ Readonly my $CMCSEXEIMPORT_OTHER =>
++ qr{$CMCSEXEIMPORT
++ cmcsexeimport success.*
++ $COVERAGERUNNER_CMCSEXEIMPORT_OTHER}sm;
++
++ # Cmmerge test into global
++ Readonly my $CMMERGE_TEST_INTO_GLOBAL =>
++ qr{$CMCSEXEIMPORT_OTHER
++ cmcsexeimport other success.*
++ $COVERAGERUNNER_CMMERGE_TEST_WITH_GLOBAL}sm;
++
++ # Successful run
++ Readonly my $TESTSCRIPT_SUCCESS =>
++ qr{$CMMERGE_TEST_INTO_GLOBAL
++ cmmerge success test into global.*}sm;
++
++ sub test_run
++ {
++ my ($params_ref) = @_;
++
++ my @args = @{$params_ref->{ args }};
++ my $expected_stdout = $params_ref->{ expected_stdout };
++ my $expected_stderr = $params_ref->{ expected_stderr };
++ my $expected_success = $params_ref->{ expected_success };
++ my $testname = $params_ref->{ testname } // q{};
++
++ my $status;
++ my ($output, $error) = capture {
++ $status = system( 'perl', "$FindBin::Bin/../testrunner.pl", @args );
++ };
++
++ if ($expected_success) {
++ is ( $status, 0, "$testname exits zero" );
++ }
++ else {
++ isnt( $status, 0, "$testname exits non-zero" );
++ }
++
++ is_or_like( $output, $expected_stdout, "$testname output looks correct" );
++ is_or_like( $error, $expected_stderr, "$testname error looks correct" );
++
++ return;
++ }
++
++ sub test_success
++ {
++ my $tempdir = tempdir( 'testcocoon_plugin.XXXXXX', TMPDIR => 1, CLEANUP => 1 );
++ init_test_env($tempdir);
++
++ # Put our mock command first in PATH
++ local $ENV{ PATH } = $ENV{ PATH };
++ Env::Path->PATH->Prepend( $tempdir );
++
++ create_mock_command(
++ name => 'mytest',
++ directory => $tempdir,
++ sequence => [
++ { exitcode => 0, stdout => "test success\n"},
++ ],
++ );
++
++ create_mock_command(
++ name => 'cmmerge',
++ directory => $tempdir,
++ sequence => [
++ { exitcode => 0, stdout => "cmmerge success other csmes\n"},
++ { exitcode => 0, stdout => "cmmerge success plugin with test\n"},
++ { exitcode => 0, stdout => "cmmerge success test into global\n"},
++ ],
++ );
++
++ create_mock_command(
++ name => 'cmcsexeimport',
++ directory => $tempdir,
++ sequence => [
++ { exitcode => 0, stdout => "cmcsexeimport success\n"},
++ { exitcode => 0, stdout => "cmcsexeimport other success\n"},
++ ],
++ );
++
++ my $initdir = getcwd;
++ my $testdir = catfile( $tempdir, 'module', 'mytest');
++ chdir($testdir);
++
++ my $module_gitdir = catfile( $tempdir, 'module');
++ my $tests_csmes = catfile( $tempdir, 'tests.csmes' );
++
++ test_run({
++ testname => 'test success',
++ args => [ '--plugin', 'testcocoon', '--testcocoon-tests-output', $tests_csmes, '--testcocoon-qt-gitmodule-dir', $module_gitdir, '--testcocoon-qt-gitmodule', 'module', '--', "$tempdir/mytest",'testname' ],
++ expected_success => 1,
++ expected_stdout => $TESTSCRIPT_SUCCESS,
++ expected_stderr => q{},
++ });
++
++ chdir($initdir);
++
++ return;
++ }
++
++ sub test_invalid_tests_output_arg
++ {
++ my $tempdir = tempdir( 'testcocoon_plugin.XXXXXX', TMPDIR => 1, CLEANUP => 1 );
++
++ test_run({
++ testname => 'missing test output',
++ args => [ '--plugin', 'testcocoon', '--testcocoon-qt-gitmodule-dir', 'fake_gitmodule_dir', '--', "$tempdir/mytest",'testname' ],
++ expected_success => 0,
++ expected_stdout => q{},
++ expected_stderr => $MISSING_REQUIRED_TESTS_OUTPUT,
++ });
++
++ return;
++ }
++
++ sub test_invalid_qt_gitmodule_dir
++ {
++ my $tempdir = tempdir( 'testcocoon_plugin.XXXXXX', TMPDIR => 1, CLEANUP => 1 );
++ my $tests_csmes = catfile( $tempdir, 'tests.csmes' );
++
++ test_run({
++ testname => 'missing git module dir',
++ args => [ '--plugin', 'testcocoon', '--testcocoon-tests-output', $tests_csmes, '--', "$tempdir/mytest",'testname' ],
++ expected_success => 0,
++ expected_stdout => q{},
++ expected_stderr => $MISSING_REQUIRED_GITDIR,
++ });
++
++ return;
++ }
++
++ sub test_missing_qt_gitmodule
++ {
++ my $tempdir = tempdir( 'testcocoon_plugin.XXXXXX', TMPDIR => 1, CLEANUP => 1 );
++ init_test_env($tempdir);
++
++ my $module_gitdir = catfile( $tempdir, 'module');
++ my $tests_csmes = catfile( $tempdir, 'tests.csmes' );
++
++ test_run({
++ testname => 'missing git module name',
++ args => [ '--plugin', 'testcocoon', '--testcocoon-qt-gitmodule-dir', $module_gitdir, '--testcocoon-tests-output', $tests_csmes, '--', "$tempdir/mytest",'testname' ],
++ expected_success => 0,
++ expected_stdout => q{},
++ expected_stderr => $MISSING_REQUIRED_MODULE,
++ });
++
++ return;
++ }
++
++ sub test_merge_other_csmes_failure
++ {
++ my $tempdir = tempdir( 'testcocoon_plugin.XXXXXX', TMPDIR => 1, CLEANUP => 1 );
++ init_test_env($tempdir);
++
++ # Put our mock command first in PATH
++ local $ENV{ PATH } = $ENV{ PATH };
++ Env::Path->PATH->Prepend( $tempdir );
++
++ create_mock_command(
++ name => 'mytest',
++ directory => $tempdir,
++ sequence => [
++ { exitcode => 0, stdout => "test success\n"},
++ ],
++ );
++
++ create_mock_command(
++ name => 'cmmerge',
++ directory => $tempdir,
++ sequence => [
++ { exitcode => 1, stderr => "cmmerge failure other csmes\n"},
++ ],
++ );
++
++ my $initdir = getcwd;
++ my $testdir = catfile( $tempdir, 'module', 'mytest');
++ chdir($testdir);
++
++ my $module_gitdir = catfile( $tempdir, 'module');
++ my $tests_csmes = catfile( $tempdir, 'tests.csmes' );
++
++ test_run({
++ testname => 'cmmerge other csmes failure',
++ args => [ '--plugin', 'testcocoon', '--testcocoon-tests-output', $tests_csmes, '--testcocoon-qt-gitmodule-dir', $module_gitdir, '--testcocoon-qt-gitmodule', 'module', '--', "$tempdir/mytest",'testname' ],
++ expected_success => 0,
++ expected_stdout => $CMMERGE_OTHERCSMES,
++ expected_stderr => qr{cmmerge failure other csmes\n},
++ });
++
++ chdir($initdir);
++
++ return;
++ }
++
++ sub test_cmmerge_plugin_with_test_failure
++ {
++ my $tempdir = tempdir( 'testcocoon_plugin.XXXXXX', TMPDIR => 1, CLEANUP => 1 );
++ init_test_env($tempdir);
++
++ # Put our mock command first in PATH
++ local $ENV{ PATH } = $ENV{ PATH };
++ Env::Path->PATH->Prepend( $tempdir );
++
++ create_mock_command(
++ name => 'mytest',
++ directory => $tempdir,
++ sequence => [
++ { exitcode => 0, stdout => "test success\n"},
++ ],
++ );
++
++ create_mock_command(
++ name => 'cmmerge',
++ directory => $tempdir,
++ sequence => [
++ { exitcode => 0, stdout => "cmmerge success other csmes\n"},
++ { exitcode => 1, stderr => "cmmerge failure plugin\n"},
++ ],
++ );
++
++ my $initdir = getcwd;
++ my $testdir = catfile( $tempdir, 'module', 'mytest');
++ chdir($testdir);
++
++ my $module_gitdir = catfile( $tempdir, 'module');
++ my $tests_csmes = catfile( $tempdir, 'tests.csmes' );
++
++ test_run({
++ testname => 'cmmerge into global plugins failure',
++ args => [ '--plugin', 'testcocoon', '--testcocoon-tests-output', $tests_csmes, '--testcocoon-qt-gitmodule-dir', $module_gitdir, '--testcocoon-qt-gitmodule', 'module', '--', "$tempdir/mytest",'testname' ],
++ expected_success => 0,
++ expected_stdout => $CMMERGE_PLUGIN_WITH_TEST,
++ expected_stderr => qr{cmmerge failure plugin\n},
++ });
++
++ chdir($initdir);
++
++ return;
++ }
++
++ sub test_cmcsexeimport_failure
++ {
++ my $tempdir = tempdir( 'testcocoon_plugin.XXXXXX', TMPDIR => 1, CLEANUP => 1 );
++ init_test_env($tempdir);
++
++ # Put our mock command first in PATH
++ local $ENV{ PATH } = $ENV{ PATH };
++ Env::Path->PATH->Prepend( $tempdir );
++
++ create_mock_command(
++ name => 'mytest',
++ directory => $tempdir,
++ sequence => [
++ { exitcode => 0, stdout => "test success\n"},
++ ],
++ );
++
++ create_mock_command(
++ name => 'cmmerge',
++ directory => $tempdir,
++ sequence => [
++ { exitcode => 0, stdout => "cmmerge success other csmes\n"},
++ { exitcode => 0, stdout => "cmmerge success plugin with test\n"},
++ ],
++ );
++
++ create_mock_command(
++ name => 'cmcsexeimport',
++ directory => $tempdir,
++ sequence => [
++ { exitcode => 3, stderr => "cmcsexeimport failure\n"},
++ ],
++ );
++
++ my $initdir = getcwd;
++ my $testdir = catfile( $tempdir, 'module', 'mytest');
++ chdir($testdir);
++
++ my $module_gitdir = catfile( $tempdir, 'module');
++ my $tests_csmes = catfile( $tempdir, 'tests.csmes' );
++
++ test_run({
++ testname => 'cmcsexeimport failure',
++ args => [ '--plugin', 'testcocoon', '--testcocoon-tests-output', $tests_csmes, '--testcocoon-qt-gitmodule-dir', $module_gitdir, '--testcocoon-qt-gitmodule', 'module', '--', "$tempdir/mytest",'testname' ],
++ expected_success => 0,
++ expected_stdout => $CMCSEXEIMPORT,
++ expected_stderr => qr{cmcsexeimport failure\n},
++ });
++
++ chdir($initdir);
++
++ return;
++ }
++
++ sub test_cmmerge_test_into_global_failure
++ {
++ my $tempdir = tempdir( 'testcocoon_plugin.XXXXXX', TMPDIR => 1, CLEANUP => 1 );
++ init_test_env($tempdir);
++
++ # Put our mock command first in PATH
++ local $ENV{ PATH } = $ENV{ PATH };
++ Env::Path->PATH->Prepend( $tempdir );
++
++ create_mock_command(
++ name => 'mytest',
++ directory => $tempdir,
++ sequence => [
++ { exitcode => 0, stdout => "test success\n"},
++ ],
++ );
++
++ create_mock_command(
++ name => 'cmmerge',
++ directory => $tempdir,
++ sequence => [
++ { exitcode => 0, stdout => "cmmerge success other csmes\n"},
++ { exitcode => 0, stdout => "cmmerge success plugin with test\n"},
++ { exitcode => 4, stderr => "cmmerge failure test into global\n"},
++ ],
++ );
++
++ create_mock_command(
++ name => 'cmcsexeimport',
++ directory => $tempdir,
++ sequence => [
++ { exitcode => 0, stdout => "cmcsexeimport success\n"},
++ { exitcode => 0, stdout => "cmcsexeimport other success\n"},
++ ],
++ );
++
++ my $initdir = getcwd;
++ my $testdir = catfile( $tempdir, 'module', 'mytest');
++ chdir($testdir);
++
++ my $module_gitdir = catfile( $tempdir, 'module');
++ my $tests_csmes = catfile( $tempdir, 'tests.csmes' );
++
++ test_run({
++ testname => 'cmmerge test into global failure',
++ args => [ '--plugin', 'testcocoon', '--testcocoon-tests-output', $tests_csmes, '--testcocoon-qt-gitmodule-dir', $module_gitdir, '--testcocoon-qt-gitmodule', 'module', '--', "$tempdir/mytest",'testname' ],
++ expected_success => 0,
++ expected_stdout => $CMMERGE_TEST_INTO_GLOBAL,
++ expected_stderr => qr{cmmerge failure test into global\n},
++ });
++
++ chdir($initdir);
++
++ return;
++ }
++
++ sub init_test_env
++ {
++ my ($tempdir) = @_;
++
++ # Create qtbase path
++ my $qtbase_path = catfile($tempdir, 'qtbase');
++ if (! -d $qtbase_path && ! mkpath( $qtbase_path )) {
++ die "mkpath $qtbase_path: $!";
++ }
++
++ # Create "module" path
++ my $gitmodule_path = catfile($tempdir, 'module');
++ if (! -d $gitmodule_path && ! mkpath( $gitmodule_path )) {
++ die "mkpath $gitmodule_path: $!";
++ }
++
++ # Create "mytest" path
++ my $mytest_path = catfile($gitmodule_path, 'mytest');
++ if (! -d $mytest_path && ! mkpath( $mytest_path )) {
++ die "mkpath $mytest_path: $!";
++ }
++
++ # Create qtbase plugins path
++ my $plugins_path = catfile($qtbase_path, 'plugins');
++ if (! -d $plugins_path && ! mkpath( $plugins_path )) {
++ die "mkpath $plugins_path: $!";
++ }
++
++ my $plugin1_path = catfile($plugins_path, 'plugin1');
++ if (! -d $plugin1_path && ! mkpath( $plugin1_path )) {
++ die "mkpath $plugin1_path: $!";
++ }
++
++ # Create csmes for 1 plugin
++ open(my $plugin1, ">", "$plugin1_path/plugin1.csmes") or die $!;
++ close($plugin1);
++
++ # Create current test csexe and csmes
++ open(my $mytestcsmes, ">", "$mytest_path/mytest.csmes") or die $!;
++ close($mytestcsmes);
++ open(my $mytestcsexe, ">", "$mytest_path/mytest.csexe") or die $!;
++ close($mytestcsexe);
++ open(my $myothercsmes, ">", "$mytest_path/myother.csmes") or die $!;
++ close($myothercsmes);
++ open(my $myothercsexe, ">", "$mytest_path/myother.csexe") or die $!;
++ close($myothercsexe);
++ open(my $moccsexe, ">", "$mytest_path/moc.csexe") or die $!;
++ close($moccsexe);
++
++ # Create a tests result database csmes
++ open(my $test, ">", "$tempdir/tests.csmes") or die $!;
++ close($test);
++
++ return;
++ }
++
++ sub run
++ {
++ test_success;
++ test_invalid_tests_output_arg;
++ test_invalid_qt_gitmodule_dir;
++ test_merge_other_csmes_failure;
++ test_cmmerge_plugin_with_test_failure;
++ test_cmcsexeimport_failure;
++ test_cmmerge_test_into_global_failure;
++ test_missing_qt_gitmodule;
++
++ done_testing;
++
++ return;
++ }
++
++ run if (!caller);
++ 1;
++