Revision control
++ # Copyright (C) 2017 The Qt Company Ltd.
++ # SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
++
++ package QtQA::App::TestRunner::Plugin::flaky;
++ use strict;
++ use warnings;
++
++ use Carp;
++ use Getopt::Long qw(GetOptionsFromArray);
++ use List::Util qw(max);
++ use Readonly;
++
++ # different flaky modes.
++
++ # WORST: always take the worst result
++ # (i.e. fail if an autotest fails at least once)
++ Readonly my $WORST => 'worst';
++
++ # BEST: always take the best result
++ # (i.e. pass if an autotest passes at least once)
++ Readonly my $BEST => 'best';
++
++ # IGNORE: ignore result
++ # (i.e. pass if the autotest is unstable, regardless of
++ # the pass/fail result of the autotest)
++ Readonly my $IGNORE => 'ignore';
++
++ Readonly my %FLAKY_MODES => (
++ $WORST => 1,
++ $BEST => 1,
++ $IGNORE => 1,
++ );
++
++ sub new
++ {
++ my ($class, %args) = @_;
++
++ $args{ attempt } = 1;
++
++ # `WORST' is essentially equal to operating only in an advisory mode,
++ # so it is the safest default.
++ my $mode = $WORST;
++
++ GetOptionsFromArray( $args{ argv },
++ 'flaky-mode=s' => \$mode,
++ ) || pod2usage(1);
++
++ if (!$FLAKY_MODES{ $mode }) {
++ die "`$mode' is not a valid --flaky-mode; try one of ".join(q{,}, keys %FLAKY_MODES);
++ }
++
++ $args{ mode } = $mode;
++
++ return bless \%args, $class;
++ }
++
++ sub run_completed
++ {
++ my ($self) = @_;
++
++ my $testrunner = $self->{ testrunner };
++ my $proc = $testrunner->proc( );
++
++ my $status = $proc->status( );
++
++ if ($self->{ attempt } == 1) {
++ # First try, test has failed ...
++ if ($status) {
++ ++$self->{ attempt };
++ $self->{ first_attempt_status } = $status;
++ $testrunner->print_info( "test failed, running again to see if it is flaky...\n" );
++ return { retry => 1 };
++ }
++
++ # First try, test has succeeded ...
++ return;
++ }
++
++ # Second try, test gave same results both times...
++ if ($status == $self->{ first_attempt_status }) {
++ $testrunner->print_info( "test failure could be reproduced twice consecutively\n" );
++ return;
++ }
++
++ # Second try, test gave different results each time.
++ return $self->handle_flaky_test( $self->{ first_attempt_status }, $status );
++ }
++
++ sub about_to_run
++ {
++ my ($self, $args_ref) = @_;
++
++ # on attempt other than the first, omit '-silent' argument, so we get all
++ # details about the failure.
++ if ($self->{ attempt } > 1) {
++ @{ $args_ref } = grep { $_ ne '-silent' } @{ $args_ref };
++ }
++
++ return;
++ }
++
++ # Once a test has been determined as definitely being flaky,
++ # this function will do something based on the current flaky mode.
++ sub handle_flaky_test
++ {
++ my ($self, $first_status, $second_status) = @_;
++
++ if ($first_status == 0) {
++ confess 'internal error: should not be called if test passed on first attempt';
++ }
++
++ my $testrunner = $self->{ testrunner };
++
++ if ($second_status == 0) {
++ $testrunner->print_info(
++ "test failed on first attempt and passed on second attempt!\n"
++ .' first attempt: exited with '.$self->format_status( $first_status )."\n"
++ );
++ }
++ else {
++ $testrunner->print_info(
++ "test failed on first and second attempts, but with different behavior each time:\n"
++ .' first attempt: exited with '.$self->format_status( $first_status )."\n"
++ .' second attempt: exited with '.$self->format_status( $second_status )."\n"
++ );
++ }
++
++ $testrunner->print_info( "the test seems to be flaky, please fix this\n" );
++
++ if ($self->{ mode } eq $IGNORE) {
++ $testrunner->print_info( "this flaky test is being ignored\n" );
++ $testrunner->proc( )->{ status } = 0;
++ }
++ elsif ($self->{ mode } eq $BEST && $second_status == 0) {
++ $testrunner->print_info( "this flaky test is being treated as a PASS\n" );
++ }
++ else {
++ $testrunner->print_info( "this flaky test is being treated as a FAIL\n" );
++ # We need to tell the caller to force a failure, otherwise it will
++ # consider that the test has passed. Take the "worst" exit code
++ # (well, a higher exit code doesn't necessarily imply worse results,
++ # but testlib uses the number of failures as an exitcode, and also high
++ # exitcodes stand out more easily in test logs).
++ my $exitcode = max( $first_status >> 8, $second_status >> 8 ) || 1;
++ return { force_failure_exitcode => $exitcode };
++ }
++
++ return;
++ }
++
++ # Given an exit status, return a human-readable string for the corresponding
++ # exit code or signal. e.g., transform 139 into "signal 11".
++ sub format_status
++ {
++ my ($self, $status) = @_;
++
++ if ($status == -1) {
++ return "status $status";
++ }
++
++ my $signal = ($status & 127);
++ if ($signal) {
++ return "signal $signal"
++ }
++
++ return "exit code ".($status >> 8);
++ }
++
++ # Compares two status values and returns 1 if they should be considered equal for the purpose
++ # of determining test stability.
++ sub status_eq
++ {
++ my ($self, $a, $b) = @_;
++
++ # We don't care if the process dumped core, since this is done by the OS _after_ the
++ # process already crashed. It has no bearing on the stability of a failure.
++ $a |= 128;
++ $b |= 128;
++
++ return ($a == $b);
++ }
++
++ =head1 NAME
++
++ QtQA::App::TestRunner::Plugin::flaky - try to handle unstable autotests
++
++ =head1 SYNOPSIS
++
++ # default: advisory mode only
++ $ testrunner --plugin flaky --capture-logs $HOME/test-logs -- tst_flaky; echo $?
++ ********* Start testing of tst_Flaky *********
++ Config: Using QTest library 5.0.0, Qt 5.0.0
++ PASS : tst_Flaky::initTestCase()
++ FAIL! : tst_Flaky::some_function() (The quux was not bar)
++ PASS : tst_Flaky::cleanupTestCase()
++ Totals: 1 passed, 1 failed, 0 skipped
++ ********* Finished testing of tst_Flaky *********
++ QtQA::App::TestRunner: test failed, running again to see if it is flaky...
++ ********* Start testing of tst_Flaky *********
++ Config: Using QTest library 5.0.0, Qt 5.0.0
++ PASS : tst_Flaky::initTestCase()
++ PASS : tst_Flaky::some_function()
++ PASS : tst_Flaky::cleanupTestCase()
++ Totals: 3 passed, 0 failed, 0 skipped
++ ********* Finished testing of tst_Flaky *********
++ QtQA::App::TestRunner: test failed on first attempt and passed on second attempt!
++ QtQA::App::TestRunner: the test seems to be flaky, please fix this
++ QtQA::App::TestRunner: this flaky test is being treated as a FAIL
++ 1
++
++ # can also permit or ignore flaky tests ...
++ $ testrunner --plugin flaky --flaky-mode best -- tst_flaky; echo $?
++ ********* Start testing of tst_Flaky *********
++ Config: Using QTest library 5.0.0, Qt 5.0.0
++ PASS : tst_Flaky::initTestCase()
++ FAIL! : tst_Flaky::some_function() (The quux was not bar)
++ PASS : tst_Flaky::cleanupTestCase()
++ Totals: 1 passed, 1 failed, 0 skipped
++ ********* Finished testing of tst_Flaky *********
++ QtQA::App::TestRunner: test failed, running again to see if it is flaky...
++ ********* Start testing of tst_Flaky *********
++ Config: Using QTest library 5.0.0, Qt 5.0.0
++ PASS : tst_Flaky::initTestCase()
++ PASS : tst_Flaky::some_function()
++ PASS : tst_Flaky::cleanupTestCase()
++ Totals: 3 passed, 0 failed, 0 skipped
++ ********* Finished testing of tst_Flaky *********
++ QtQA::App::TestRunner: test failed on first attempt and passed on second attempt!
++ QtQA::App::TestRunner: the test seems to be flaky, please fix this
++ QtQA::App::TestRunner: this flaky test is being treated as a PASS
++ 0
++
++ =head1 DESCRIPTION
++
++ This plugin provides a simple mechanism to help determine if an autotest failure
++ is stable. When active, any failing autotest will be re-run at least once to check
++ if the failure can be reproduced.
++
++ If the failing autotest was initially run with the '-silent' argument, this argument
++ will be omitted on the second run.
++
++ An autotest which fails twice in a row, but with a different exit status each time,
++ is also considered unstable (example: a test which fails "normally" once, but
++ segfaults at the second run).
++
++ By default, the plugin does not override the pass/fail state of the test.
++ This can be configured by the B<--flaky-mode> argument, which accepts the values:
++
++ =over
++
++ =item B<worst>
++
++ Always take the worst result of an autotest as the canonical result (default).
++
++ =item B<best>
++
++ Always take the best result of an autotest as the canonical result.
++
++ =item B<ignore>
++
++ Ignore any autotest which gives unstable results.
++
++ =back
++
++ To aid in the understanding of the difference between these values,
++ the following table is provided which enumerates all possible cases:
++
++ +======================================================================================+
++ | flaky-mode | pass | stable fail | fail then pass | fail then fail differently |
++ +============+========+==============+==================+==============================+
++ | worst | PASS | FAIL | FAIL | FAIL |
++ | best | PASS | FAIL | PASS | FAIL |
++ | ignore | PASS | FAIL | PASS | PASS |
++ +======================================================================================+
++
++ =head1 CAVEATS
++
++ Note that this can only prove when a test is I<unstable>.
++ Even running a test successfully one trillion times wouldn't prove that it's
++ stable.
++
++ Use of any flaky-mode other than B<worst> may lead to genuine issues being hidden
++ indefinitely.
++
++ =cut
++
++ 1;