113481Sgiacomo.travaglini@arm.com#!/usr/bin/env python 213481Sgiacomo.travaglini@arm.com# 313481Sgiacomo.travaglini@arm.com# Copyright 2009 Google Inc. All Rights Reserved. 413481Sgiacomo.travaglini@arm.com# 513481Sgiacomo.travaglini@arm.com# Redistribution and use in source and binary forms, with or without 613481Sgiacomo.travaglini@arm.com# modification, are permitted provided that the following conditions are 713481Sgiacomo.travaglini@arm.com# met: 813481Sgiacomo.travaglini@arm.com# 913481Sgiacomo.travaglini@arm.com# * Redistributions of source code must retain the above copyright 1013481Sgiacomo.travaglini@arm.com# notice, this list of conditions and the following disclaimer. 1113481Sgiacomo.travaglini@arm.com# * Redistributions in binary form must reproduce the above 1213481Sgiacomo.travaglini@arm.com# copyright notice, this list of conditions and the following disclaimer 1313481Sgiacomo.travaglini@arm.com# in the documentation and/or other materials provided with the 1413481Sgiacomo.travaglini@arm.com# distribution. 1513481Sgiacomo.travaglini@arm.com# * Neither the name of Google Inc. nor the names of its 1613481Sgiacomo.travaglini@arm.com# contributors may be used to endorse or promote products derived from 1713481Sgiacomo.travaglini@arm.com# this software without specific prior written permission. 1813481Sgiacomo.travaglini@arm.com# 1913481Sgiacomo.travaglini@arm.com# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 2013481Sgiacomo.travaglini@arm.com# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 2113481Sgiacomo.travaglini@arm.com# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 2213481Sgiacomo.travaglini@arm.com# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 2313481Sgiacomo.travaglini@arm.com# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 2413481Sgiacomo.travaglini@arm.com# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 2513481Sgiacomo.travaglini@arm.com# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 2613481Sgiacomo.travaglini@arm.com# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 2713481Sgiacomo.travaglini@arm.com# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 2813481Sgiacomo.travaglini@arm.com# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 2913481Sgiacomo.travaglini@arm.com# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 3013481Sgiacomo.travaglini@arm.com 3113481Sgiacomo.travaglini@arm.com"""Verifies that test shuffling works.""" 3213481Sgiacomo.travaglini@arm.com 3313481Sgiacomo.travaglini@arm.com__author__ = 'wan@google.com (Zhanyong Wan)' 3413481Sgiacomo.travaglini@arm.com 3513481Sgiacomo.travaglini@arm.comimport os 3613481Sgiacomo.travaglini@arm.comimport gtest_test_utils 3713481Sgiacomo.travaglini@arm.com 3813481Sgiacomo.travaglini@arm.com# Command to run the gtest_shuffle_test_ program. 3913481Sgiacomo.travaglini@arm.comCOMMAND = gtest_test_utils.GetTestExecutablePath('gtest_shuffle_test_') 4013481Sgiacomo.travaglini@arm.com 4113481Sgiacomo.travaglini@arm.com# The environment variables for test sharding. 4213481Sgiacomo.travaglini@arm.comTOTAL_SHARDS_ENV_VAR = 'GTEST_TOTAL_SHARDS' 4313481Sgiacomo.travaglini@arm.comSHARD_INDEX_ENV_VAR = 'GTEST_SHARD_INDEX' 4413481Sgiacomo.travaglini@arm.com 4513481Sgiacomo.travaglini@arm.comTEST_FILTER = 'A*.A:A*.B:C*' 4613481Sgiacomo.travaglini@arm.com 4713481Sgiacomo.travaglini@arm.comALL_TESTS = [] 4813481Sgiacomo.travaglini@arm.comACTIVE_TESTS = [] 4913481Sgiacomo.travaglini@arm.comFILTERED_TESTS = [] 5013481Sgiacomo.travaglini@arm.comSHARDED_TESTS = [] 5113481Sgiacomo.travaglini@arm.com 5213481Sgiacomo.travaglini@arm.comSHUFFLED_ALL_TESTS = [] 5313481Sgiacomo.travaglini@arm.comSHUFFLED_ACTIVE_TESTS = [] 5413481Sgiacomo.travaglini@arm.comSHUFFLED_FILTERED_TESTS = [] 5513481Sgiacomo.travaglini@arm.comSHUFFLED_SHARDED_TESTS = [] 5613481Sgiacomo.travaglini@arm.com 5713481Sgiacomo.travaglini@arm.com 5813481Sgiacomo.travaglini@arm.comdef AlsoRunDisabledTestsFlag(): 5913481Sgiacomo.travaglini@arm.com return '--gtest_also_run_disabled_tests' 6013481Sgiacomo.travaglini@arm.com 6113481Sgiacomo.travaglini@arm.com 6213481Sgiacomo.travaglini@arm.comdef FilterFlag(test_filter): 6313481Sgiacomo.travaglini@arm.com return '--gtest_filter=%s' % (test_filter,) 6413481Sgiacomo.travaglini@arm.com 6513481Sgiacomo.travaglini@arm.com 6613481Sgiacomo.travaglini@arm.comdef RepeatFlag(n): 6713481Sgiacomo.travaglini@arm.com return '--gtest_repeat=%s' % (n,) 6813481Sgiacomo.travaglini@arm.com 6913481Sgiacomo.travaglini@arm.com 7013481Sgiacomo.travaglini@arm.comdef ShuffleFlag(): 7113481Sgiacomo.travaglini@arm.com return '--gtest_shuffle' 7213481Sgiacomo.travaglini@arm.com 7313481Sgiacomo.travaglini@arm.com 7413481Sgiacomo.travaglini@arm.comdef RandomSeedFlag(n): 7513481Sgiacomo.travaglini@arm.com return '--gtest_random_seed=%s' % (n,) 7613481Sgiacomo.travaglini@arm.com 7713481Sgiacomo.travaglini@arm.com 7813481Sgiacomo.travaglini@arm.comdef RunAndReturnOutput(extra_env, args): 7913481Sgiacomo.travaglini@arm.com """Runs the test program and returns its output.""" 8013481Sgiacomo.travaglini@arm.com 8113481Sgiacomo.travaglini@arm.com environ_copy = os.environ.copy() 8213481Sgiacomo.travaglini@arm.com environ_copy.update(extra_env) 8313481Sgiacomo.travaglini@arm.com 8413481Sgiacomo.travaglini@arm.com return gtest_test_utils.Subprocess([COMMAND] + args, env=environ_copy).output 8513481Sgiacomo.travaglini@arm.com 8613481Sgiacomo.travaglini@arm.com 8713481Sgiacomo.travaglini@arm.comdef GetTestsForAllIterations(extra_env, args): 8813481Sgiacomo.travaglini@arm.com """Runs the test program and returns a list of test lists. 8913481Sgiacomo.travaglini@arm.com 9013481Sgiacomo.travaglini@arm.com Args: 9113481Sgiacomo.travaglini@arm.com extra_env: a map from environment variables to their values 9213481Sgiacomo.travaglini@arm.com args: command line flags to pass to gtest_shuffle_test_ 9313481Sgiacomo.travaglini@arm.com 9413481Sgiacomo.travaglini@arm.com Returns: 9513481Sgiacomo.travaglini@arm.com A list where the i-th element is the list of tests run in the i-th 9613481Sgiacomo.travaglini@arm.com test iteration. 9713481Sgiacomo.travaglini@arm.com """ 9813481Sgiacomo.travaglini@arm.com 9913481Sgiacomo.travaglini@arm.com test_iterations = [] 10013481Sgiacomo.travaglini@arm.com for line in RunAndReturnOutput(extra_env, args).split('\n'): 10113481Sgiacomo.travaglini@arm.com if line.startswith('----'): 10213481Sgiacomo.travaglini@arm.com tests = [] 10313481Sgiacomo.travaglini@arm.com test_iterations.append(tests) 10413481Sgiacomo.travaglini@arm.com elif line.strip(): 10513481Sgiacomo.travaglini@arm.com tests.append(line.strip()) # 'TestCaseName.TestName' 10613481Sgiacomo.travaglini@arm.com 10713481Sgiacomo.travaglini@arm.com return test_iterations 10813481Sgiacomo.travaglini@arm.com 10913481Sgiacomo.travaglini@arm.com 11013481Sgiacomo.travaglini@arm.comdef GetTestCases(tests): 11113481Sgiacomo.travaglini@arm.com """Returns a list of test cases in the given full test names. 11213481Sgiacomo.travaglini@arm.com 11313481Sgiacomo.travaglini@arm.com Args: 11413481Sgiacomo.travaglini@arm.com tests: a list of full test names 11513481Sgiacomo.travaglini@arm.com 11613481Sgiacomo.travaglini@arm.com Returns: 11713481Sgiacomo.travaglini@arm.com A list of test cases from 'tests', in their original order. 11813481Sgiacomo.travaglini@arm.com Consecutive duplicates are removed. 11913481Sgiacomo.travaglini@arm.com """ 12013481Sgiacomo.travaglini@arm.com 12113481Sgiacomo.travaglini@arm.com test_cases = [] 12213481Sgiacomo.travaglini@arm.com for test in tests: 12313481Sgiacomo.travaglini@arm.com test_case = test.split('.')[0] 12413481Sgiacomo.travaglini@arm.com if not test_case in test_cases: 12513481Sgiacomo.travaglini@arm.com test_cases.append(test_case) 12613481Sgiacomo.travaglini@arm.com 12713481Sgiacomo.travaglini@arm.com return test_cases 12813481Sgiacomo.travaglini@arm.com 12913481Sgiacomo.travaglini@arm.com 13013481Sgiacomo.travaglini@arm.comdef CalculateTestLists(): 13113481Sgiacomo.travaglini@arm.com """Calculates the list of tests run under different flags.""" 13213481Sgiacomo.travaglini@arm.com 13313481Sgiacomo.travaglini@arm.com if not ALL_TESTS: 13413481Sgiacomo.travaglini@arm.com ALL_TESTS.extend( 13513481Sgiacomo.travaglini@arm.com GetTestsForAllIterations({}, [AlsoRunDisabledTestsFlag()])[0]) 13613481Sgiacomo.travaglini@arm.com 13713481Sgiacomo.travaglini@arm.com if not ACTIVE_TESTS: 13813481Sgiacomo.travaglini@arm.com ACTIVE_TESTS.extend(GetTestsForAllIterations({}, [])[0]) 13913481Sgiacomo.travaglini@arm.com 14013481Sgiacomo.travaglini@arm.com if not FILTERED_TESTS: 14113481Sgiacomo.travaglini@arm.com FILTERED_TESTS.extend( 14213481Sgiacomo.travaglini@arm.com GetTestsForAllIterations({}, [FilterFlag(TEST_FILTER)])[0]) 14313481Sgiacomo.travaglini@arm.com 14413481Sgiacomo.travaglini@arm.com if not SHARDED_TESTS: 14513481Sgiacomo.travaglini@arm.com SHARDED_TESTS.extend( 14613481Sgiacomo.travaglini@arm.com GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', 14713481Sgiacomo.travaglini@arm.com SHARD_INDEX_ENV_VAR: '1'}, 14813481Sgiacomo.travaglini@arm.com [])[0]) 14913481Sgiacomo.travaglini@arm.com 15013481Sgiacomo.travaglini@arm.com if not SHUFFLED_ALL_TESTS: 15113481Sgiacomo.travaglini@arm.com SHUFFLED_ALL_TESTS.extend(GetTestsForAllIterations( 15213481Sgiacomo.travaglini@arm.com {}, [AlsoRunDisabledTestsFlag(), ShuffleFlag(), RandomSeedFlag(1)])[0]) 15313481Sgiacomo.travaglini@arm.com 15413481Sgiacomo.travaglini@arm.com if not SHUFFLED_ACTIVE_TESTS: 15513481Sgiacomo.travaglini@arm.com SHUFFLED_ACTIVE_TESTS.extend(GetTestsForAllIterations( 15613481Sgiacomo.travaglini@arm.com {}, [ShuffleFlag(), RandomSeedFlag(1)])[0]) 15713481Sgiacomo.travaglini@arm.com 15813481Sgiacomo.travaglini@arm.com if not SHUFFLED_FILTERED_TESTS: 15913481Sgiacomo.travaglini@arm.com SHUFFLED_FILTERED_TESTS.extend(GetTestsForAllIterations( 16013481Sgiacomo.travaglini@arm.com {}, [ShuffleFlag(), RandomSeedFlag(1), FilterFlag(TEST_FILTER)])[0]) 16113481Sgiacomo.travaglini@arm.com 16213481Sgiacomo.travaglini@arm.com if not SHUFFLED_SHARDED_TESTS: 16313481Sgiacomo.travaglini@arm.com SHUFFLED_SHARDED_TESTS.extend( 16413481Sgiacomo.travaglini@arm.com GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', 16513481Sgiacomo.travaglini@arm.com SHARD_INDEX_ENV_VAR: '1'}, 16613481Sgiacomo.travaglini@arm.com [ShuffleFlag(), RandomSeedFlag(1)])[0]) 16713481Sgiacomo.travaglini@arm.com 16813481Sgiacomo.travaglini@arm.com 16913481Sgiacomo.travaglini@arm.comclass GTestShuffleUnitTest(gtest_test_utils.TestCase): 17013481Sgiacomo.travaglini@arm.com """Tests test shuffling.""" 17113481Sgiacomo.travaglini@arm.com 17213481Sgiacomo.travaglini@arm.com def setUp(self): 17313481Sgiacomo.travaglini@arm.com CalculateTestLists() 17413481Sgiacomo.travaglini@arm.com 17513481Sgiacomo.travaglini@arm.com def testShufflePreservesNumberOfTests(self): 17613481Sgiacomo.travaglini@arm.com self.assertEqual(len(ALL_TESTS), len(SHUFFLED_ALL_TESTS)) 17713481Sgiacomo.travaglini@arm.com self.assertEqual(len(ACTIVE_TESTS), len(SHUFFLED_ACTIVE_TESTS)) 17813481Sgiacomo.travaglini@arm.com self.assertEqual(len(FILTERED_TESTS), len(SHUFFLED_FILTERED_TESTS)) 17913481Sgiacomo.travaglini@arm.com self.assertEqual(len(SHARDED_TESTS), len(SHUFFLED_SHARDED_TESTS)) 18013481Sgiacomo.travaglini@arm.com 18113481Sgiacomo.travaglini@arm.com def testShuffleChangesTestOrder(self): 18213481Sgiacomo.travaglini@arm.com self.assert_(SHUFFLED_ALL_TESTS != ALL_TESTS, SHUFFLED_ALL_TESTS) 18313481Sgiacomo.travaglini@arm.com self.assert_(SHUFFLED_ACTIVE_TESTS != ACTIVE_TESTS, SHUFFLED_ACTIVE_TESTS) 18413481Sgiacomo.travaglini@arm.com self.assert_(SHUFFLED_FILTERED_TESTS != FILTERED_TESTS, 18513481Sgiacomo.travaglini@arm.com SHUFFLED_FILTERED_TESTS) 18613481Sgiacomo.travaglini@arm.com self.assert_(SHUFFLED_SHARDED_TESTS != SHARDED_TESTS, 18713481Sgiacomo.travaglini@arm.com SHUFFLED_SHARDED_TESTS) 18813481Sgiacomo.travaglini@arm.com 18913481Sgiacomo.travaglini@arm.com def testShuffleChangesTestCaseOrder(self): 19013481Sgiacomo.travaglini@arm.com self.assert_(GetTestCases(SHUFFLED_ALL_TESTS) != GetTestCases(ALL_TESTS), 19113481Sgiacomo.travaglini@arm.com GetTestCases(SHUFFLED_ALL_TESTS)) 19213481Sgiacomo.travaglini@arm.com self.assert_( 19313481Sgiacomo.travaglini@arm.com GetTestCases(SHUFFLED_ACTIVE_TESTS) != GetTestCases(ACTIVE_TESTS), 19413481Sgiacomo.travaglini@arm.com GetTestCases(SHUFFLED_ACTIVE_TESTS)) 19513481Sgiacomo.travaglini@arm.com self.assert_( 19613481Sgiacomo.travaglini@arm.com GetTestCases(SHUFFLED_FILTERED_TESTS) != GetTestCases(FILTERED_TESTS), 19713481Sgiacomo.travaglini@arm.com GetTestCases(SHUFFLED_FILTERED_TESTS)) 19813481Sgiacomo.travaglini@arm.com self.assert_( 19913481Sgiacomo.travaglini@arm.com GetTestCases(SHUFFLED_SHARDED_TESTS) != GetTestCases(SHARDED_TESTS), 20013481Sgiacomo.travaglini@arm.com GetTestCases(SHUFFLED_SHARDED_TESTS)) 20113481Sgiacomo.travaglini@arm.com 20213481Sgiacomo.travaglini@arm.com def testShuffleDoesNotRepeatTest(self): 20313481Sgiacomo.travaglini@arm.com for test in SHUFFLED_ALL_TESTS: 20413481Sgiacomo.travaglini@arm.com self.assertEqual(1, SHUFFLED_ALL_TESTS.count(test), 20513481Sgiacomo.travaglini@arm.com '%s appears more than once' % (test,)) 20613481Sgiacomo.travaglini@arm.com for test in SHUFFLED_ACTIVE_TESTS: 20713481Sgiacomo.travaglini@arm.com self.assertEqual(1, SHUFFLED_ACTIVE_TESTS.count(test), 20813481Sgiacomo.travaglini@arm.com '%s appears more than once' % (test,)) 20913481Sgiacomo.travaglini@arm.com for test in SHUFFLED_FILTERED_TESTS: 21013481Sgiacomo.travaglini@arm.com self.assertEqual(1, SHUFFLED_FILTERED_TESTS.count(test), 21113481Sgiacomo.travaglini@arm.com '%s appears more than once' % (test,)) 21213481Sgiacomo.travaglini@arm.com for test in SHUFFLED_SHARDED_TESTS: 21313481Sgiacomo.travaglini@arm.com self.assertEqual(1, SHUFFLED_SHARDED_TESTS.count(test), 21413481Sgiacomo.travaglini@arm.com '%s appears more than once' % (test,)) 21513481Sgiacomo.travaglini@arm.com 21613481Sgiacomo.travaglini@arm.com def testShuffleDoesNotCreateNewTest(self): 21713481Sgiacomo.travaglini@arm.com for test in SHUFFLED_ALL_TESTS: 21813481Sgiacomo.travaglini@arm.com self.assert_(test in ALL_TESTS, '%s is an invalid test' % (test,)) 21913481Sgiacomo.travaglini@arm.com for test in SHUFFLED_ACTIVE_TESTS: 22013481Sgiacomo.travaglini@arm.com self.assert_(test in ACTIVE_TESTS, '%s is an invalid test' % (test,)) 22113481Sgiacomo.travaglini@arm.com for test in SHUFFLED_FILTERED_TESTS: 22213481Sgiacomo.travaglini@arm.com self.assert_(test in FILTERED_TESTS, '%s is an invalid test' % (test,)) 22313481Sgiacomo.travaglini@arm.com for test in SHUFFLED_SHARDED_TESTS: 22413481Sgiacomo.travaglini@arm.com self.assert_(test in SHARDED_TESTS, '%s is an invalid test' % (test,)) 22513481Sgiacomo.travaglini@arm.com 22613481Sgiacomo.travaglini@arm.com def testShuffleIncludesAllTests(self): 22713481Sgiacomo.travaglini@arm.com for test in ALL_TESTS: 22813481Sgiacomo.travaglini@arm.com self.assert_(test in SHUFFLED_ALL_TESTS, '%s is missing' % (test,)) 22913481Sgiacomo.travaglini@arm.com for test in ACTIVE_TESTS: 23013481Sgiacomo.travaglini@arm.com self.assert_(test in SHUFFLED_ACTIVE_TESTS, '%s is missing' % (test,)) 23113481Sgiacomo.travaglini@arm.com for test in FILTERED_TESTS: 23213481Sgiacomo.travaglini@arm.com self.assert_(test in SHUFFLED_FILTERED_TESTS, '%s is missing' % (test,)) 23313481Sgiacomo.travaglini@arm.com for test in SHARDED_TESTS: 23413481Sgiacomo.travaglini@arm.com self.assert_(test in SHUFFLED_SHARDED_TESTS, '%s is missing' % (test,)) 23513481Sgiacomo.travaglini@arm.com 23613481Sgiacomo.travaglini@arm.com def testShuffleLeavesDeathTestsAtFront(self): 23713481Sgiacomo.travaglini@arm.com non_death_test_found = False 23813481Sgiacomo.travaglini@arm.com for test in SHUFFLED_ACTIVE_TESTS: 23913481Sgiacomo.travaglini@arm.com if 'DeathTest.' in test: 24013481Sgiacomo.travaglini@arm.com self.assert_(not non_death_test_found, 24113481Sgiacomo.travaglini@arm.com '%s appears after a non-death test' % (test,)) 24213481Sgiacomo.travaglini@arm.com else: 24313481Sgiacomo.travaglini@arm.com non_death_test_found = True 24413481Sgiacomo.travaglini@arm.com 24513481Sgiacomo.travaglini@arm.com def _VerifyTestCasesDoNotInterleave(self, tests): 24613481Sgiacomo.travaglini@arm.com test_cases = [] 24713481Sgiacomo.travaglini@arm.com for test in tests: 24813481Sgiacomo.travaglini@arm.com [test_case, _] = test.split('.') 24913481Sgiacomo.travaglini@arm.com if test_cases and test_cases[-1] != test_case: 25013481Sgiacomo.travaglini@arm.com test_cases.append(test_case) 25113481Sgiacomo.travaglini@arm.com self.assertEqual(1, test_cases.count(test_case), 25213481Sgiacomo.travaglini@arm.com 'Test case %s is not grouped together in %s' % 25313481Sgiacomo.travaglini@arm.com (test_case, tests)) 25413481Sgiacomo.travaglini@arm.com 25513481Sgiacomo.travaglini@arm.com def testShuffleDoesNotInterleaveTestCases(self): 25613481Sgiacomo.travaglini@arm.com self._VerifyTestCasesDoNotInterleave(SHUFFLED_ALL_TESTS) 25713481Sgiacomo.travaglini@arm.com self._VerifyTestCasesDoNotInterleave(SHUFFLED_ACTIVE_TESTS) 25813481Sgiacomo.travaglini@arm.com self._VerifyTestCasesDoNotInterleave(SHUFFLED_FILTERED_TESTS) 25913481Sgiacomo.travaglini@arm.com self._VerifyTestCasesDoNotInterleave(SHUFFLED_SHARDED_TESTS) 26013481Sgiacomo.travaglini@arm.com 26113481Sgiacomo.travaglini@arm.com def testShuffleRestoresOrderAfterEachIteration(self): 26213481Sgiacomo.travaglini@arm.com # Get the test lists in all 3 iterations, using random seed 1, 2, 26313481Sgiacomo.travaglini@arm.com # and 3 respectively. Google Test picks a different seed in each 26413481Sgiacomo.travaglini@arm.com # iteration, and this test depends on the current implementation 26513481Sgiacomo.travaglini@arm.com # picking successive numbers. This dependency is not ideal, but 26613481Sgiacomo.travaglini@arm.com # makes the test much easier to write. 26713481Sgiacomo.travaglini@arm.com [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = ( 26813481Sgiacomo.travaglini@arm.com GetTestsForAllIterations( 26913481Sgiacomo.travaglini@arm.com {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)])) 27013481Sgiacomo.travaglini@arm.com 27113481Sgiacomo.travaglini@arm.com # Make sure running the tests with random seed 1 gets the same 27213481Sgiacomo.travaglini@arm.com # order as in iteration 1 above. 27313481Sgiacomo.travaglini@arm.com [tests_with_seed1] = GetTestsForAllIterations( 27413481Sgiacomo.travaglini@arm.com {}, [ShuffleFlag(), RandomSeedFlag(1)]) 27513481Sgiacomo.travaglini@arm.com self.assertEqual(tests_in_iteration1, tests_with_seed1) 27613481Sgiacomo.travaglini@arm.com 27713481Sgiacomo.travaglini@arm.com # Make sure running the tests with random seed 2 gets the same 27813481Sgiacomo.travaglini@arm.com # order as in iteration 2 above. Success means that Google Test 27913481Sgiacomo.travaglini@arm.com # correctly restores the test order before re-shuffling at the 28013481Sgiacomo.travaglini@arm.com # beginning of iteration 2. 28113481Sgiacomo.travaglini@arm.com [tests_with_seed2] = GetTestsForAllIterations( 28213481Sgiacomo.travaglini@arm.com {}, [ShuffleFlag(), RandomSeedFlag(2)]) 28313481Sgiacomo.travaglini@arm.com self.assertEqual(tests_in_iteration2, tests_with_seed2) 28413481Sgiacomo.travaglini@arm.com 28513481Sgiacomo.travaglini@arm.com # Make sure running the tests with random seed 3 gets the same 28613481Sgiacomo.travaglini@arm.com # order as in iteration 3 above. Success means that Google Test 28713481Sgiacomo.travaglini@arm.com # correctly restores the test order before re-shuffling at the 28813481Sgiacomo.travaglini@arm.com # beginning of iteration 3. 28913481Sgiacomo.travaglini@arm.com [tests_with_seed3] = GetTestsForAllIterations( 29013481Sgiacomo.travaglini@arm.com {}, [ShuffleFlag(), RandomSeedFlag(3)]) 29113481Sgiacomo.travaglini@arm.com self.assertEqual(tests_in_iteration3, tests_with_seed3) 29213481Sgiacomo.travaglini@arm.com 29313481Sgiacomo.travaglini@arm.com def testShuffleGeneratesNewOrderInEachIteration(self): 29413481Sgiacomo.travaglini@arm.com [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = ( 29513481Sgiacomo.travaglini@arm.com GetTestsForAllIterations( 29613481Sgiacomo.travaglini@arm.com {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)])) 29713481Sgiacomo.travaglini@arm.com 29813481Sgiacomo.travaglini@arm.com self.assert_(tests_in_iteration1 != tests_in_iteration2, 29913481Sgiacomo.travaglini@arm.com tests_in_iteration1) 30013481Sgiacomo.travaglini@arm.com self.assert_(tests_in_iteration1 != tests_in_iteration3, 30113481Sgiacomo.travaglini@arm.com tests_in_iteration1) 30213481Sgiacomo.travaglini@arm.com self.assert_(tests_in_iteration2 != tests_in_iteration3, 30313481Sgiacomo.travaglini@arm.com tests_in_iteration2) 30413481Sgiacomo.travaglini@arm.com 30513481Sgiacomo.travaglini@arm.com def testShuffleShardedTestsPreservesPartition(self): 30613481Sgiacomo.travaglini@arm.com # If we run M tests on N shards, the same M tests should be run in 30713481Sgiacomo.travaglini@arm.com # total, regardless of the random seeds used by the shards. 30813481Sgiacomo.travaglini@arm.com [tests1] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', 30913481Sgiacomo.travaglini@arm.com SHARD_INDEX_ENV_VAR: '0'}, 31013481Sgiacomo.travaglini@arm.com [ShuffleFlag(), RandomSeedFlag(1)]) 31113481Sgiacomo.travaglini@arm.com [tests2] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', 31213481Sgiacomo.travaglini@arm.com SHARD_INDEX_ENV_VAR: '1'}, 31313481Sgiacomo.travaglini@arm.com [ShuffleFlag(), RandomSeedFlag(20)]) 31413481Sgiacomo.travaglini@arm.com [tests3] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', 31513481Sgiacomo.travaglini@arm.com SHARD_INDEX_ENV_VAR: '2'}, 31613481Sgiacomo.travaglini@arm.com [ShuffleFlag(), RandomSeedFlag(25)]) 31713481Sgiacomo.travaglini@arm.com sorted_sharded_tests = tests1 + tests2 + tests3 31813481Sgiacomo.travaglini@arm.com sorted_sharded_tests.sort() 31913481Sgiacomo.travaglini@arm.com sorted_active_tests = [] 32013481Sgiacomo.travaglini@arm.com sorted_active_tests.extend(ACTIVE_TESTS) 32113481Sgiacomo.travaglini@arm.com sorted_active_tests.sort() 32213481Sgiacomo.travaglini@arm.com self.assertEqual(sorted_active_tests, sorted_sharded_tests) 32313481Sgiacomo.travaglini@arm.com 32413481Sgiacomo.travaglini@arm.comif __name__ == '__main__': 32513481Sgiacomo.travaglini@arm.com gtest_test_utils.Main() 326