Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions spec/helper.js
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,7 @@ if (dns.setDefaultResultOrder) {
jasmine.DEFAULT_TIMEOUT_INTERVAL = process.env.PARSE_SERVER_TEST_TIMEOUT || 10000;
jasmine.getEnv().addReporter(new CurrentSpecReporter());
jasmine.getEnv().addReporter(new SpecReporter());
global.retryFlakyTests();

global.normalizeAsyncTests();
global.on_db = (db, callback, elseCallback) => {
if (process.env.PARSE_SERVER_TEST_DB == db) {
return callback();
Expand Down
69 changes: 13 additions & 56 deletions spec/support/CurrentSpecReporter.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,21 +4,10 @@ const { performance } = require('perf_hooks');

global.currentSpec = null;

/**
* Names of tests that fail randomly and are considered flaky. These tests will be retried
* a number of times to reduce the chance of false negatives. The test name must be the same
* as the one displayed in the CI log test output.
*/
const flakyTests = [];

/** The minimum execution time in seconds for a test to be considered slow. */
const slowTestLimit = 2;

/** The number of times to retry a flaky test. */
const retries = 5;

const timerMap = {};
const retryMap = {};
const duplicates = [];
class CurrentSpecReporter {
specStarted(spec) {
Expand Down Expand Up @@ -52,61 +41,29 @@ global.displayTestStats = function() {
console.warn('Duplicate spec: ' + spec);
});
console.log('\n');
Object.keys(retryMap).forEach((spec) => {
console.warn(`Flaky test: ${spec} failed ${retryMap[spec]} times`);
});
console.log('\n');
};

global.retryFlakyTests = function() {
/**
* Wraps test functions that use both `async` and a `done` callback, which Jasmine
* does not support. This converts `async (done) => { ... }` to a promise-based
* function so Jasmine does not throw:
* "An asynchronous before/it/after function was defined with the async keyword
* but also took a done callback."
*/
global.normalizeAsyncTests = function() {
const originalSpecConstructor = jasmine.Spec;

jasmine.Spec = function(attrs) {
const spec = new originalSpecConstructor(attrs);
const originalTestFn = spec.queueableFn.fn;
const runOriginalTest = () => {
if (originalTestFn.length == 0) {
// handle async testing
return originalTestFn();
} else {
// handle done() callback
if (originalTestFn.length > 0) {
spec.queueableFn.fn = function() {
return new Promise((resolve) => {
originalTestFn(resolve);
});
}
};
spec.queueableFn.fn = async function() {
const isFlaky = flakyTests.includes(spec.result.fullName);
const runs = isFlaky ? retries : 1;
let exceptionCaught;
let returnValue;

for (let i = 0; i < runs; ++i) {
spec.result.failedExpectations = [];
returnValue = undefined;
exceptionCaught = undefined;
try {
returnValue = await runOriginalTest();
} catch (exception) {
exceptionCaught = exception;
}
const failed = !spec.markedPending &&
(exceptionCaught || spec.result.failedExpectations.length != 0);
if (!failed) {
break;
}
if (isFlaky) {
retryMap[spec.result.fullName] = (retryMap[spec.result.fullName] || 0) + 1;
await global.afterEachFn();
}
}
if (exceptionCaught) {
throw exceptionCaught;
}
return returnValue;
};
};
Comment thread
mtrezza marked this conversation as resolved.
Outdated
}
return spec;
};
}
};

module.exports = CurrentSpecReporter;
Loading