Running Jest tests in parallel in Gitlab with multiple jobs

I've been working a lot lately with Jest tests. Some projects had 1k+ of Jest unit tests. Another project was using Jest for running integration and E2E tests. In both cases we wanted to get feedback on CI as fast as possible. Our initial solution was following:

Split tests into different folderds
Make sure that there's one Jest spec file per test - this way Jest will run tests in parallel better
Create a series of npm scripts in package.json file
Create a list of similar jobs in .gitlab-ci.yml that trigger different npm scripts
Gather together artifacts from this tests and aggregate a one Test Report.

The big problem with this approach is that it's not really scalable. Also, with unit tests it's not always convinient to group them by folders and always check that you have not missed or forgot some folder to inlucde into the test run.

Luckily, one of my colleguas showed me a better approach to this. It appears that Jest has test-sequencer which can be extended. It allows to split tests into separate groups and assign a list of tests per node. Gitlab has a feature that allows you to run the same job in multiple instances. It's called parallelization

This two factors enabled us to write an easily code that would scale in the future with no manual input fromt he developers. I have prepared a small Gist that gathers all the changes that are required to add to your Jest framework and things should just work faster :)

	stages:
	- prepare
	- build
	- test
	- post_test

	unit tests:
	timeout: 20 minutes
	stage: test
	variables:
	CI_NODE_INDEX: $CI_NODE_INDEX
	CI_NODE_TOTAL: $CI_NODE_TOTAL
	parallel: 5
	script:
	- npm run test:ci
	artifacts:
	name: test-reports/
	paths:
	- test-reports/
	cache:
	key: jest
	paths:
	- .tmp/cache/jest/

	# =========================
	# post test
	# =========================
	test report:
	stage: post_test
	needs:
	- unit tests
	script:
	- npm run test:ci:test-report
	artifacts:
	paths:
	- test-reports/
	reports:
	junit: test-reports/test-results.xml

	coverage:
	stage: post_test
	needs:
	- unit tests
	script:
	- npm run test:ci:coverage-report
	artifacts:
	name: test-coverage
	paths:
	- test-reports/
	reports:
	coverage_report:
	coverage_format: cobertura
	path: test-reports/cobertura-coverage.xml
	coverage: '/Total Coverage: (\d+\.\d+\%)/'

view raw .gitlab-ci.yml hosted with ❤ by GitHub

	const { create } = require('istanbul-reports')
	const { createCoverageMap } = require('istanbul-lib-coverage')
	const { createContext } = require('istanbul-lib-report')
	const { resolve } = require('path')
	const { sync } = require('glob')
	const { GLOBAL_THRESHOLDS } = require('../../jest.config')

	const coverageMap = createCoverageMap()

	const REPORTS_FOLDER = 'test-reports'
	const coverageDir = resolve(__dirname, `../../${REPORTS_FOLDER}`)
	const reportFiles = sync(`${coverageDir}/*/coverage/coverage-final.json`)

	const COVERAGE_TYPES = ['lines', 'statements', 'functions', 'branches']

	/* eslint-disable no-console */

	// Normalize coverage report generated by jest that has additional "data" key
	// https://github.com/facebook/jest/issues/2418#issuecomment-423806659
	const normalizeReport = report => {
	const normalizedReport = Object.assign({}, report)
	Object.entries(normalizedReport).forEach(([k, v]) => {
	if (v.data) normalizedReport[k] = v.data
	})
	return normalizedReport
	}

	/**
	* prepare unit test coverage files and merge to single test context to build a report
	*/
	reportFiles
	.map(reportFile => {
	console.log('Found report file: ' + reportFile)
	return require(reportFile)
	})
	.map(normalizeReport)
	.forEach(report => coverageMap.merge(report))
	const context = createContext({
	coverageMap: coverageMap,
	dir: REPORTS_FOLDER,
	})

	/**
	* create and output a Cobertura report for MR coverage visualization
	* https://docs.gitlab.com/ee/ci/testing/test_coverage_visualization.html
	*/
	create('cobertura', {}).execute(context)
	console.log(
	`Cobertura coverage report generated and outputted to ${coverageDir}`
	)

	/**
	* create coverage summary and check for met threshold gates, job should
	* fail if any defined thresholds are not met
	*/
	// create json coverage summary, which also prints summary to terminal
	create('json-summary', {}).execute(context)
	const coverageSummary = require(`${coverageDir}/coverage-summary.json`)

	/**
	* print results to terminal for easy visibility of coverage summary, text-summary
	* will print out directly a coverage summary of each type, and then print a custom
	* total coverage line for use to display total % in merge requests
	*/
	create('text-summary', {}).execute(context)
	const totalSum = COVERAGE_TYPES.map(
	type => coverageSummary.total[type].pct
	).reduce((total, percent) => total + percent, 0)
	const avgCoverage = totalSum / COVERAGE_TYPES.length
	console.debug(`
	========= Total Coverage ==============
	Total Coverage: ${avgCoverage.toFixed(2)}%
	=======================================
	`)

	/**
	* use the JSON summary report to do checks against coverage thresholds that
	* cannot be used in parallelization. check each type and determine if threshold
	* is met, otherwise fail this job
	*/
	const checkCoverageAgainstThresholds = coverageSummary => {
	const total = coverageSummary?.total \|\| {}

	return COVERAGE_TYPES.map(type => {
	// If the threshold is a number use it, otherwise lookup the threshold type
	var threshold = GLOBAL_THRESHOLDS[type]

	// Check for no threshold
	if (!threshold) {
	return {
	type,
	skipped: true,
	failed: false,
	}
	}

	const value = total[type]?.pct
	return {
	type,
	required: threshold,
	value,
	failed: value < threshold,
	}
	})
	}

	const coverageThresholds = checkCoverageAgainstThresholds(coverageSummary)
	const failedCoverageTypes = coverageThresholds.filter(type => type.failed)
	if (failedCoverageTypes.length > 0) {
	failedCoverageTypes.forEach(({ type, value, required }) => {
	console.error(
	'\x1b[31m%s\x1b[0m',
	`Global coverage threshold for ${type} (${required}%) not met: ${value}%`
	)
	})
	console.log(`❌ Coverage thresholds have not been met`)
	process.exit(1)
	}

	console.log(`✅ Coverage thresholds met`)
	process.exit(0)

view raw custom-coverage-reporter.js hosted with ❤ by GitHub

	/**
	* Custom Jest test sequencer, taken from Gitlab's codebase. Used to separate
	* and run Jest test suites in parallel based on available job nodes.
	* Structure is based on the default sequencer from Jest.
	*/
	const Sequencer = require('@jest/test-sequencer').default

	/* eslint-disable no-console */
	class ParallelCiSequencer extends Sequencer {
	constructor() {
	super()
	this.ciNodeIndex = Number(process.env.CI_NODE_INDEX \|\| '1')
	this.ciNodeTotal = Number(process.env.CI_NODE_TOTAL \|\| '1')
	}

	sort(tests) {
	const sortedTests = this.sortByPath(tests)
	const testsForThisRunner = this.distributeAcrossCINodes(sortedTests)

	console.log(`CI_NODE_INDEX: ${this.ciNodeIndex}`)
	console.log(`CI_NODE_TOTAL: ${this.ciNodeTotal}`)
	console.log(`Total number of tests: ${tests.length}`)
	console.log(
	`Total number of tests for this runner: ${testsForThisRunner.length}`
	)

	return testsForThisRunner
	}

	sortByPath(tests) {
	return tests.sort((test1, test2) => {
	if (test1.path < test2.path) {
	return -1
	}
	if (test1.path > test2.path) {
	return 1
	}
	return 0
	})
	}

	distributeAcrossCINodes(tests) {
	return tests.filter((test, index) => {
	return index % this.ciNodeTotal === this.ciNodeIndex - 1
	})
	}
	}

	module.exports = ParallelCiSequencer

view raw jest-tests-custom-sequencer.js hosted with ❤ by GitHub

	const aliasList = require('./internals/webpack/aliasList')

	const IS_PARALLEL_TESTS = process.env.CI_NODE_INDEX && process.env.CI_NODE_TOTAL
	const GLOBAL_THRESHOLDS = {
	branches: 76,
	functions: 81,
	statements: 84,
	lines: 84,
	}

	module.exports = {
	/**
	* for pipeline test steps, include junit report for Gitlab report artifacts
	* and custom coverage reporter for regex of coverage %'s
	*/
	reporters: [
	'default',
	[
	'jest-junit',
	{
	outputDirectory: IS_PARALLEL_TESTS
	? `<rootDir>/test-reports/test-${process.env.CI_NODE_INDEX}-${process.env.CI_NODE_TOTAL}`
	: '<rootDir>/test-reports',
	outputName: 'test-results.xml',
	},
	],
	],
	maxWorkers: 4,
	collectCoverageFrom: ['src/*/.js'],
	coveragePathIgnorePatterns: [
	'/node_modules/(?!intl-messageformat\|intl-messageformat-parser).+\\.js$',
	'/testing/',
	],
	/**
	* for pipeline test steps, include cobertura reporter for Gitlab coverage report
	* and json-summary for custom coverage reporter
	*/
	coverageReporters: [
	'json',
	'lcov',
	['cobertura', { file: 'coverage-results.xml' }],
	// `json-summary`'s `coverage-summary.json` output needed for custom coverage reporter
	'json-summary',
	/**
	* `text-summary` removes giant coverage matrix from terminal print-out,
	* replace with "text" if you wish to see this locally
	*/
	'text-summary',
	],
	// remove coverage thresholds for cicd parallel tests as they will be incomplete suites
	coverageThreshold: IS_PARALLEL_TESTS
	? {}
	: {
	global: GLOBAL_THRESHOLDS,
	},
	coverageDirectory: IS_PARALLEL_TESTS
	? `<rootDir>/test-reports/test-${process.env.CI_NODE_INDEX}-${process.env.CI_NODE_TOTAL}/coverage`
	: '<rootDir>/test-reports/coverage',
	cacheDirectory: '<rootDir>/.tmp/cache/jest',
	// export for use in custom coverage threshold gates when parallel test
	GLOBAL_THRESHOLDS,
	}

view raw jest.config.js hosted with ❤ by GitHub