Skip to content

Commit 1bca384

Browse files
vlsivlsi
committed
ci: add randomized matrix for better test coverage
See https://github.com/vlsi/github-actions-random-matrix
1 parent bd04dda commit 1bca384

File tree

4 files changed

+323
-26
lines changed

4 files changed

+323
-26
lines changed

‎.github/workflows/main.yml‎

Lines changed: 35 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -18,42 +18,51 @@ concurrency:
1818
cancel-in-progress: true
1919

2020
jobs:
21-
windows:
22-
name: 'Windows (JDK 17)'
23-
runs-on: windows-latest
21+
matrix_prep:
22+
name: Matrix Preparation
23+
runs-on: ubuntu-latest
24+
outputs:
25+
matrix: ${{ steps.set-matrix.outputs.matrix }}
26+
env:
27+
# Ask matrix.js to produce 5 jobs
28+
MATRIX_JOBS: 5
2429
steps:
25-
- uses: actions/[email protected]
26-
with:
27-
fetch-depth: 50
28-
- name: 'Set up JDK 17'
29-
uses: actions/setup-java@v1
30-
with:
31-
java-version: 17
32-
- uses: burrunan/gradle-cache-action@v1
33-
name: Test
34-
with:
35-
job-id: jdk17
36-
multi-cache-enabled: false
37-
# An explicit skip for Sha512 tasks is required due to https://github.com/gradle/gradle/issues/16789
38-
arguments: --scan --no-parallel build -x distTar -x distTarSource -x distTarSha512 -x distTarSourceSha512
30+
- uses: actions/checkout@v2
31+
with:
32+
fetch-depth: 50
33+
- id: set-matrix
34+
run: |
35+
node .github/workflows/matrix.js
3936
40-
mac:
41-
name: 'macOS (JDK 17)'
42-
runs-on: macos-latest
37+
test:
38+
needs: matrix_prep
39+
name: '${{ matrix.name }}'
40+
runs-on: ${{ matrix.os }}
41+
env:
42+
TZ: ${{ matrix.tz }}
43+
strategy:
44+
matrix: ${{fromJson(needs.matrix_prep.outputs.matrix)}}
45+
fail-fast: false
46+
# max-parallel: 4
4347
steps:
44-
- uses: actions/checkout@v1.1.0
48+
- uses: actions/checkout@v2
4549
with:
4650
fetch-depth: 50
47-
- name: 'Set up JDK 17'
48-
uses: actions/setup-java@v1
51+
- name: Set up Java ${{ matrix.java_version }}, ${{ matrix.java_distribution }}
52+
uses: actions/setup-java@v2
4953
with:
50-
java-version: 17
54+
java-version: ${{ matrix.java_version }}
55+
distribution: ${{ matrix.java_distribution }}
56+
architecture: x64
5157
- uses: burrunan/gradle-cache-action@v1
5258
name: Test
5359
with:
54-
job-id: jdk14
60+
job-id: jdk${{ matrix.java_version }}
5561
multi-cache-enabled: false
56-
arguments: --scan --no-parallel build -x distTar -x distTarSource -x distTarSha512 -x distTarSourceSha512 -Dskip.test_TestDNSCacheManager.testWithCustomResolverAnd1Server=true
62+
# An explicit skip for Sha512 tasks is required due to https://github.com/gradle/gradle/issues/16789
63+
arguments: --scan --no-parallel build -x distTar -x distTarSource -x distTarSha512 -x distTarSourceSha512
64+
env:
65+
_JAVA_OPTIONS: ${{ matrix.testExtraJvmArgs }}
5766

5867
errorprone:
5968
name: 'Error Prone (JDK 11)'

‎.github/workflows/matrix.js‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
// The script generates a random subset of valid jdk, os, timezone, and other axes.
2+
// You can preview the results by running "node matrix.js"
3+
// See https://github.com/vlsi/github-actions-random-matrix
4+
let {MatrixBuilder} = require('./matrix_builder');
5+
const matrix = new MatrixBuilder();
6+
matrix.addAxis({
7+
name: 'java_distribution',
8+
values: [
9+
'zulu',
10+
'temurin',
11+
'liberica',
12+
'microsoft',
13+
]
14+
});
15+
16+
// TODO: support different JITs (see https://github.com/actions/setup-java/issues/279)
17+
matrix.addAxis({name: 'jit', title: '', values: ['hotspot']});
18+
19+
matrix.addAxis({
20+
name: 'java_version',
21+
// Strings allow versions like 18-ea
22+
values: [
23+
'8',
24+
'11',
25+
'17',
26+
]
27+
});
28+
29+
matrix.addAxis({
30+
name: 'tz',
31+
values: [
32+
'America/New_York',
33+
'Pacific/Chatham',
34+
'UTC'
35+
]
36+
});
37+
38+
matrix.addAxis({
39+
name: 'os',
40+
title: x => x.replace('-latest', ''),
41+
values: [
42+
// TODO: X11 is not available. Un-comment when https://github.com/burrunan/gradle-cache-action/issues/48 is resolved
43+
// 'ubuntu-latest',
44+
'windows-latest',
45+
'macos-latest'
46+
]
47+
});
48+
49+
// Test cases when Object#hashCode produces the same results
50+
// It allows capturing cases when the code uses hashCode as a unique identifier
51+
matrix.addAxis({
52+
name: 'hash',
53+
values: [
54+
{value: 'regular', title: '', weight: 42},
55+
{value: 'same', title: 'same hashcode', weight: 1}
56+
]
57+
});
58+
matrix.addAxis({
59+
name: 'locale',
60+
title: x => x.language + '_' + x.country,
61+
values: [
62+
{language: 'de', country: 'DE'},
63+
{language: 'fr', country: 'FR'},
64+
// TODO: fix :src:dist-check:batchBUG_62847
65+
// Fails with "ERROR o.a.j.u.JMeterUtils: Could not find resources for 'ru_EN'"
66+
// {language: 'ru', country: 'RU'},
67+
{language: 'tr', country: 'TR'},
68+
]
69+
});
70+
71+
matrix.setNamePattern(['java_version', 'java_distribution', 'hash', 'os', 'tz', 'locale']);
72+
73+
// Microsoft Java has no distribution for 8
74+
matrix.exclude({java_distribution: 'microsoft', java_version: 8});
75+
// Ensure at least one job with "same" hashcode exists
76+
matrix.generateRow({hash: {value: 'same'}});
77+
// Ensure at least one Windows and at least one Linux job is present (macOS is almost the same as Linux)
78+
matrix.generateRow({os: 'windows-latest'});
79+
// TODO: un-comment when xvfb will be possible
80+
// matrix.generateRow({os: 'ubuntu-latest'});
81+
// Ensure there will be at least one job with Java 8
82+
matrix.generateRow({java_version: 8});
83+
// Ensure there will be at least one job with Java 11
84+
matrix.generateRow({java_version: 11});
85+
// Ensure there will be at least one job with Java 17
86+
matrix.generateRow({java_version: 17});
87+
const include = matrix.generateRows(process.env.MATRIX_JOBS || 5);
88+
if (include.length === 0) {
89+
throw new Error('Matrix list is empty');
90+
}
91+
include.sort((a, b) => a.name.localeCompare(b.name, undefined, {numeric: true}));
92+
include.forEach(v => {
93+
let jvmArgs = [];
94+
if (v.hash.value === 'same') {
95+
jvmArgs.push('-XX:+UnlockExperimentalVMOptions', '-XX:hashCode=2');
96+
}
97+
// Gradle does not work in tr_TR locale, so pass locale to test only: https://github.com/gradle/gradle/issues/17361
98+
jvmArgs.push(`-Duser.country=${v.locale.country}`);
99+
jvmArgs.push(`-Duser.language=${v.locale.language}`);
100+
if (v.jit === 'hotspot' && Math.random() > 0.5) {
101+
// The following options randomize instruction selection in JIT compiler
102+
// so it might reveal missing synchronization in TestNG code
103+
v.name += ', stress JIT';
104+
jvmArgs.push('-XX:+UnlockDiagnosticVMOptions');
105+
if (v.java_version >= 8) {
106+
// Randomize instruction scheduling in GCM
107+
// share/opto/c2_globals.hpp
108+
jvmArgs.push('-XX:+StressGCM');
109+
// Randomize instruction scheduling in LCM
110+
// share/opto/c2_globals.hpp
111+
jvmArgs.push('-XX:+StressLCM');
112+
}
113+
if (v.java_version >= 16) {
114+
// Randomize worklist traversal in IGVN
115+
// share/opto/c2_globals.hpp
116+
jvmArgs.push('-XX:+StressIGVN');
117+
}
118+
if (v.java_version >= 17) {
119+
// Randomize worklist traversal in CCP
120+
// share/opto/c2_globals.hpp
121+
jvmArgs.push('-XX:+StressCCP');
122+
}
123+
}
124+
v.testExtraJvmArgs = jvmArgs.join(' ');
125+
delete v.hash;
126+
});
127+
128+
console.log(include);
129+
console.log('::set-output name=matrix::' + JSON.stringify({include}));

‎.github/workflows/matrix_builder.js‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
// License: Apache-2.0
2+
// Copyright Vladimir Sitnikov, 2021
3+
// See https://github.com/vlsi/github-actions-random-matrix
4+
5+
class Axis {
6+
constructor({name, title, values}) {
7+
this.name = name;
8+
this.title = title;
9+
this.values = values;
10+
// If all entries have same weight, the axis has uniform distribution
11+
this.uniform = values.reduce((a, b) => a === (b.weight || 1) ? a : 0, values[0].weight || 1) !== 0
12+
this.totalWeigth = this.uniform ? values.length : values.reduce((a, b) => a + (b.weight || 1), 0);
13+
}
14+
15+
static matches(row, filter) {
16+
if (typeof filter === 'function') {
17+
return filter(row);
18+
}
19+
if (Array.isArray(filter)) {
20+
// e.g. row={os: 'windows'}; filter=[{os: 'linux'}, {os: 'linux'}]
21+
return filter.find(v => Axis.matches(row, v));
22+
}
23+
if (typeof filter === 'object') {
24+
// e.g. row={jdk: {name: 'openjdk', version: 8}}; filter={jdk: {version: 8}}
25+
for (const [key, value] of Object.entries(filter)) {
26+
if (!row.hasOwnProperty(key) || !Axis.matches(row[key], value)) {
27+
return false;
28+
}
29+
}
30+
return true;
31+
}
32+
return row == filter;
33+
}
34+
35+
pickValue(filter) {
36+
let values = this.values;
37+
if (filter) {
38+
values = values.filter(v => Axis.matches(v, filter));
39+
}
40+
if (values.length == 0) {
41+
const filterStr = typeof filter === 'string' ? filter.toString() : JSON.stringify(filter);
42+
throw Error(`No values produces for axis '${this.name}' from ${JSON.stringify(this.values)}, filter=${filterStr}`);
43+
}
44+
if (values.length == 1) {
45+
return values[0];
46+
}
47+
if (this.uniform) {
48+
return values[Math.floor(Math.random() * values.length)];
49+
}
50+
const totalWeight = !filter ? this.totalWeigth : values.reduce((a, b) => a + (b.weight || 1), 0);
51+
let weight = Math.random() * totalWeight;
52+
for (let i = 0; i < values.length; i++) {
53+
const value = values[i];
54+
weight -= value.weight || 1;
55+
if (weight <= 0) {
56+
return value;
57+
}
58+
}
59+
return values[values.length - 1];
60+
}
61+
}
62+
63+
class MatrixBuilder {
64+
constructor() {
65+
this.axes = [];
66+
this.axisByName = {};
67+
this.rows = [];
68+
this.duplicates = {};
69+
this.excludes = [];
70+
this.includes = [];
71+
}
72+
73+
/**
74+
* Specifies include filter (all the generated rows would comply with all the include filters)
75+
* @param filter
76+
*/
77+
include(filter) {
78+
this.includes.push(filter);
79+
}
80+
81+
/**
82+
* Specifies exclude filter (e.g. exclude a forbidden combination)
83+
* @param filter
84+
*/
85+
exclude(filter) {
86+
this.excludes.push(filter);
87+
}
88+
89+
addAxis({name, title, values}) {
90+
const axis = new Axis({name, title, values});
91+
this.axes.push(axis);
92+
this.axisByName[name] = axis;
93+
return axis;
94+
}
95+
96+
setNamePattern(names) {
97+
this.namePattern = names;
98+
}
99+
100+
/**
101+
* Adds a row that matches the given filter to the resulting matrix.
102+
* filter values could be
103+
* - literal values: filter={os: 'windows-latest'}
104+
* - arrays: filter={os: ['windows-latest', 'linux-latest']}
105+
* - functions: filter={os: x => x!='windows-latest'}
106+
* @param filter object with keys matching axes names
107+
* @returns {*}
108+
*/
109+
generateRow(filter) {
110+
let res;
111+
if (filter) {
112+
// If matching row already exists, no need to generate more
113+
res = this.rows.find(v => Axis.matches(v, filter));
114+
if (res) {
115+
return res;
116+
}
117+
}
118+
for (let i = 0; i < 142; i++) {
119+
res = this.axes.reduce(
120+
(prev, next) =>
121+
Object.assign(prev, {
122+
[next.name]: next.pickValue(filter ? filter[next.name] : undefined)
123+
}),
124+
{}
125+
);
126+
if (this.excludes.length > 0 && this.excludes.find(f => Axis.matches(res, f)) ||
127+
this.includes.length > 0 && !this.includes.find(f => Axis.matches(res, f))) {
128+
continue;
129+
}
130+
const key = JSON.stringify(res);
131+
if (!this.duplicates.hasOwnProperty(key)) {
132+
this.duplicates[key] = true;
133+
res.name =
134+
this.namePattern.map(axisName => {
135+
let value = res[axisName];
136+
const title = value.title;
137+
if (typeof title != 'undefined') {
138+
return title;
139+
}
140+
const computeTitle = this.axisByName[axisName].title;
141+
return computeTitle ? computeTitle(value) : value;
142+
}).filter(Boolean).join(", ");
143+
this.rows.push(res);
144+
return res;
145+
}
146+
}
147+
throw Error(`Unable to generate row. Please check include and exclude filters`);
148+
}
149+
150+
generateRows(maxRows, filter) {
151+
for (let i = 0; this.rows.length < maxRows && i < maxRows; i++) {
152+
this.generateRow(filter);
153+
}
154+
return this.rows;
155+
}
156+
}
157+
158+
module.exports = {Axis, MatrixBuilder};

‎xdocs/changes.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,7 @@ Summary
102102
<ch_section>Non-functional changes</ch_section>
103103
<ul>
104104
<li><pr>5713</pr>update spockAdded randomized test GitHub Actions matrix for better coverage of locales and time zones</li>
105+
<li>Added randomized test GitHub Actions matrix for better coverage of locales and time zones</li>
105106
</ul>
106107

107108
<!-- =================== Bug fixes =================== -->

0 commit comments

Comments
 (0)