ci: add test matrix randomization

.github/workflows/main.yml

@@ -18,35 +18,52 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  matrix_prep:
+    name: Matrix Preparation
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    env:
+      # Ask matrix.js to produce 7 jobs
+      MATRIX_JOBS: 7
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 50
+      - id: set-matrix
+        run: |
+          node .github/workflows/matrix.js
+
   Test:
-    name: JDK ${{ matrix.jdk }}, ${{ matrix.os }}
+    needs: matrix_prep
+    name: '${{ matrix.name }}'
     runs-on: ${{ matrix.os }}
+    env:
+      TZ: ${{ matrix.tz }}
     strategy:
-      matrix:
-        jdk: [8, 11, 17]
-        os: [ubuntu-latest, windows-latest, macos-latest]
-      fail-fast: true
+      matrix: ${{fromJson(needs.matrix_prep.outputs.matrix)}}
+      fail-fast: false
       max-parallel: 4
     steps:
     - uses: actions/checkout@v2
       with:
         fetch-depth: 50
-    - name: Set up Java 8
+    - name: Set up Java 8, ${{ matrix.compile_java_distribution }}
       id: setup-java-8
-      uses: actions/setup-java@v1
+      uses: actions/setup-java@v2
       with:
         java-version: 8
-    - name: Set up Java ${{ matrix.jdk }}
-      if: ${{ matrix.jdk != '8' }}
-      uses: actions/setup-java@v1
+        distribution: ${{ matrix.compile_java_distribution }}
+        architecture: x64
+    - name: Set up Java ${{ matrix.java_version }}, ${{ matrix.java_distribution }}
+      if: ${{ matrix.java_version != '8' }}
+      uses: actions/setup-java@v2
       with:
-        java-version: ${{ matrix.jdk }}
-    - name: Compile
-      # download dependencies, etc, so test log looks better
-      run: mvn -B --toolchains .github/workflows/toolchains.xml compile
-      env:
-        JAVA_HOME_8: ${{ steps.setup-java-8.outputs.path }}
+        java-version: ${{ matrix.java_version }}
+        distribution: ${{ matrix.java_distribution }}
+        architecture: x64
     - name: Test
-      run: mvn -B --toolchains .github/workflows/toolchains.xml verify
+      run: mvn -B --no-transfer-progress --toolchains .github/workflows/toolchains.xml verify
       env:
+        _JAVA_OPTIONS: ${{ matrix.testExtraJvmArgs }}
         JAVA_HOME_8: ${{ steps.setup-java-8.outputs.path }}

.github/workflows/matrix.js

@@ -0,0 +1,137 @@
+// The script generates a random subset of valid jdk, os, timezone, and other axes.
+// You can preview the results by running "node matrix.js"
+// See https://github.com/vlsi/github-actions-random-matrix
+let {MatrixBuilder} = require('./matrix_builder');
+const matrix = new MatrixBuilder();
+matrix.addAxis({
+  name: 'java_distribution',
+  values: [
+    'zulu',
+    'temurin',
+    'liberica',
+    'microsoft',
+  ]
+});
+
+// Java 8 JDK (== the one for compile the project)
+matrix.addAxis({
+  name: 'compile_java_distribution',
+  values: [
+    'zulu',
+    'temurin',
+    'liberica',
+  ]
+});
+
+// TODO: support different JITs (see https://github.com/actions/setup-java/issues/279)
+matrix.addAxis({name: 'jit', title: '', values: ['hotspot']});
+
+matrix.addAxis({
+  name: 'java_version',
+  // Strings allow versions like 18-ea
+  values: [
+    '8',
+    '11',
+    '17',
+  ]
+});
+
+matrix.addAxis({
+  name: 'tz',
+  values: [
+    'America/New_York',
+    'Pacific/Chatham',
+    'UTC'
+  ]
+});
+
+matrix.addAxis({
+  name: 'os',
+  title: x => x.replace('-latest', ''),
+  values: [
+    'ubuntu-latest',
+    'windows-latest',
+    'macos-latest'
+  ]
+});
+
+// Test cases when Object#hashCode produces the same results
+// It allows capturing cases when the code uses hashCode as a unique identifier
+matrix.addAxis({
+  name: 'hash',
+  values: [
+    {value: 'regular', title: '', weight: 42},
+    {value: 'same', title: 'same hashcode', weight: 1}
+  ]
+});
+matrix.addAxis({
+  name: 'locale',
+  title: x => x.language + '_' + x.country,
+  values: [
+    {language: 'de', country: 'DE'},
+    {language: 'fr', country: 'FR'},
+    {language: 'ru', country: 'RU'},
+    {language: 'tr', country: 'TR'},
+  ]
+});
+
+matrix.setNamePattern(['java_version', 'java_distribution', 'hash', 'os', 'tz', 'locale', 'compile_java_distribution']);
+
+// Microsoft Java has no distribution for 8
+matrix.exclude({java_distribution: 'microsoft', java_version: 8});
+// Ensure at least one job with "same" hashcode exists
+matrix.generateRow({hash: {value: 'same'}});
+// Ensure at least one Windows and at least one Linux job is present (macOS is almost the same as Linux)
+matrix.generateRow({os: 'windows-latest'});
+matrix.generateRow({os: 'ubuntu-latest'});
+// Ensure there will be at least one job with Java 8
+matrix.generateRow({java_version: 8});
+// Ensure there will be at least one job with Java 11
+matrix.generateRow({java_version: 11});
+// Ensure there will be at least one job with Java 17
+matrix.generateRow({java_version: 17});
+const include = matrix.generateRows(process.env.MATRIX_JOBS || 5);
+if (include.length === 0) {
+  throw new Error('Matrix list is empty');
+}
+include.sort((a, b) => a.name.localeCompare(b.name, undefined, {numeric: true}));
+include.forEach(v => {
+  let jvmArgs = [];
+  if (v.hash.value === 'same') {
+    jvmArgs.push('-XX:+UnlockExperimentalVMOptions', '-XX:hashCode=2');
+  }
+  // Gradle does not work in tr_TR locale, so pass locale to test only: https://github.com/gradle/gradle/issues/17361
+  jvmArgs.push(`-Duser.country=${v.locale.country}`);
+  jvmArgs.push(`-Duser.language=${v.locale.language}`);
+  if (v.jit === 'hotspot' && Math.random() > 0.5) {
+    // The following options randomize instruction selection in JIT compiler
+    // so it might reveal missing synchronization in TestNG code
+    v.name += ', stress JIT';
+    jvmArgs.push('-XX:+UnlockDiagnosticVMOptions');
+    if (v.java_version >= 8) {
+      // Randomize instruction scheduling in GCM
+      // share/opto/c2_globals.hpp
+      jvmArgs.push('-XX:+StressGCM');
+      // Randomize instruction scheduling in LCM
+      // share/opto/c2_globals.hpp
+      jvmArgs.push('-XX:+StressLCM');
+    }
+    // TODO: support extra Stress options when we are able to distinguish options ofr Java 8 (compilation) and Java 11+ (execution)
+    /*
+    if (v.java_version >= 16) {
+      // Randomize worklist traversal in IGVN
+      // share/opto/c2_globals.hpp
+      jvmArgs.push('-XX:+StressIGVN');
+    }
+    if (v.java_version >= 17) {
+      // Randomize worklist traversal in CCP
+      // share/opto/c2_globals.hpp
+      jvmArgs.push('-XX:+StressCCP');
+    }*/
+  }
+  v.testExtraJvmArgs = jvmArgs.join(' ');
+  delete v.hash;
+});
+
+console.log(include);
+console.log('::set-output name=matrix::' + JSON.stringify({include}));

.github/workflows/matrix_builder.js

@@ -0,0 +1,158 @@
+// License: Apache-2.0
+// Copyright Vladimir Sitnikov, 2021
+// See https://github.com/vlsi/github-actions-random-matrix
+
+class Axis {
+  constructor({name, title, values}) {
+    this.name = name;
+    this.title = title;
+    this.values = values;
+    // If all entries have same weight, the axis has uniform distribution
+    this.uniform = values.reduce((a, b) => a === (b.weight || 1) ? a : 0, values[0].weight || 1) !== 0
+    this.totalWeigth = this.uniform ? values.length : values.reduce((a, b) => a + (b.weight || 1), 0);
+  }
+
+  static matches(row, filter) {
+    if (typeof filter === 'function') {
+      return filter(row);
+    }
+    if (Array.isArray(filter)) {
+      // e.g. row={os: 'windows'}; filter=[{os: 'linux'}, {os: 'linux'}]
+      return filter.find(v => Axis.matches(row, v));
+    }
+    if (typeof filter === 'object') {
+      // e.g. row={jdk: {name: 'openjdk', version: 8}}; filter={jdk: {version: 8}}
+      for (const [key, value] of Object.entries(filter)) {
+        if (!row.hasOwnProperty(key) || !Axis.matches(row[key], value)) {
+          return false;
+        }
+      }
+      return true;
+    }
+    return row == filter;
+  }
+
+  pickValue(filter) {
+    let values = this.values;
+    if (filter) {
+      values = values.filter(v => Axis.matches(v, filter));
+    }
+    if (values.length == 0) {
+      const filterStr = typeof filter === 'string' ? filter.toString() : JSON.stringify(filter);
+      throw Error(`No values produces for axis '${this.name}' from ${JSON.stringify(this.values)}, filter=${filterStr}`);
+    }
+    if (values.length == 1) {
+      return values[0];
+    }
+    if (this.uniform) {
+      return values[Math.floor(Math.random() * values.length)];
+    }
+    const totalWeight = !filter ? this.totalWeigth : values.reduce((a, b) => a + (b.weight || 1), 0);
+    let weight = Math.random() * totalWeight;
+    for (let i = 0; i < values.length; i++) {
+      const value = values[i];
+      weight -= value.weight || 1;
+      if (weight <= 0) {
+        return value;
+      }
+    }
+    return values[values.length - 1];
+  }
+}
+
+class MatrixBuilder {
+  constructor() {
+    this.axes = [];
+    this.axisByName = {};
+    this.rows = [];
+    this.duplicates = {};
+    this.excludes = [];
+    this.includes = [];
+  }
+
+  /**
+   * Specifies include filter (all the generated rows would comply with all the include filters)
+   * @param filter
+   */
+  include(filter) {
+    this.includes.push(filter);
+  }
+
+  /**
+   * Specifies exclude filter (e.g. exclude a forbidden combination)
+   * @param filter
+   */
+  exclude(filter) {
+    this.excludes.push(filter);
+  }
+
+  addAxis({name, title, values}) {
+    const axis = new Axis({name, title, values});
+    this.axes.push(axis);
+    this.axisByName[name] = axis;
+    return axis;
+  }
+
+  setNamePattern(names) {
+    this.namePattern = names;
+  }
+
+  /**
+   * Adds a row that matches the given filter to the resulting matrix.
+   * filter values could be
+   *  - literal values: filter={os: 'windows-latest'}
+   *  - arrays: filter={os: ['windows-latest', 'linux-latest']}
+   *  - functions: filter={os: x => x!='windows-latest'}
+   * @param filter object with keys matching axes names
+   * @returns {*}
+   */
+  generateRow(filter) {
+    let res;
+    if (filter) {
+      // If matching row already exists, no need to generate more
+      res = this.rows.find(v => Axis.matches(v, filter));
+      if (res) {
+        return res;
+      }
+    }
+    for (let i = 0; i < 142; i++) {
+      res = this.axes.reduce(
+        (prev, next) =>
+          Object.assign(prev, {
+            [next.name]: next.pickValue(filter ? filter[next.name] : undefined)
+          }),
+        {}
+      );
+      if (this.excludes.length > 0 && this.excludes.find(f => Axis.matches(res, f)) ||
+        this.includes.length > 0 && !this.includes.find(f => Axis.matches(res, f))) {
+        continue;
+      }
+      const key = JSON.stringify(res);
+      if (!this.duplicates.hasOwnProperty(key)) {
+        this.duplicates[key] = true;
+        res.name =
+          this.namePattern.map(axisName => {
+            let value = res[axisName];
+            const title = value.title;
+            if (typeof title != 'undefined') {
+              return title;
+            }
+            const computeTitle = this.axisByName[axisName].title;
+            return computeTitle ? computeTitle(value) : value;
+          }).filter(Boolean).join(", ");
+        this.rows.push(res);
+        return res;
+      }
+    }
+    throw Error(`Unable to generate row. Please check include and exclude filters`);
+  }
+
+  generateRows(maxRows, filter) {
+    for (let i = 0; this.rows.length < maxRows && i < maxRows; i++) {
+      this.generateRow(filter);
+    }
+    return this.rows;
+  }
+}
+
+module.exports = {Axis, MatrixBuilder};