Skip to content

Commit 4ff9462

Browse files
matanlureymatanlurey
authored
Fix local testing, gradle XML errors, and enable on CI. (flutter#152383)
TIL you cannot have XML comments before the initial `<?xml` declaration. Follow-up to flutter#152326.
1 parent 7f4f1a0 commit 4ff9462

File tree

11 files changed

+159
-61
lines changed

11 files changed

+159
-61
lines changed

‎.ci.yaml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1297,11 +1297,15 @@ targets:
12971297
- name: Linux_android_emu flutter_driver_android_test
12981298
recipe: flutter/flutter_drone
12991299
timeout: 60
1300-
bringup: true
1300+
bringup: false
13011301
properties:
13021302
shard: flutter_driver_android
13031303
tags: >
13041304
["framework", "hostonly", "shard", "linux"]
1305+
dependencies: >-
1306+
[
1307+
{"dependency": "goldctl", "version": "git_revision:720a542f6fe4f92922c3b8f0fdcc4d2ac6bb83cd"}
1308+
]
13051309
13061310
- name: Linux realm_checker
13071311
recipe: flutter/flutter_drone

‎dev/bots/suite_runners/run_flutter_driver_android_tests.dart‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,18 +2,50 @@
22
// Use of this source code is governed by a BSD-style license that can be
33
// found in the LICENSE file.
44

5+
import 'dart:convert';
6+
import 'dart:io' as io;
7+
58
import 'package:path/path.dart' as path;
69
import '../run_command.dart';
710
import '../utils.dart';
811

12+
/// To run this test locally:
13+
///
14+
/// 1. Connect an Android device or emulator.
15+
/// 2. Run the following command from the root of the Flutter repository:
16+
///
17+
/// ```sh
18+
/// SHARD=flutter_driver_android bin/cache/dart-sdk/bin/dart dev/bots/test.dart
19+
/// ```
20+
///
21+
/// For debugging, it is recommended to instead just run and launch these tests
22+
/// individually _in_ the `dev/integration_tests/android_driver_test` directory.
923
Future<void> runFlutterDriverAndroidTests() async {
1024
print('Running Flutter Driver Android tests...');
1125

26+
// Print out the results of `adb devices`, for uh, science:
27+
print('Listing devices...');
28+
final io.ProcessResult devices = await _adb(
29+
<String>[
30+
'devices',
31+
],
32+
);
33+
print(devices.stdout);
34+
print(devices.stderr);
35+
36+
// We need to configure the emulator to disable confirmations before the
37+
// application starts. Some of these configuration options won't work once
38+
// the application is running.
39+
print('Configuring device...');
40+
await _configureForScreenshotTesting();
41+
1242
// TODO(matanlurey): Should we be using another instrumentation method?
1343
await runCommand(
1444
'flutter',
1545
<String>[
1646
'drive',
47+
'--test-arguments=test',
48+
'--test-arguments=--reporter=expanded',
1749
],
1850
workingDirectory: path.join(
1951
'dev',
@@ -22,3 +54,60 @@ Future<void> runFlutterDriverAndroidTests() async {
2254
),
2355
);
2456
}
57+
58+
// TODO(matanlurey): Move this code into flutter_driver instead of here.
59+
Future<void> _configureForScreenshotTesting() async {
60+
// Disable confirmation for immersive mode.
61+
final io.ProcessResult immersive = await _adb(
62+
<String>[
63+
'shell',
64+
'settings',
65+
'put',
66+
'secure',
67+
'immersive_mode_confirmations',
68+
'confirmed',
69+
],
70+
);
71+
72+
if (immersive.exitCode != 0) {
73+
throw StateError('Failed to configure device: ${immersive.stderr}');
74+
}
75+
76+
const Map<String, String> settings = <String, String>{
77+
'show_surface_updates': '1',
78+
'transition_animation_scale': '0',
79+
'window_animation_scale': '0',
80+
'animator_duration_scale': '0',
81+
};
82+
83+
for (final MapEntry<String, String> entry in settings.entries) {
84+
final io.ProcessResult result = await _adb(
85+
<String>[
86+
'shell',
87+
'settings',
88+
'put',
89+
'global',
90+
entry.key,
91+
entry.value,
92+
],
93+
);
94+
95+
if (result.exitCode != 0) {
96+
throw StateError('Failed to configure device: ${result.stderr}');
97+
}
98+
}
99+
}
100+
101+
Future<io.ProcessResult> _adb(
102+
List<String> args, {
103+
Encoding? stdoutEncoding = io.systemEncoding,
104+
}) {
105+
// TODO(matanlurey): Ideally we should specify the device target here.
106+
return io.Process.run(
107+
'adb',
108+
<String>[
109+
...args,
110+
],
111+
stdoutEncoding: stdoutEncoding,
112+
);
113+
}

‎dev/integration_tests/android_driver_test/android/app/src/main/res/drawable-v21/launch_background.xml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,8 @@
1+
<?xml version="1.0" encoding="utf-8"?>
12
<!-- Copyright 2014 The Flutter Authors. All rights reserved.
23
Use of this source code is governed by a BSD-style license that can be
34
found in the LICENSE file. -->
45

5-
<?xml version="1.0" encoding="utf-8"?>
6-
<!-- Modify this file to customize your launch splash screen -->
76
<layer-list xmlns:android="http://schemas.android.com/apk/res/android">
87
<item android:drawable="?android:colorBackground" />
98

‎dev/integration_tests/android_driver_test/android/app/src/main/res/drawable/launch_background.xml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,8 @@
1+
<?xml version="1.0" encoding="utf-8"?>
12
<!-- Copyright 2014 The Flutter Authors. All rights reserved.
23
Use of this source code is governed by a BSD-style license that can be
34
found in the LICENSE file. -->
45

5-
<?xml version="1.0" encoding="utf-8"?>
6-
<!-- Modify this file to customize your launch splash screen -->
76
<layer-list xmlns:android="http://schemas.android.com/apk/res/android">
87
<item android:drawable="@android:color/white" />
98

‎dev/integration_tests/android_driver_test/android/app/src/main/res/values-night/styles.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1+
<?xml version="1.0" encoding="utf-8"?>
12
<!-- Copyright 2014 The Flutter Authors. All rights reserved.
23
Use of this source code is governed by a BSD-style license that can be
34
found in the LICENSE file. -->
45

5-
<?xml version="1.0" encoding="utf-8"?>
66
<resources>
77
<!-- Theme applied to the Android Window while the process is starting when the OS's Dark Mode setting is on -->
88
<style name="LaunchTheme" parent="@android:style/Theme.Black.NoTitleBar">

‎dev/integration_tests/android_driver_test/android/app/src/main/res/values/styles.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1+
<?xml version="1.0" encoding="utf-8"?>
12
<!-- Copyright 2014 The Flutter Authors. All rights reserved.
23
Use of this source code is governed by a BSD-style license that can be
34
found in the LICENSE file. -->
45

5-
<?xml version="1.0" encoding="utf-8"?>
66
<resources>
77
<!-- Theme applied to the Android Window while the process is starting when the OS's Dark Mode setting is off -->
88
<style name="LaunchTheme" parent="@android:style/Theme.Light.NoTitleBar">

‎dev/integration_tests/android_driver_test/test_driver/_flutter_goldens_fork.dart‎

Lines changed: 54 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ import 'package:platform/platform.dart';
1919
import 'package:process/process.dart';
2020

2121
const LocalFileSystem _localFs = LocalFileSystem();
22+
const String _kGoldctlKey = 'GOLDCTL';
23+
const String _kGoldctlPresubmitKey = 'GOLD_TRYJOB';
2224

2325
// TODO(matanlurey): Refactor flutter_goldens to just re-use that code instead.
2426
Future<void> testExecutable(
@@ -31,19 +33,41 @@ Future<void> testExecutable(
3133
'where the "goldenFileComparator" has not yet been set. This is to ensure '
3234
'that the correct comparator is used for the current test environment.',
3335
);
34-
final io.Directory tmpDir = io.Directory.systemTemp.createTempSync('android_driver_test');
36+
if (!io.Platform.environment.containsKey(_kGoldctlKey)) {
37+
io.stderr.writeln(
38+
'Environment variable $_kGoldctlKey is not set. Assuming this is a local '
39+
'test run and will not upload results to Skia Gold.',
40+
);
41+
return testMain();
42+
}
43+
final io.Directory tmpDir = io.Directory.systemTemp.createTempSync(
44+
'android_driver_test',
45+
);
46+
final bool isPresubmit = io.Platform.environment.containsKey(
47+
_kGoldctlPresubmitKey,
48+
);
49+
io.stderr.writeln(
50+
'=== Using Skia Gold ===\n'
51+
'Environment variable $_kGoldctlKey is set, using Skia Gold: \n'
52+
' - tmpDir: ${tmpDir.path}\n'
53+
' - namePrefix: $namePrefix\n'
54+
' - isPresubmit: $isPresubmit\n',
55+
);
56+
final SkiaGoldClient skiaGoldClient = SkiaGoldClient(
57+
_localFs.directory(tmpDir.path),
58+
fs: _localFs,
59+
process: const LocalProcessManager(),
60+
platform: const LocalPlatform(),
61+
httpClient: io.HttpClient(),
62+
log: io.stderr.writeln,
63+
);
64+
await skiaGoldClient.auth();
3565
goldenFileComparator = _GoldenFileComparator(
36-
SkiaGoldClient(
37-
_localFs.directory(tmpDir.path),
38-
fs: _localFs,
39-
process: const LocalProcessManager(),
40-
platform: const LocalPlatform(),
41-
httpClient: io.HttpClient(),
42-
log: io.stderr.writeln,
43-
),
66+
skiaGoldClient,
4467
namePrefix: namePrefix,
45-
isPresubmit: false,
68+
isPresubmit: isPresubmit,
4669
);
70+
return testMain();
4771
}
4872

4973
final class _GoldenFileComparator extends GoldenFileComparator {
@@ -68,22 +92,37 @@ final class _GoldenFileComparator extends GoldenFileComparator {
6892
}
6993

7094
golden = _addPrefix(golden);
71-
await update(golden, imageBytes);
72-
73-
final io.File goldenFile = _getGoldenFile(golden);
95+
final io.File goldenFile = await update(golden, imageBytes);
7496
if (isPresubmit) {
75-
await skiaClient.tryjobAdd(golden.path, _localFs.file(goldenFile.path));
97+
final String? result = await skiaClient.tryjobAdd(
98+
golden.path,
99+
_localFs.file(goldenFile.path),
100+
);
101+
if (result != null) {
102+
io.stderr.writeln(
103+
'Skia Gold detected an error when comparing "$golden":\n\n$result',
104+
);
105+
io.stderr.writeln('Still succeeding, will be triaged in Flutter Gold');
106+
} else {
107+
io.stderr.writeln(
108+
'Skia Gold comparison succeeded comparing "$golden".',
109+
);
110+
}
76111
return true;
77112
} else {
78113
return skiaClient.imgtestAdd(golden.path, _localFs.file(goldenFile.path));
79114
}
80115
}
81116

82117
@override
83-
Future<void> update(Uri golden, Uint8List imageBytes) async {
118+
Future<io.File> update(Uri golden, Uint8List imageBytes) async {
119+
io.stderr.writeln(
120+
'Updating golden file: $golden (${imageBytes.length} bytes)...',
121+
);
84122
final io.File goldenFile = _getGoldenFile(golden);
85123
await goldenFile.parent.create(recursive: true);
86124
await goldenFile.writeAsBytes(imageBytes, flush: true);
125+
return goldenFile;
87126
}
88127

89128
io.File _getGoldenFile(Uri uri) {

‎packages/flutter_driver/lib/src/native/android.dart‎

Lines changed: 0 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -84,33 +84,6 @@ final class AndroidNativeDriver implements NativeDriver {
8484
await _tmpDir.delete(recursive: true);
8585
}
8686

87-
@override
88-
Future<void> configureForScreenshotTesting() async {
89-
const Map<String, String> settings = <String, String>{
90-
'show_surface_updates': '1',
91-
'transition_animation_scale': '0',
92-
'window_animation_scale': '0',
93-
'animator_duration_scale': '0',
94-
};
95-
96-
for (final MapEntry<String, String> entry in settings.entries) {
97-
final io.ProcessResult result = await _adb(
98-
<String>[
99-
'shell',
100-
'settings',
101-
'put',
102-
'global',
103-
entry.key,
104-
entry.value,
105-
],
106-
);
107-
108-
if (result.exitCode != 0) {
109-
throw StateError('Failed to configure device: ${result.stderr}');
110-
}
111-
}
112-
}
113-
11487
@override
11588
Future<NativeScreenshot> screenshot() async {
11689
// Similar pause to the one in `<FlutterDriver>.screenshot()`.

‎packages/flutter_driver/lib/src/native/driver.dart‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -26,16 +26,6 @@ abstract interface class NativeDriver {
2626
/// After calling this method, the driver is no longer usable.
2727
Future<void> close();
2828

29-
/// Configures the device for screenshot testing.
30-
///
31-
/// Where possible, this method should suppress system UI elements that are
32-
/// not part of the application under test, such as the status bar or
33-
/// navigation bar, and disable animations that might interfere with
34-
/// screenshot comparison.
35-
///
36-
/// The exact details of what is configured are platform-specific.
37-
Future<void> configureForScreenshotTesting();
38-
3929
/// Take a screenshot using a platform-specific mechanism.
4030
///
4131
/// The image is returned as an opaque handle that can be used to retrieve

‎packages/flutter_goldens/lib/skia_client.dart‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -331,7 +331,11 @@ class SkiaGoldClient {
331331
///
332332
/// The [testName] and [goldenFile] parameters reference the current
333333
/// comparison being evaluated by the [FlutterPreSubmitFileComparator].
334-
Future<void> tryjobAdd(String testName, File goldenFile) async {
334+
///
335+
/// If the tryjob fails due to pixel differences, the method will succeed
336+
/// as the failure will be triaged in the 'Flutter Gold' dashboard, and the
337+
/// `stdout` will contain the failure message; otherwise will return `null`.
338+
Future<String?> tryjobAdd(String testName, File goldenFile) async {
335339
final List<String> imgtestCommand = <String>[
336340
_goldctl,
337341
'imgtest', 'add',
@@ -368,6 +372,7 @@ class SkiaGoldClient {
368372
..writeln('result-state.json: ${resultContents ?? 'No result file found.'}');
369373
throw SkiaException(buf.toString());
370374
}
375+
return result.exitCode == 0 ? null : resultStdout;
371376
}
372377

373378
// Constructs arguments for `goldctl` for controlling how pixels are compared.

0 commit comments

Comments
 (0)