Skip to content
This repository was archived by the owner on May 28, 2025. It is now read-only.

Commit b2728d5

Browse files
borsbors
committed
Auto merge of rust-lang#135674 - scottmcm:assume-better, r=estebank
Update our range `assume`s to the format that LLVM prefers I found out in llvm/llvm-project#123278 (comment) that the way I started emitting the `assume`s in rust-lang#109993 was suboptimal, and as seen in that LLVM issue the way we're doing it -- with two `assume`s sometimes -- can at times lead to CVP/SCCP not realize what's happening because one of them turns into a `ne` instead of conveying a range. So this updates how it's emitted from ``` assume( x >= LOW ); assume( x <= HIGH ); ``` or ``` // (for ranges that wrap the range) assume( (x <= LOW) | (x >= HIGH) ); ``` to ``` assume( (x - LOW) <= (HIGH - LOW) ); ``` so that we don't need multiple `icmp`s nor multiple `assume`s for a single value, and both wrappping and non-wrapping ranges emit the same shape. (And we don't bother emitting the subtraction if `LOW` is zero, since that's trivial for us to check too.)
2 parents c234b83 + 6fe8200 commit b2728d5

File tree

4 files changed

+114
-84
lines changed

4 files changed

+114
-84
lines changed

‎compiler/rustc_codegen_ssa/src/mir/rvalue.rs‎

Lines changed: 18 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -387,10 +387,15 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
387387
use abi::Primitive::*;
388388
imm = bx.from_immediate(imm);
389389

390-
// When scalars are passed by value, there's no metadata recording their
391-
// valid ranges. For example, `char`s are passed as just `i32`, with no
392-
// way for LLVM to know that they're 0x10FFFF at most. Thus we assume
393-
// the range of the input value too, not just the output range.
390+
// If we have a scalar, we must already know its range. Either
391+
//
392+
// 1) It's a parameter with `range` parameter metadata,
393+
// 2) It's something we `load`ed with `!range` metadata, or
394+
// 3) After a transmute we `assume`d the range (see below).
395+
//
396+
// That said, last time we tried removing this, it didn't actually help
397+
// the rustc-perf results, so might as well keep doing it
398+
// <https://github.com/rust-lang/rust/pull/135610#issuecomment-2599275182>
394399
self.assume_scalar_range(bx, imm, from_scalar, from_backend_ty);
395400

396401
imm = match (from_scalar.primitive(), to_scalar.primitive()) {
@@ -411,7 +416,14 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
411416
bx.bitcast(int_imm, to_backend_ty)
412417
}
413418
};
419+
420+
// This `assume` remains important for cases like (a conceptual)
421+
// transmute::<u32, NonZeroU32>(x) == 0
422+
// since it's never passed to something with parameter metadata (especially
423+
// after MIR inlining) so the only way to tell the backend about the
424+
// constraint that the `transmute` introduced is to `assume` it.
414425
self.assume_scalar_range(bx, imm, to_scalar, to_backend_ty);
426+
415427
imm = bx.to_immediate_scalar(imm, to_scalar);
416428
imm
417429
}
@@ -433,31 +445,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
433445
return;
434446
}
435447

436-
let abi::WrappingRange { start, end } = scalar.valid_range(self.cx);
437-
438-
if start <= end {
439-
if start > 0 {
440-
let low = bx.const_uint_big(backend_ty, start);
441-
let cmp = bx.icmp(IntPredicate::IntUGE, imm, low);
442-
bx.assume(cmp);
443-
}
444-
445-
let type_max = scalar.size(self.cx).unsigned_int_max();
446-
if end < type_max {
447-
let high = bx.const_uint_big(backend_ty, end);
448-
let cmp = bx.icmp(IntPredicate::IntULE, imm, high);
449-
bx.assume(cmp);
450-
}
451-
} else {
452-
let low = bx.const_uint_big(backend_ty, start);
453-
let cmp_low = bx.icmp(IntPredicate::IntUGE, imm, low);
454-
455-
let high = bx.const_uint_big(backend_ty, end);
456-
let cmp_high = bx.icmp(IntPredicate::IntULE, imm, high);
457-
458-
let or = bx.or(cmp_low, cmp_high);
459-
bx.assume(or);
460-
}
448+
let range = scalar.valid_range(self.cx);
449+
bx.assume_integer_range(imm, backend_ty, range);
461450
}
462451

463452
pub(crate) fn codegen_rvalue_unsized(

‎compiler/rustc_codegen_ssa/src/traits/builder.rs‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,27 @@ pub trait BuilderMethods<'a, 'tcx>:
217217
dest: PlaceRef<'tcx, Self::Value>,
218218
);
219219

220+
/// Emits an `assume` that the integer value `imm` of type `ty` is contained in `range`.
221+
///
222+
/// This *always* emits the assumption, so you probably want to check the
223+
/// optimization level and `Scalar::is_always_valid` before calling it.
224+
fn assume_integer_range(&mut self, imm: Self::Value, ty: Self::Type, range: WrappingRange) {
225+
let WrappingRange { start, end } = range;
226+
227+
// Perhaps one day we'll be able to use assume operand bundles for this,
228+
// but for now this encoding with a single icmp+assume is best per
229+
// <https://github.com/llvm/llvm-project/issues/123278#issuecomment-2597440158>
230+
let shifted = if start == 0 {
231+
imm
232+
} else {
233+
let low = self.const_uint_big(ty, start);
234+
self.sub(imm, low)
235+
};
236+
let width = self.const_uint_big(ty, u128::wrapping_sub(end, start));
237+
let cmp = self.icmp(IntPredicate::IntULE, shifted, width);
238+
self.assume(cmp);
239+
}
240+
220241
fn range_metadata(&mut self, load: Self::Value, range: WrappingRange);
221242
fn nonnull_metadata(&mut self, load: Self::Value);
222243

‎tests/codegen/intrinsics/transmute-niched.rs‎

Lines changed: 67 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,13 @@ pub enum SmallEnum {
1717
// CHECK-LABEL: @check_to_enum(
1818
#[no_mangle]
1919
pub unsafe fn check_to_enum(x: i8) -> SmallEnum {
20-
// OPT: %0 = icmp uge i8 %x, 10
21-
// OPT: call void @llvm.assume(i1 %0)
22-
// OPT: %1 = icmp ule i8 %x, 12
20+
// CHECK-NOT: icmp
21+
// CHECK-NOT: assume
22+
// OPT: %0 = sub i8 %x, 10
23+
// OPT: %1 = icmp ule i8 %0, 2
2324
// OPT: call void @llvm.assume(i1 %1)
24-
// DBG-NOT: icmp
25-
// DBG-NOT: assume
25+
// CHECK-NOT: icmp
26+
// CHECK-NOT: assume
2627
// CHECK: ret i8 %x
2728

2829
transmute(x)
@@ -31,12 +32,13 @@ pub unsafe fn check_to_enum(x: i8) -> SmallEnum {
3132
// CHECK-LABEL: @check_from_enum(
3233
#[no_mangle]
3334
pub unsafe fn check_from_enum(x: SmallEnum) -> i8 {
34-
// OPT: %0 = icmp uge i8 %x, 10
35-
// OPT: call void @llvm.assume(i1 %0)
36-
// OPT: %1 = icmp ule i8 %x, 12
35+
// CHECK-NOT: icmp
36+
// CHECK-NOT: assume
37+
// OPT: %0 = sub i8 %x, 10
38+
// OPT: %1 = icmp ule i8 %0, 2
3739
// OPT: call void @llvm.assume(i1 %1)
38-
// DBG-NOT: icmp
39-
// DBG-NOT: assume
40+
// CHECK-NOT: icmp
41+
// CHECK-NOT: assume
4042
// CHECK: ret i8 %x
4143

4244
transmute(x)
@@ -45,12 +47,13 @@ pub unsafe fn check_from_enum(x: SmallEnum) -> i8 {
4547
// CHECK-LABEL: @check_to_ordering(
4648
#[no_mangle]
4749
pub unsafe fn check_to_ordering(x: u8) -> std::cmp::Ordering {
48-
// OPT: %0 = icmp uge i8 %x, -1
49-
// OPT: %1 = icmp ule i8 %x, 1
50-
// OPT: %2 = or i1 %0, %1
51-
// OPT: call void @llvm.assume(i1 %2)
52-
// DBG-NOT: icmp
53-
// DBG-NOT: assume
50+
// CHECK-NOT: icmp
51+
// CHECK-NOT: assume
52+
// OPT: %0 = sub i8 %x, -1
53+
// OPT: %1 = icmp ule i8 %0, 2
54+
// OPT: call void @llvm.assume(i1 %1)
55+
// CHECK-NOT: icmp
56+
// CHECK-NOT: assume
5457
// CHECK: ret i8 %x
5558

5659
transmute(x)
@@ -59,12 +62,13 @@ pub unsafe fn check_to_ordering(x: u8) -> std::cmp::Ordering {
5962
// CHECK-LABEL: @check_from_ordering(
6063
#[no_mangle]
6164
pub unsafe fn check_from_ordering(x: std::cmp::Ordering) -> u8 {
62-
// OPT: %0 = icmp uge i8 %x, -1
63-
// OPT: %1 = icmp ule i8 %x, 1
64-
// OPT: %2 = or i1 %0, %1
65-
// OPT: call void @llvm.assume(i1 %2)
66-
// DBG-NOT: icmp
67-
// DBG-NOT: assume
65+
// CHECK-NOT: icmp
66+
// CHECK-NOT: assume
67+
// OPT: %0 = sub i8 %x, -1
68+
// OPT: %1 = icmp ule i8 %0, 2
69+
// OPT: call void @llvm.assume(i1 %1)
70+
// CHECK-NOT: icmp
71+
// CHECK-NOT: assume
6872
// CHECK: ret i8 %x
6973

7074
transmute(x)
@@ -98,14 +102,15 @@ pub enum Minus100ToPlus100 {
98102
// CHECK-LABEL: @check_enum_from_char(
99103
#[no_mangle]
100104
pub unsafe fn check_enum_from_char(x: char) -> Minus100ToPlus100 {
105+
// CHECK-NOT: icmp
106+
// CHECK-NOT: assume
101107
// OPT: %0 = icmp ule i32 %x, 1114111
102108
// OPT: call void @llvm.assume(i1 %0)
103-
// OPT: %1 = icmp uge i32 %x, -100
104-
// OPT: %2 = icmp ule i32 %x, 100
105-
// OPT: %3 = or i1 %1, %2
106-
// OPT: call void @llvm.assume(i1 %3)
107-
// DBG-NOT: icmp
108-
// DBG-NOT: assume
109+
// OPT: %1 = sub i32 %x, -100
110+
// OPT: %2 = icmp ule i32 %1, 200
111+
// OPT: call void @llvm.assume(i1 %2)
112+
// CHECK-NOT: icmp
113+
// CHECK-NOT: assume
109114
// CHECK: ret i32 %x
110115

111116
transmute(x)
@@ -114,14 +119,15 @@ pub unsafe fn check_enum_from_char(x: char) -> Minus100ToPlus100 {
114119
// CHECK-LABEL: @check_enum_to_char(
115120
#[no_mangle]
116121
pub unsafe fn check_enum_to_char(x: Minus100ToPlus100) -> char {
117-
// OPT: %0 = icmp uge i32 %x, -100
118-
// OPT: %1 = icmp ule i32 %x, 100
119-
// OPT: %2 = or i1 %0, %1
122+
// CHECK-NOT: icmp
123+
// CHECK-NOT: assume
124+
// OPT: %0 = sub i32 %x, -100
125+
// OPT: %1 = icmp ule i32 %0, 200
126+
// OPT: call void @llvm.assume(i1 %1)
127+
// OPT: %2 = icmp ule i32 %x, 1114111
120128
// OPT: call void @llvm.assume(i1 %2)
121-
// OPT: %3 = icmp ule i32 %x, 1114111
122-
// OPT: call void @llvm.assume(i1 %3)
123-
// DBG-NOT: icmp
124-
// DBG-NOT: assume
129+
// CHECK-NOT: icmp
130+
// CHECK-NOT: assume
125131
// CHECK: ret i32 %x
126132

127133
transmute(x)
@@ -130,16 +136,20 @@ pub unsafe fn check_enum_to_char(x: Minus100ToPlus100) -> char {
130136
// CHECK-LABEL: @check_swap_pair(
131137
#[no_mangle]
132138
pub unsafe fn check_swap_pair(x: (char, NonZero<u32>)) -> (NonZero<u32>, char) {
139+
// CHECK-NOT: icmp
140+
// CHECK-NOT: assume
133141
// OPT: %0 = icmp ule i32 %x.0, 1114111
134142
// OPT: call void @llvm.assume(i1 %0)
135-
// OPT: %1 = icmp uge i32 %x.0, 1
136-
// OPT: call void @llvm.assume(i1 %1)
137-
// OPT: %2 = icmp uge i32 %x.1, 1
143+
// OPT: %1 = sub i32 %x.0, 1
144+
// OPT: %2 = icmp ule i32 %1, -2
138145
// OPT: call void @llvm.assume(i1 %2)
139-
// OPT: %3 = icmp ule i32 %x.1, 1114111
140-
// OPT: call void @llvm.assume(i1 %3)
141-
// DBG-NOT: icmp
142-
// DBG-NOT: assume
146+
// OPT: %3 = sub i32 %x.1, 1
147+
// OPT: %4 = icmp ule i32 %3, -2
148+
// OPT: call void @llvm.assume(i1 %4)
149+
// OPT: %5 = icmp ule i32 %x.1, 1114111
150+
// OPT: call void @llvm.assume(i1 %5)
151+
// CHECK-NOT: icmp
152+
// CHECK-NOT: assume
143153
// CHECK: %[[P1:.+]] = insertvalue { i32, i32 } poison, i32 %x.0, 0
144154
// CHECK: %[[P2:.+]] = insertvalue { i32, i32 } %[[P1]], i32 %x.1, 1
145155
// CHECK: ret { i32, i32 } %[[P2]]
@@ -150,14 +160,15 @@ pub unsafe fn check_swap_pair(x: (char, NonZero<u32>)) -> (NonZero<u32>, char) {
150160
// CHECK-LABEL: @check_bool_from_ordering(
151161
#[no_mangle]
152162
pub unsafe fn check_bool_from_ordering(x: std::cmp::Ordering) -> bool {
153-
// OPT: %0 = icmp uge i8 %x, -1
154-
// OPT: %1 = icmp ule i8 %x, 1
155-
// OPT: %2 = or i1 %0, %1
163+
// CHECK-NOT: icmp
164+
// CHECK-NOT: assume
165+
// OPT: %0 = sub i8 %x, -1
166+
// OPT: %1 = icmp ule i8 %0, 2
167+
// OPT: call void @llvm.assume(i1 %1)
168+
// OPT: %2 = icmp ule i8 %x, 1
156169
// OPT: call void @llvm.assume(i1 %2)
157-
// OPT: %3 = icmp ule i8 %x, 1
158-
// OPT: call void @llvm.assume(i1 %3)
159-
// DBG-NOT: icmp
160-
// DBG-NOT: assume
170+
// CHECK-NOT: icmp
171+
// CHECK-NOT: assume
161172
// CHECK: %[[R:.+]] = trunc i8 %x to i1
162173
// CHECK: ret i1 %[[R]]
163174

@@ -168,14 +179,15 @@ pub unsafe fn check_bool_from_ordering(x: std::cmp::Ordering) -> bool {
168179
#[no_mangle]
169180
pub unsafe fn check_bool_to_ordering(x: bool) -> std::cmp::Ordering {
170181
// CHECK: %_0 = zext i1 %x to i8
182+
// CHECK-NOT: icmp
183+
// CHECK-NOT: assume
171184
// OPT: %0 = icmp ule i8 %_0, 1
172185
// OPT: call void @llvm.assume(i1 %0)
173-
// OPT: %1 = icmp uge i8 %_0, -1
174-
// OPT: %2 = icmp ule i8 %_0, 1
175-
// OPT: %3 = or i1 %1, %2
176-
// OPT: call void @llvm.assume(i1 %3)
177-
// DBG-NOT: icmp
178-
// DBG-NOT: assume
186+
// OPT: %1 = sub i8 %_0, -1
187+
// OPT: %2 = icmp ule i8 %1, 2
188+
// OPT: call void @llvm.assume(i1 %2)
189+
// CHECK-NOT: icmp
190+
// CHECK-NOT: assume
179191
// CHECK: ret i8 %_0
180192

181193
transmute(x)

‎tests/codegen/transmute-optimized.rs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,3 +110,11 @@ pub fn char_is_negative(c: char) -> bool {
110110
let x: i32 = unsafe { std::mem::transmute(c) };
111111
x < 0
112112
}
113+
114+
// CHECK-LABEL: i1 @transmute_to_char_is_negative(i32
115+
#[no_mangle]
116+
pub fn transmute_to_char_is_negative(x: i32) -> bool {
117+
// CHECK: ret i1 false
118+
let _c: char = unsafe { std::mem::transmute(x) };
119+
x < 0
120+
}

0 commit comments

Comments
 (0)