@@ -4,6 +4,7 @@ use std::{cmp, iter};
44
55use rustc_hashes:: Hash64 ;
66use rustc_index:: Idx ;
7+ use rustc_index:: bit_set:: BitMatrix ;
78use tracing:: debug;
89
910use crate :: {
@@ -12,6 +13,9 @@ use crate::{
1213 Variants , WrappingRange ,
1314} ;
1415
16+ mod coroutine;
17+ mod simple;
18+
1519#[ cfg( feature = "nightly" ) ]
1620mod ty;
1721
@@ -60,31 +64,44 @@ pub enum LayoutCalculatorError<F> {
6064
6165 /// The fields or variants have irreconcilable reprs
6266 ReprConflict ,
67+
68+ /// The length of an SIMD type is zero
69+ ZeroLengthSimdType ,
70+
71+ /// The length of an SIMD type exceeds the maximum number of lanes
72+ OversizedSimdType { max_lanes : u64 } ,
73+
74+ /// An element type of an SIMD type isn't a primitive
75+ NonPrimitiveSimdType ( F ) ,
6376}
6477
6578impl < F > LayoutCalculatorError < F > {
6679 pub fn without_payload ( & self ) -> LayoutCalculatorError < ( ) > {
67- match self {
68- LayoutCalculatorError :: UnexpectedUnsized ( _) => {
69- LayoutCalculatorError :: UnexpectedUnsized ( ( ) )
70- }
71- LayoutCalculatorError :: SizeOverflow => LayoutCalculatorError :: SizeOverflow ,
72- LayoutCalculatorError :: EmptyUnion => LayoutCalculatorError :: EmptyUnion ,
73- LayoutCalculatorError :: ReprConflict => LayoutCalculatorError :: ReprConflict ,
80+ use LayoutCalculatorError :: * ;
81+ match * self {
82+ UnexpectedUnsized ( _) => UnexpectedUnsized ( ( ) ) ,
83+ SizeOverflow => SizeOverflow ,
84+ EmptyUnion => EmptyUnion ,
85+ ReprConflict => ReprConflict ,
86+ ZeroLengthSimdType => ZeroLengthSimdType ,
87+ OversizedSimdType { max_lanes } => OversizedSimdType { max_lanes } ,
88+ NonPrimitiveSimdType ( _) => NonPrimitiveSimdType ( ( ) ) ,
7489 }
7590 }
7691
7792 /// Format an untranslated diagnostic for this type
7893 ///
7994 /// Intended for use by rust-analyzer, as neither it nor `rustc_abi` depend on fluent infra.
8095 pub fn fallback_fmt ( & self , f : & mut fmt:: Formatter < ' _ > ) -> fmt:: Result {
96+ use LayoutCalculatorError :: * ;
8197 f. write_str ( match self {
82- LayoutCalculatorError :: UnexpectedUnsized ( _) => {
83- "an unsized type was found where a sized type was expected"
98+ UnexpectedUnsized ( _) => "an unsized type was found where a sized type was expected" ,
99+ SizeOverflow => "size overflow" ,
100+ EmptyUnion => "type is a union with no fields" ,
101+ ReprConflict => "type has an invalid repr" ,
102+ ZeroLengthSimdType | OversizedSimdType { .. } | NonPrimitiveSimdType ( _) => {
103+ "invalid simd type definition"
84104 }
85- LayoutCalculatorError :: SizeOverflow => "size overflow" ,
86- LayoutCalculatorError :: EmptyUnion => "type is a union with no fields" ,
87- LayoutCalculatorError :: ReprConflict => "type has an invalid repr" ,
88105 } )
89106 }
90107}
@@ -102,41 +119,115 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
102119 Self { cx }
103120 }
104121
105- pub fn scalar_pair < FieldIdx : Idx , VariantIdx : Idx > (
122+ pub fn array_like < FieldIdx : Idx , VariantIdx : Idx , F > (
106123 & self ,
107- a : Scalar ,
108- b : Scalar ,
109- ) -> LayoutData < FieldIdx , VariantIdx > {
110- let dl = self . cx . data_layout ( ) ;
111- let b_align = b. align ( dl) ;
112- let align = a. align ( dl) . max ( b_align) . max ( dl. aggregate_align ) ;
113- let b_offset = a. size ( dl) . align_to ( b_align. abi ) ;
114- let size = ( b_offset + b. size ( dl) ) . align_to ( align. abi ) ;
124+ element : & LayoutData < FieldIdx , VariantIdx > ,
125+ count_if_sized : Option < u64 > , // None for slices
126+ ) -> LayoutCalculatorResult < FieldIdx , VariantIdx , F > {
127+ let count = count_if_sized. unwrap_or ( 0 ) ;
128+ let size =
129+ element. size . checked_mul ( count, & self . cx ) . ok_or ( LayoutCalculatorError :: SizeOverflow ) ?;
115130
116- // HACK(nox): We iter on `b` and then `a` because `max_by_key`
117- // returns the last maximum.
118- let largest_niche = Niche :: from_scalar ( dl, b_offset, b)
119- . into_iter ( )
120- . chain ( Niche :: from_scalar ( dl, Size :: ZERO , a) )
121- . max_by_key ( |niche| niche. available ( dl) ) ;
131+ Ok ( LayoutData {
132+ variants : Variants :: Single { index : VariantIdx :: new ( 0 ) } ,
133+ fields : FieldsShape :: Array { stride : element. size , count } ,
134+ backend_repr : BackendRepr :: Memory { sized : count_if_sized. is_some ( ) } ,
135+ largest_niche : element. largest_niche . filter ( |_| count != 0 ) ,
136+ uninhabited : element. uninhabited && count != 0 ,
137+ align : element. align ,
138+ size,
139+ max_repr_align : None ,
140+ unadjusted_abi_align : element. align . abi ,
141+ randomization_seed : element. randomization_seed . wrapping_add ( Hash64 :: new ( count) ) ,
142+ } )
143+ }
122144
123- let combined_seed = a. size ( & self . cx ) . bytes ( ) . wrapping_add ( b. size ( & self . cx ) . bytes ( ) ) ;
145+ pub fn simd_type <
146+ FieldIdx : Idx ,
147+ VariantIdx : Idx ,
148+ F : AsRef < LayoutData < FieldIdx , VariantIdx > > + fmt:: Debug ,
149+ > (
150+ & self ,
151+ element : F ,
152+ count : u64 ,
153+ repr_packed : bool ,
154+ ) -> LayoutCalculatorResult < FieldIdx , VariantIdx , F > {
155+ let elt = element. as_ref ( ) ;
156+ if count == 0 {
157+ return Err ( LayoutCalculatorError :: ZeroLengthSimdType ) ;
158+ } else if count > crate :: MAX_SIMD_LANES {
159+ return Err ( LayoutCalculatorError :: OversizedSimdType {
160+ max_lanes : crate :: MAX_SIMD_LANES ,
161+ } ) ;
162+ }
124163
125- LayoutData {
164+ let BackendRepr :: Scalar ( e_repr) = elt. backend_repr else {
165+ return Err ( LayoutCalculatorError :: NonPrimitiveSimdType ( element) ) ;
166+ } ;
167+
168+ // Compute the size and alignment of the vector
169+ let dl = self . cx . data_layout ( ) ;
170+ let size =
171+ elt. size . checked_mul ( count, dl) . ok_or_else ( || LayoutCalculatorError :: SizeOverflow ) ?;
172+ let ( repr, align) = if repr_packed && !count. is_power_of_two ( ) {
173+ // Non-power-of-two vectors have padding up to the next power-of-two.
174+ // If we're a packed repr, remove the padding while keeping the alignment as close
175+ // to a vector as possible.
176+ (
177+ BackendRepr :: Memory { sized : true } ,
178+ AbiAndPrefAlign {
179+ abi : Align :: max_aligned_factor ( size) ,
180+ pref : dl. llvmlike_vector_align ( size) . pref ,
181+ } ,
182+ )
183+ } else {
184+ ( BackendRepr :: SimdVector { element : e_repr, count } , dl. llvmlike_vector_align ( size) )
185+ } ;
186+ let size = size. align_to ( align. abi ) ;
187+
188+ Ok ( LayoutData {
126189 variants : Variants :: Single { index : VariantIdx :: new ( 0 ) } ,
127190 fields : FieldsShape :: Arbitrary {
128- offsets : [ Size :: ZERO , b_offset ] . into ( ) ,
129- memory_index : [ 0 , 1 ] . into ( ) ,
191+ offsets : [ Size :: ZERO ] . into ( ) ,
192+ memory_index : [ 0 ] . into ( ) ,
130193 } ,
131- backend_repr : BackendRepr :: ScalarPair ( a , b ) ,
132- largest_niche,
194+ backend_repr : repr ,
195+ largest_niche : elt . largest_niche ,
133196 uninhabited : false ,
134- align,
135197 size,
198+ align,
136199 max_repr_align : None ,
137- unadjusted_abi_align : align. abi ,
138- randomization_seed : Hash64 :: new ( combined_seed) ,
139- }
200+ unadjusted_abi_align : elt. align . abi ,
201+ randomization_seed : elt. randomization_seed . wrapping_add ( Hash64 :: new ( count) ) ,
202+ } )
203+ }
204+
205+ /// Compute the layout for a coroutine.
206+ ///
207+ /// This uses dedicated code instead of [`Self::layout_of_struct_or_enum`], as coroutine
208+ /// fields may be shared between multiple variants (see the [`coroutine`] module for details).
209+ pub fn coroutine <
210+ ' a ,
211+ F : Deref < Target = & ' a LayoutData < FieldIdx , VariantIdx > > + fmt:: Debug + Copy ,
212+ VariantIdx : Idx ,
213+ FieldIdx : Idx ,
214+ LocalIdx : Idx ,
215+ > (
216+ & self ,
217+ local_layouts : & IndexSlice < LocalIdx , F > ,
218+ prefix_layouts : IndexVec < FieldIdx , F > ,
219+ variant_fields : & IndexSlice < VariantIdx , IndexVec < FieldIdx , LocalIdx > > ,
220+ storage_conflicts : & BitMatrix < LocalIdx , LocalIdx > ,
221+ tag_to_layout : impl Fn ( Scalar ) -> F ,
222+ ) -> LayoutCalculatorResult < FieldIdx , VariantIdx , F > {
223+ coroutine:: layout (
224+ self ,
225+ local_layouts,
226+ prefix_layouts,
227+ variant_fields,
228+ storage_conflicts,
229+ tag_to_layout,
230+ )
140231 }
141232
142233 pub fn univariant <
@@ -214,25 +305,6 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
214305 layout
215306 }
216307
217- pub fn layout_of_never_type < FieldIdx : Idx , VariantIdx : Idx > (
218- & self ,
219- ) -> LayoutData < FieldIdx , VariantIdx > {
220- let dl = self . cx . data_layout ( ) ;
221- // This is also used for uninhabited enums, so we use `Variants::Empty`.
222- LayoutData {
223- variants : Variants :: Empty ,
224- fields : FieldsShape :: Primitive ,
225- backend_repr : BackendRepr :: Memory { sized : true } ,
226- largest_niche : None ,
227- uninhabited : true ,
228- align : dl. i8_align ,
229- size : Size :: ZERO ,
230- max_repr_align : None ,
231- unadjusted_abi_align : dl. i8_align . abi ,
232- randomization_seed : Hash64 :: ZERO ,
233- }
234- }
235-
236308 pub fn layout_of_struct_or_enum <
237309 ' a ,
238310 FieldIdx : Idx ,
@@ -260,7 +332,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
260332 Some ( present_first) => present_first,
261333 // Uninhabited because it has no variants, or only absent ones.
262334 None if is_enum => {
263- return Ok ( self . layout_of_never_type ( ) ) ;
335+ return Ok ( LayoutData :: never_type ( & self . cx ) ) ;
264336 }
265337 // If it's a struct, still compute a layout so that we can still compute the
266338 // field offsets.
@@ -949,7 +1021,8 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
9491021 // Common prim might be uninit.
9501022 Scalar :: Union { value : prim }
9511023 } ;
952- let pair = self . scalar_pair :: < FieldIdx , VariantIdx > ( tag, prim_scalar) ;
1024+ let pair =
1025+ LayoutData :: < FieldIdx , VariantIdx > :: scalar_pair ( & self . cx , tag, prim_scalar) ;
9531026 let pair_offsets = match pair. fields {
9541027 FieldsShape :: Arbitrary { ref offsets, ref memory_index } => {
9551028 assert_eq ! ( memory_index. raw, [ 0 , 1 ] ) ;
@@ -1341,7 +1414,8 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
13411414 } else {
13421415 ( ( j, b) , ( i, a) )
13431416 } ;
1344- let pair = self . scalar_pair :: < FieldIdx , VariantIdx > ( a, b) ;
1417+ let pair =
1418+ LayoutData :: < FieldIdx , VariantIdx > :: scalar_pair ( & self . cx , a, b) ;
13451419 let pair_offsets = match pair. fields {
13461420 FieldsShape :: Arbitrary { ref offsets, ref memory_index } => {
13471421 assert_eq ! ( memory_index. raw, [ 0 , 1 ] ) ;
0 commit comments