ensure repr(int) on enums uses the same ABI as int

Author: asquared31415

compiler/rustc_abi/src/callconv/reg.rs

@@ -25,7 +25,7 @@ pub enum RegKind {
 impl RegKind {
     pub fn from_primitive(primitive: Primitive) -> Self {
         match primitive {
-            Primitive::Int(..) | Primitive::Pointer(_) => RegKind::Integer,
+            Primitive::Bool | Primitive::Int(..) | Primitive::Pointer(_) => RegKind::Integer,
             Primitive::Float(Float::PpcF128) => RegKind::PpcF128,
             Primitive::Float(Float::F16 | Float::F16B | Float::F32 | Float::F64 | Float::F128) => {
                 RegKind::Float

compiler/rustc_abi/src/layout.rs

@@ -882,8 +882,10 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
         // won't be so conservative.
 
         // Use the initial field alignment
-        let mut ity = if repr.c() || repr.int.is_some() {
+        let mut ity = if repr.c() {
             min_ity
+        } else if let Some(ity) = repr.int {
+            Integer::from_attr(dl, ity)
         } else {
             Integer::for_align(dl, start_align).unwrap_or(min_ity)
         };
@@ -917,10 +919,18 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
                 // We might have no inhabited variants, so pretend there's at least one.
                 .unwrap_or(WrappingRange { start: 0, end: 0 })
         };
-        let tag = Scalar::Initialized {
-            value: Primitive::Int(ity, signed),
-            valid_range: tag_valid_range,
+
+        // for an enum that does NOT have an explicit repr, and only has discriminants 0..=1, use Bool for it
+        let tag = if repr.int.is_none()
+            && !repr.c()
+            && matches!(tag_valid_range, WrappingRange { start: 0, end: 1 })
+            && ity == Integer::I8
+        {
+            Scalar::Initialized { value: Primitive::Bool, valid_range: tag_valid_range }
+        } else {
+            Scalar::Initialized { value: Primitive::Int(ity, signed), valid_range: tag_valid_range }
         };
+        tracing::info!("variants: {:#?}\ntag: {:#?}", variants, tag);
         let mut abi = BackendRepr::Memory { sized: true };
 
         let uninhabited = layout_variants.iter().all(|v| v.is_uninhabited());
@@ -975,6 +985,9 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
                         // Allow integers of the same size with differing signedness.
                         // We arbitrarily choose the signedness of the first variant.
                         (p @ Primitive::Int(x, _), Primitive::Int(y, _)) if x == y => p,
+                        // Allow booleans to unify with integers of size 1
+                        (p @ Primitive::Int(Integer::I8, _), Primitive::Bool)
+                        | (Primitive::Bool, p @ Primitive::Int(Integer::I8, _)) => p,
                         // Allow integers mixed with pointers of the same layout.
                         // We must represent this using a pointer, to avoid
                         // roundtripping pointers through ptrtoint/inttoptr.

compiler/rustc_abi/src/layout/simple.rs

@@ -68,11 +68,12 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
                     Primitive::Int(_, false) => 2,
                     Primitive::Float(_) => 3,
                     Primitive::Pointer(_) => 4,
+                    Primitive::Bool => 5,
                 } << 32,
             )
             // distinguishes references from pointers
             .wrapping_add((range.start as u64).rotate_right(16))
-            // distinguishes char from u32 and bool from u8
+            // distinguishes char from u32 (and mixes bool up a bit too, even though it shouldn't be needed)
             .wrapping_add((range.end as u64).rotate_right(16));
 
         LayoutData {

compiler/rustc_abi/src/layout/ty.rs

@@ -7,8 +7,8 @@ use rustc_macros::StableHash;
 
 use crate::layout::{FieldIdx, VariantIdx};
 use crate::{
-    Align, BackendRepr, FieldsShape, Float, HasDataLayout, LayoutData, Niche, Numeric, PointeeInfo,
-    Primitive, Size, Variants,
+    Align, BackendRepr, FieldsShape, Float, HasDataLayout, Integer, LayoutData, Niche, Numeric,
+    PointeeInfo, Primitive, Size, Variants,
 };
 
 // Explicitly import `Float` to avoid ambiguity with `Primitive::Float`.
@@ -349,6 +349,7 @@ impl<'a, Ty> TyAndLayout<'a, Ty> {
         // one of the match branches.
         let primitive = scalar.primitive();
         match primitive {
+            Primitive::Bool => Some(Numeric::Int(Integer::I8, false)),
             Primitive::Int(integer, is_signed) => Some(Numeric::Int(integer, is_signed)),
             Primitive::Float(
                 float @ (Float::F16 | Float::F32 | Float::F64 | Float::F128 | Float::PpcF128),

compiler/rustc_abi/src/lib.rs

@@ -1476,6 +1476,7 @@ impl Numeric {
 #[derive(Copy, Clone, PartialEq, Eq, Hash)]
 #[cfg_attr(feature = "nightly", derive(StableHash))]
 pub enum Primitive {
+    Bool,
     /// The `bool` is the signedness of the `Integer` type.
     ///
     /// One would think we would not care about such details this low down,
@@ -1491,6 +1492,7 @@ pub enum Primitive {
 impl fmt::Debug for Primitive {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         let name = match *self {
+            Primitive::Bool => "bool",
             Primitive::Int(integer, is_signed) => {
                 if is_signed {
                     integer.int_ty_str()
@@ -1517,6 +1519,7 @@ impl Primitive {
         let dl = cx.data_layout();
 
         match self {
+            Bool => Size::from_bits(1),
             Int(i, _) => i.size(),
             Float(f) => f.size(),
             Pointer(a) => dl.pointer_size_in(a),
@@ -1532,6 +1535,7 @@ impl Primitive {
         let dl = cx.data_layout();
 
         match self {
+            Bool => dl.i1_align,
             Int(i, _) => i.align(dl),
             Float(f) => f.align(dl),
             Pointer(a) => dl.pointer_align_in(a),
@@ -1566,6 +1570,7 @@ impl fmt::Debug for Scalar {
         match self {
             Scalar::Initialized { value, valid_range } => {
                 let (size, is_signed) = match *value {
+                    Primitive::Bool => (Size::from_bits(1), false),
                     Primitive::Int(integer, is_signed) => (integer.size(), is_signed),
                     Primitive::Float(float) => (float.size(), false),
                     Primitive::Pointer(_) => (Size::from_bits(128), false),
@@ -1582,11 +1587,10 @@ impl fmt::Debug for Scalar {
 impl Scalar {
     #[inline]
     pub fn is_bool(&self) -> bool {
-        use Integer::*;
         matches!(
             self,
             Scalar::Initialized {
-                value: Primitive::Int(I8, false),
+                value: Primitive::Bool,
                 valid_range: WrappingRange { start: 0, end: 1 }
             }
         )
@@ -2132,14 +2136,33 @@ impl Niche {
         // In practice this means that enums with `count > 1` are unlikely to claim niche zero,
         // since they have to fit perfectly. If niche zero is already reserved, the selection of
         // bounds are of little interest.
+        //
+        // Because bools still have a size of 1 byte, we may need to potentially adjust the scalar
+        // here to compensate for reserving more space than the single bit. Niche reservation will
+        // never try to go larger than the in-memory size though, so it's always an adjustment
+        // bool -> u8.
         let move_start = |v: WrappingRange| {
             let start = v.start.wrapping_sub(count) & max_value;
-            Some((start, Scalar::Initialized { value, valid_range: v.with_start(start) }))
+            let valid_range = v.with_start(start);
+            let mut value = value;
+            if let Primitive::Bool = value
+                && valid_range.end.wrapping_sub(valid_range.start) >= 2
+            {
+                value = Primitive::Int(Integer::I8, false);
+            }
+            Some((start, Scalar::Initialized { value, valid_range }))
         };
         let move_end = |v: WrappingRange| {
             let start = v.end.wrapping_add(1) & max_value;
             let end = v.end.wrapping_add(count) & max_value;
-            Some((start, Scalar::Initialized { value, valid_range: v.with_end(end) }))
+            let valid_range = v.with_end(end);
+            let mut value = value;
+            if let Primitive::Bool = value
+                && valid_range.end.wrapping_sub(valid_range.start) >= 2
+            {
+                value = Primitive::Int(Integer::I8, false);
+            }
+            Some((start, Scalar::Initialized { value, valid_range }))
         };
         let distance_end_zero = max_value - v.end;
         // FIXME: this ought to work for `bool` too, but that seems to be hitting a miscompilation

compiler/rustc_abi/src/wrapping_range.rs

@@ -99,7 +99,7 @@ impl WrappingRange {
     }
 
     /// The wrapping distance from `self.start` to `self.end`.
-    fn width(&self, size: Size) -> u128 {
+    pub fn width(&self, size: Size) -> u128 {
         size.truncate(u128::wrapping_sub(self.end, self.start))
     }
 

compiler/rustc_codegen_cranelift/src/common.rs

@@ -26,6 +26,7 @@ pub(crate) fn pointer_ty(tcx: TyCtxt<'_>) -> types::Type {
 
 pub(crate) fn scalar_to_clif_type(tcx: TyCtxt<'_>, scalar: Scalar) -> Type {
     match scalar.primitive() {
+        Primitive::Bool => types::I8,
         Primitive::Int(int, _sign) => match int {
             Integer::I8 => types::I8,
             Integer::I16 => types::I16,

compiler/rustc_codegen_gcc/src/type_of.rs

@@ -267,6 +267,7 @@ impl<'tcx> LayoutGccExt<'tcx> for TyAndLayout<'tcx> {
         offset: Size,
     ) -> Type<'gcc> {
         match scalar.primitive() {
+            Bool => cx.type_u8(),
             Int(i, true) => cx.type_from_integer(i),
             Int(i, false) => cx.type_from_unsigned_integer(i),
             Float(f) => cx.type_from_float(f),

compiler/rustc_codegen_llvm/src/abi.rs

@@ -153,6 +153,7 @@ impl LlvmType for Reg {
                 // But providing a more accurate type means fewer casts in LLVM IR,
                 // which helps with optimization.
                 let ty = match hint_vector_elem {
+                    Primitive::Bool => cx.type_i8(),
                     Primitive::Int(integer, _) => match integer {
                         Integer::I8 => cx.type_ix(8),
                         Integer::I16 => cx.type_ix(16),
@@ -493,10 +494,6 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
         let apply_range_attr = |idx: AttributePlace, scalar: rustc_abi::Scalar| {
             if cx.sess().opts.optimize != config::OptLevel::No
                 && matches!(scalar.primitive(), Primitive::Int(..))
-                // If the value is a boolean, the range is 0..2 and that ultimately
-                // become 0..0 when the type becomes i1, which would be rejected
-                // by the LLVM verifier.
-                && !scalar.is_bool()
                 // LLVM also rejects full range.
                 && !scalar.is_always_valid(cx)
             {

compiler/rustc_codegen_llvm/src/builder.rs

@@ -797,7 +797,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
             }
 
             match scalar.primitive() {
-                abi::Primitive::Int(..) => {
+                abi::Primitive::Bool | abi::Primitive::Int(..) => {
                     if !scalar.is_always_valid(bx) {
                         bx.range_metadata(load, scalar.valid_range(bx));
                     }