[WIP] Add existential structs to Checked C (#683)

Cherry-picked from commit e273f31

    This large change adds support for existential struct types to Checked C.  These can be used to replace uses of void pointers in callbacks, abstract data types, and closures.  There is a corresponding commit of tests for existential types to the Checked C repo.

    This changes adds IR support, parsing support, semantic checking/typechecking support, and code generation support.  Serialization/deserialization support still needs to be implemented.   There are also a number of cleanups to do, which are documented in GitHub issues.   There are three parts to the change:
    - Support for existential types.
    - The pack operation.
    - The unpack operation, which is done as a declaration with an initializer.

Author: Abel Nieto

clang/include/clang/AST/ASTContext.h

@@ -32,6 +32,7 @@
 #include "clang/AST/TemplateBase.h"
 #include "clang/AST/TemplateName.h"
 #include "clang/AST/Type.h"
+#include "clang/AST/TypeOrdering.h"
 #include "clang/Basic/AddressSpaces.h"
 #include "clang/Basic/AttrKinds.h"
 #include "clang/Basic/IdentifierTable.h"
@@ -295,6 +296,18 @@ class ASTContext : public RefCountedBase<ASTContext> {
   /// returns 'true'. The parameters in a type application can be retrieved via RecordDecl::typeParams().
   llvm::DenseMap<const RecordDecl *, llvm::SmallVector<RecordDecl *, 4> > DelayedTypeApps;
 
+  /// Mapping from a (type-variable, inner-type) pair to the corresponding existential type.
+  /// e.g. if we've previously created a type E = '_Exists(T, struct Foo<T>)',
+  /// then, this map links '(T, struct Foo<T>) -> E'.
+  /// This map alone is not enough to guarantee unique existential types.
+  /// e.g. consider two types '_Exists(T, struct Foo<T>)' and '_Exists(U, struct Foo<U>)',
+  /// where 'T = TypeVariableType(0, 0)' and 'U = TypeVariableType(1, 0)' (say, because
+  /// U shows up at a higher nestedness level). Then the two types will look different
+  /// when compared with pointer equality, but they should be considered the same type.
+  /// Another way to say this is to say that this map contains entries that are (functional)
+  /// duplicates.
+  llvm::DenseMap<std::pair<const Type *, QualType>, const ExistentialType *> CachedExistTypes;
+
   /// Representation of a "canonical" template template parameter that
   /// is used in canonical template names.
   class CanonicalTemplateTemplateParm : public llvm::FoldingSetNode {
@@ -3138,6 +3151,17 @@ class ASTContext : public RefCountedBase<ASTContext> {
   /// Remove all type applications that have 'Base' as their base RecordDecl.
   /// Return 'true' if the removed key was in the cache, and 'false' otherwise.
   bool removeDelayedTypeApps(RecordDecl *Base);
+
+  // Checked C: Existential Types
+
+  /// Get the existential type corresponding to the pair (type-var, inner-type).
+  /// If there is no cached existential, return `nullptr`.
+  const ExistentialType *getCachedExistentialType(const Type *TypeVar, QualType InnerType);
+
+  /// Add the mapping `(type-var, inner-type) -> exist-type` to the cache of
+  /// existential types. This should only be called once per key (i.e. cache elements
+  /// should not be re-written).
+  void addCachedExistentialType(const Type *TypeVar, QualType InnerType, const ExistentialType *ExistType);
 };
 
 /// Utility function for constructing a nullary selector.

clang/include/clang/AST/Expr.h

@@ -3139,6 +3139,65 @@ class CompoundLiteralExpr : public Expr {
   }
 };
 
+/// \brief Represents a Checked C pack expression.
+/// Used like so '_Exists(T, struct Foo<T>) foo = _Pack(fooImpl, _Exists(T, struct Foo<T>), int)'.
+/// A pack expression is a triple (expr, exist_typ, subst), indicating that we type
+/// 'expr' with the existential type 'exist_typ', using the substitution 'T -> subst', where
+/// 'T' is the type variable bound by the existential type.
+/// e.g. in the example above, 'fooImpl' must have type 'struct Foo<int>', which we obtain by
+/// substituting 'int' for 'T' in 'struct Foo<T>'.
+class PackExpr : public Expr {
+private:
+  /// Start and end location for the pack expression.
+  SourceLocation StartLoc, EndLoc;
+
+  /// The expression that is being packed.
+  Expr *PackedExpr = nullptr;
+
+  /// The result type of the pack operation. This is always an existential type
+  /// after unwrapping the QualType.
+  QualType ExistType;
+  /// The type that will be substituted for the variable in the existential.
+  /// We can combine the existential and this type to check type of the expression
+  /// being packed.
+  /// e.g. if we do '_Pack(foo, _Exists(T, struct Foo<T, U, T>), int)', then
+  /// 'foo' must have type 'struct Foo<int, U, int>'.
+  QualType Subst;
+
+public:
+  PackExpr(Expr *PackedExpr, QualType ExistType, QualType Subst, SourceLocation StartLoc, SourceLocation EndLoc) :
+   Expr(PackExprClass, ExistType, VK_RValue, OK_Ordinary, false, false, false, false),
+   StartLoc(StartLoc), EndLoc(EndLoc), PackedExpr(PackedExpr), ExistType(ExistType), Subst(Subst) {
+    if(!ExistType->isExistentialType()) {
+      llvm_unreachable("_Pack expression expects an existential type");
+    }
+  }
+
+  /// Return the expression packed by this pack expression.
+  Expr *getPackedExpr() const { return PackedExpr; }
+
+  /// Return the type of the pack expression, which must be an existential (after unwrapping).
+  QualType getExistentialType() const { return ExistType; }
+
+  /// Return the "substitution" type that is used to verify that
+  /// the underlying expr can be packed into the existential type.
+  QualType getSubst() const { return Subst; }
+
+  SourceLocation getBeginLoc() const LLVM_READONLY { return StartLoc; }
+  SourceLocation getEndLoc() const LLVM_READONLY { return EndLoc; }
+
+  child_range children() {
+    // Pretend the children are stored in a one-element array.
+    Stmt *Child = PackedExpr;
+    return child_range(&Child, &Child + 1);
+  }
+  const_child_range children() const {
+    // Pretend the children are stored in a one-element array.
+    Stmt *Child = PackedExpr;
+    return const_child_range(&Child, &Child + 1);
+  }
+};
+
 /// \brief Represents a Checked C bounds expression.
 class BoundsExpr : public Expr {
 private:

clang/include/clang/AST/RecursiveASTVisitor.h

@@ -1026,6 +1026,11 @@ DEF_TRAVERSE_TYPE(UnresolvedUsingType, {})
 DEF_TRAVERSE_TYPE(TypedefType, {})
 DEF_TRAVERSE_TYPE(TypeVariableType, {})
 
+DEF_TRAVERSE_TYPE(ExistentialType, {
+  TRY_TO(TraverseType(QualType(T->typeVar(), 0 /* Quals */)));
+  TRY_TO(TraverseType(T->innerType()));
+})
+
 DEF_TRAVERSE_TYPE(TypeOfExprType,
                   { TRY_TO(TraverseStmt(T->getUnderlyingExpr())); })
 
@@ -1263,6 +1268,7 @@ DEF_TRAVERSE_TYPELOC(FunctionProtoType, {
 DEF_TRAVERSE_TYPELOC(UnresolvedUsingType, {})
 DEF_TRAVERSE_TYPELOC(TypedefType, {})
 DEF_TRAVERSE_TYPELOC(TypeVariableType, {})
+DEF_TRAVERSE_TYPELOC(ExistentialType, {}) // TODO: is this correct? (checkedc issue #661)
 
 DEF_TRAVERSE_TYPELOC(TypeOfExprType,
                      { TRY_TO(TraverseStmt(TL.getUnderlyingExpr())); })
@@ -2579,6 +2585,7 @@ DEF_TRAVERSE_STMT(InteropTypeExpr, {})
 DEF_TRAVERSE_STMT(PositionalParameterExpr, {})
 DEF_TRAVERSE_STMT(BoundsValueExpr, {})
 DEF_TRAVERSE_STMT(CHKCBindTemporaryExpr, {})
+DEF_TRAVERSE_STMT(PackExpr, {})
 
 // For coroutines expressions, traverse either the operand
 // as written or the implied calls, depending on what the

clang/include/clang/AST/Type.h

@@ -2063,6 +2063,7 @@ class Type : public ExtQualsTypeCommonBase {
   bool isNtCheckedArrayType() const;
   bool isUncheckedArrayType() const;
   bool isRecordType() const;
+  bool isExistentialType() const; // Checked C existential type
   bool isClassType() const;
   bool isStructureType() const;
   bool isObjCBoxableRecordType() const;
@@ -4403,7 +4404,7 @@ class TypeVariableType : public Type, public llvm::FoldingSetNode {
     return isBoundsInterfaceType;
   }
 
-  void Profile(llvm::FoldingSetNodeID &ID) {
+  void Profile(llvm::FoldingSetNodeID &ID) const {
     Profile(ID, depth, index, isBoundsInterfaceType);
   }
   static void Profile(llvm::FoldingSetNodeID &ID, unsigned int inDepth, unsigned int inIndex,
@@ -4416,7 +4417,6 @@ class TypeVariableType : public Type, public llvm::FoldingSetNode {
   static bool classof(const Type *T) { return T->getTypeClass() == TypeVariable; }
 };
 
-
 class TypedefType : public Type {
   TypedefNameDecl *Decl;
 
@@ -4441,6 +4441,57 @@ class TypedefType : public Type {
   static bool classof(const Type *T) { return T->getTypeClass() == Typedef; }
 };
 
+/// (Checked C extension)
+/// Represents the type '_Exists(T, InnerType)', where 'T' is a type variable and
+/// 'InnerType' is some type that potentially uses 'T'.
+/// There are some restrictions on what 'InnerType' can be. These restrictions are enforced
+/// via checks in the 'pack' and 'unpack' operations, but they amount to requiring that
+/// 'InnerType' is one of:
+///   1) another existential type: e.g. '_Exists(T, _Exists(U, struct Foo<T, U>))'
+///   2) a type application 'C<T>', where 'C' is a generic struct: e.g. '_Exists(T, struct List<T>)'
+///   3) a pointer that satisfies one of 1) - 3)
+class ExistentialType : public Type, public llvm::FoldingSetNode {
+  /// The type variable that is bound by the existential.
+  /// This is of type 'Type' to allow for some polymorphism:
+  /// non-canonical existential types will contain a 'TypedefType' here,
+  /// while canonical ones will contain the underlying 'TypeVariableType'.
+  /// This isn't a 'QualType' because the variable bound by an existential cannot be
+  /// qualified.
+  const Type *TypeVar = nullptr;
+  /// The type wrapped by the existential that potentially uses the bound type variable.
+  QualType InnerType;
+
+public:
+  ExistentialType(const Type *TypeVar, QualType InnerType, QualType Canon) :
+    Type(Existential, Canon, false /* Dependent */ , false /* InstantiationDependent */,
+      false /* VariablyModified */, false /* ContainsUnexpandedParameterPack */),
+    TypeVar(TypeVar), InnerType(InnerType) {
+    if (!TypedefType::classof(TypeVar) && !TypeVariableType::classof(TypeVar)) {
+      llvm_unreachable("Type variable should be a 'TypedefType' or 'TypeVariableType'");
+    }
+  }
+
+  const Type *typeVar() const { return TypeVar; }
+  QualType innerType() const { return InnerType; }
+
+  bool isSugared(void) const { return false; }
+  QualType desugar(void) const { return QualType(this, 0); }
+
+  // TODO: can't implement 'Profile' because TypedefType doesn't have a 'Profile' method (checkedc issue #661).
+  /*
+  void Profile(llvm::FoldingSetNodeID &ID) {
+    Profile(ID, TypeVar, InnerType);
+  }
+  static void Profile(llvm::FoldingSetNodeID &ID, const Type *TypeVar, QualType InnerType) {
+    if (const auto *TV = TypeVar->getAs<TypedefType>()) TV->Profile(ID);
+    else if (const auto *TV = TypeVar->getAs<TypeVariableType>()) TV->Profile(ID);
+    InnerType.Profile(ID);
+  }
+  */
+
+  static bool classof(const Type *T) { return T->getTypeClass() == Existential; }
+};
+
 /// Represents a `typeof` (or __typeof__) expression (a GCC extension).
 class TypeOfExprType : public Type {
   Expr *TOExpr;
@@ -6713,6 +6764,10 @@ inline bool Type::isRecordType() const {
   return isa<RecordType>(CanonicalType);
 }
 
+inline bool Type::isExistentialType() const {
+  return isa<ExistentialType>(CanonicalType);
+}
+
 inline bool Type::isEnumeralType() const {
   return isa<EnumType>(CanonicalType);
 }

clang/include/clang/AST/TypeLoc.h

@@ -666,6 +666,11 @@ class TypeVariableTypeLoc : public InheritingConcreteTypeLoc<TypeSpecTypeLoc,
 
 };
 
+class ExistentialTypeLoc : public InheritingConcreteTypeLoc<TypeSpecTypeLoc,
+                                                            ExistentialTypeLoc,
+                                                            ExistentialType> {
+};
+
 /// Wrapper for source info for typedefs.
 
 class TypedefTypeLoc : public InheritingConcreteTypeLoc<TypeSpecTypeLoc,

clang/include/clang/AST/TypeNodes.def

@@ -67,6 +67,7 @@ ABSTRACT_TYPE(Reference, Type)
 TYPE(LValueReference, ReferenceType)
 TYPE(RValueReference, ReferenceType)
 TYPE(TypeVariable, Type)
+TYPE(Existential, Type)
 TYPE(MemberPointer, Type)
 ABSTRACT_TYPE(Array, Type)
 TYPE(ConstantArray, ArrayType)

clang/include/clang/Basic/DiagnosticParseKinds.td

@@ -1288,7 +1288,7 @@ def err_single_itype_expr_allowed : Error<
 def err_forany_type_variable_declaration_expected : Error<
   "expected list of type variables">;
 
-def err_forany_type_variable_identifier_expected : Error<
+def err_type_variable_identifier_expected : Error<
   "expected type variable identifier">;
 
 def err_generic_specifier_unexpected_token : Error<
@@ -1312,6 +1312,15 @@ def err_expected_type_argument_list_for_generic_instance : Error<
 def err_expected_type_argument_list_for_itype_generic_instance : Error<
   "expected a type argument list for a struct with a bounds-safe interface in a checked scope">;
 
+def err_pack_expr_returns_existential_type : Error <
+  "return type of a pack expression must be an existential type, but got %0 instead">;
+
+def err_unpack_expected_one_type_variable : Error<
+  "unpack specifier expects exactly one type variable, but got %0 instead">;
+
+def err_unpack_expected_existential_initializer : Error<
+  "unpack specifer expects an initializer that has an existential type">;
+
  // Checked C pragmas
 def err_pragma_checked_scope_invalid_argument : Error<
   "unexpected argument '%0' to '#pragma CHECKED_SCOPE'; "

clang/include/clang/Basic/DiagnosticSemaKinds.td

@@ -9659,6 +9659,9 @@ def err_typecheck_bounds_type_annotation_incompatible : Error<
 def err_typecheck_bounds_type_annotation_must_be_checked_type : Error<
   "type must be a checked type">;
 
+def err_typecheck_existential_type_witness_mismatch : Error<
+  "witness type does not match existential type">;
+
 def err_bounds_type_annotation_lost_checking : Error<
   "type %diff{$ loses checking of declared type $|"
   "loses checking of declared type}0,1">;

clang/include/clang/Basic/Specifiers.h

@@ -81,6 +81,7 @@ namespace clang {
     TST_plainPtr,     // Checked C _Ptr type
     TST_arrayPtr,     // Checked C _Array_ptr type
     TST_ntarrayPtr ,  // Chcecked C _Nt_array_ptr type
+    TST_exists,       // Checked C _Exists type
 #define GENERIC_IMAGE_TYPE(ImgType, Id) TST_##ImgType##_t, // OpenCL image types
 #include "clang/Basic/OpenCLImageTypes.def"
     TST_error // erroneous type

clang/include/clang/Basic/StmtNodes.td

@@ -190,6 +190,7 @@ def PositionalParameterExpr : DStmt<Expr>;
 def BoundsCastExpr : DStmt<ExplicitCastExpr>;
 def CHKCBindTemporaryExpr : DStmt<Expr>;
 def BoundsValueExpr : DStmt<Expr>;
+def PackExpr : DStmt<Expr>;
 
 // CUDA Expressions.
 def CUDAKernelCallExpr : DStmt<CallExpr>;