Fix some vararg-related subtyping issues

base/compiler/typelimits.jl

@@ -92,10 +92,13 @@ function _limit_type_size(@nospecialize(t), @nospecialize(c), sources::SimpleVec
         return t # easy case
     elseif isa(t, DataType) && isempty(t.parameters)
         return t # fast path: unparameterized are always simple
-    elseif isa(unwrap_unionall(t), DataType) && isa(c, Type) && c !== Union{} && c <: t
-        return t # t is already wider than the comparison in the type lattice
-    elseif is_derived_type_from_any(unwrap_unionall(t), sources, depth)
-        return t # t isn't something new
+    else
+        ut = unwrap_unionall(t)
+        if isa(ut, DataType) && ut.name !== _va_typename && isa(c, Type) && c !== Union{} && c <: t
+            return t # t is already wider than the comparison in the type lattice
+        elseif is_derived_type_from_any(ut, sources, depth)
+            return t # t isn't something new
+        end
     end
     # peel off (and ignore) wrappers - they contribute no useful information, so we don't need to consider their size
     # first attempt to turn `c` into a type that contributes meaningful information

src/jltypes.c

@@ -1082,10 +1082,19 @@ static jl_value_t *inst_datatype_inner(jl_datatype_t *dt, jl_svec_t *p, jl_value
                 continue;
             if (jl_is_datatype(pi))
                 continue;
+            if (jl_is_vararg_type(pi)) {
+                pi = jl_unwrap_vararg(pi);
+                if (jl_has_free_typevars(pi))
+                    continue;
+            }
             // normalize types equal to wrappers (prepare for wrapper_id)
             jl_value_t *tw = extract_wrapper(pi);
             if (tw && tw != pi && (tn != jl_type_typename || jl_typeof(pi) == jl_typeof(tw)) &&
                     jl_types_equal(pi, tw)) {
+                if (jl_is_vararg_type(iparams[i])) {
+                    tw = jl_wrap_vararg(tw, jl_tparam1(jl_unwrap_unionall(iparams[i])));
+                    tw = jl_rewrap_unionall(tw, iparams[i]);
+                }
                 iparams[i] = tw;
                 if (p) jl_gc_wb(p, tw);
             }
@@ -1138,6 +1147,7 @@ static jl_value_t *inst_datatype_inner(jl_datatype_t *dt, jl_svec_t *p, jl_value
         }
         int did_normalize = 0;
         jl_value_t *last2 = normalize_vararg(last);
+        assert(!jl_is_unionall(last2) || !jl_is_unionall(((jl_unionall_t*)last2)->body));
         if (last2 != last) {
             last = last2;
             did_normalize = 1;

src/julia.h

@@ -1278,6 +1278,14 @@ STATIC_INLINE jl_value_t *jl_unwrap_vararg(jl_value_t *v) JL_NOTSAFEPOINT
     return jl_tparam0(jl_unwrap_unionall(v));
 }
 
+STATIC_INLINE size_t jl_vararg_length(jl_value_t *v) JL_NOTSAFEPOINT
+{
+    assert(jl_is_vararg_type(v));
+    jl_value_t *len = jl_tparam1(jl_unwrap_unionall(v));
+    assert(jl_is_long(len));
+    return jl_unbox_long(len);
+}
+
 STATIC_INLINE jl_vararg_kind_t jl_vararg_kind(jl_value_t *v) JL_NOTSAFEPOINT
 {
     if (!jl_is_vararg_type(v))