Merge remote-tracking branch 'origin/master' into teh-jn/lazydotfuse

* origin/master: (22 commits)
  separate `isbitstype(::Type)` from `isbits` (#26850)
  bugfix for regex matches ending with non-ASCII (#26831)
  [NewOptimizer] track inbounds state as a per-statement flag
  change default LOAD_PATH and DEPOT_PATH (#26804, fix #25709)
  Change url scheme to https (#26835)
  [NewOptimizer] inlining: Refactor todo object
  inference: enable CodeInfo method_for_inference_limit_heuristics support (#26822)
  [NewOptimizer] Fix _apply elision (#26821)
  add test case from issue #26607, cfunction with no args (#26838)
  add `do` in front-end deparser. fixes #17781 (#26840)
  Preserve CallInst metadata in LateLowerGCFrame pass.
  Improve differences from R documentation (#26810)
  reserve syntax that could be used for computed field types (#18466) (#26816)
  Add support for Atomic{Bool} (Fix #26542). (#26597)
  Remove argument restriction on dims2string and inds2string (#26799) (#26817)
  remove some unnecessary `eltype` methods (#26791)
  optimize: ensure merge_value_ssa doesn't drop PiNodes
  inference: improve tmerge for Conditional and Const
  ensure more iterators stay type-stable
  code loading docs (#26787)
  ...

Author: mbauman

Makefile

@@ -1,20 +1,13 @@
 JULIAHOME := $(abspath $(dir $(lastword $(MAKEFILE_LIST))))
 include $(JULIAHOME)/Make.inc
 
-# TODO: Code bundled with Julia should be installed into a versioned directory,
-# prefix/share/julia/VERSDIR, so that in the future one can have multiple
-# major versions of Julia installed concurrently. Third-party code that
-# is not controlled by Pkg should be installed into
-# prefix/share/julia/site/VERSDIR (not prefix/share/julia/VERSDIR/site ...
-# so that prefix/share/julia/VERSDIR can be overwritten without touching
-# third-party code).
 VERSDIR := v`cut -d. -f1-2 < $(JULIAHOME)/VERSION`
 
 default: $(JULIA_BUILD_MODE) # contains either "debug" or "release"
 all: debug release
 
 # sort is used to remove potential duplicates
-DIRS := $(sort $(build_bindir) $(build_depsbindir) $(build_libdir) $(build_private_libdir) $(build_libexecdir) $(build_includedir) $(build_includedir)/julia $(build_sysconfdir)/julia $(build_datarootdir)/julia $(build_datarootdir)/julia/site $(build_man1dir))
+DIRS := $(sort $(build_bindir) $(build_depsbindir) $(build_libdir) $(build_private_libdir) $(build_libexecdir) $(build_includedir) $(build_includedir)/julia $(build_sysconfdir)/julia $(build_datarootdir)/julia $(build_datarootdir)/julia/stdlib $(build_man1dir))
 ifneq ($(BUILDROOT),$(JULIAHOME))
 BUILDDIRS := $(BUILDROOT) $(addprefix $(BUILDROOT)/,base src ui doc deps test test/embedding)
 BUILDDIRMAKE := $(addsuffix /Makefile,$(BUILDDIRS))
@@ -46,8 +39,8 @@ endif
 $(foreach dir,$(DIRS),$(eval $(call dir_target,$(dir))))
 $(foreach link,base $(JULIAHOME)/test,$(eval $(call symlink_target,$(link),$(build_datarootdir)/julia,$(notdir $(link)))))
 
-build_defaultpkgdir = $(build_datarootdir)/julia/site/$(shell echo $(VERSDIR))
-$(eval $(call symlink_target,$(JULIAHOME)/stdlib,$(build_datarootdir)/julia/site,$(shell echo $(VERSDIR))))
+build_defaultpkgdir = $(build_datarootdir)/julia/stdlib/$(shell echo $(VERSDIR))
+$(eval $(call symlink_target,$(JULIAHOME)/stdlib,$(build_datarootdir)/julia/stdlib,$(shell echo $(VERSDIR))))
 
 julia_flisp.boot.inc.phony: julia-deps
 	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/src julia_flisp.boot.inc.phony
@@ -284,7 +277,7 @@ endef
 
 install: $(build_depsbindir)/stringreplace $(BUILDROOT)/doc/_build/html/en/index.html
 	@$(MAKE) $(QUIET_MAKE) all
-	@for subdir in $(bindir) $(datarootdir)/julia/site/$(VERSDIR) $(docdir) $(man1dir) $(includedir)/julia $(libdir) $(private_libdir) $(sysconfdir); do \
+	@for subdir in $(bindir) $(datarootdir)/julia/stdlib/$(VERSDIR) $(docdir) $(man1dir) $(includedir)/julia $(libdir) $(private_libdir) $(sysconfdir); do \
 		mkdir -p $(DESTDIR)$$subdir; \
 	done
 

base/Enums.jl

@@ -75,7 +75,7 @@ macro enum(T, syms...)
     if isa(T, Expr) && T.head == :(::) && length(T.args) == 2 && isa(T.args[1], Symbol)
         typename = T.args[1]
         basetype = eval(__module__, T.args[2])
-        if !isa(basetype, DataType) || !(basetype <: Integer) || !isbits(basetype)
+        if !isa(basetype, DataType) || !(basetype <: Integer) || !isbitstype(basetype)
             throw(ArgumentError("invalid base type for Enum $typename, $T=::$basetype; base type must be an integer primitive type"))
         end
     elseif !isa(T, Symbol)

base/abstractdict.jl

@@ -672,11 +672,11 @@ _oidd_nextind(a, i) = reinterpret(Int, ccall(:jl_eqtable_nextind, Csize_t, (Any,
 
 start(d::IdDict) = _oidd_nextind(d.ht, 0)
 done(d::IdDict, i) = (i == -1)
-next(d::IdDict{K,V}, i) where {K, V} = (Pair{K,V}(d.ht[i+1], d.ht[i+2]), _oidd_nextind(d.ht, i+2))
+next(d::IdDict{K, V}, i) where {K, V} = (Pair{K, V}(d.ht[i + 1]::K, d.ht[i + 2]::V), _oidd_nextind(d.ht, i + 2))
 
 length(d::IdDict) = d.count
 
-copy(d::IdDict) = IdDict(d)
+copy(d::IdDict) = typeof(d)(d)
 
 get!(d::IdDict{K,V}, @nospecialize(key), @nospecialize(default)) where {K, V} = (d[key] = get(d, key, default))::V
 
@@ -689,14 +689,14 @@ mutable struct IdSet{T} <: AbstractSet{T}
     dict::IdDict{T,Nothing}
 
     IdSet{T}() where {T} = new(IdDict{T,Nothing}())
-    IdSet{T}(s::IdSet{T}) where {T} = new(IdDict{T,Nothing}(s.dict))
+    IdSet{T}(s::IdSet{T}) where {T} = new(copy(s.dict))
 end
 
 IdSet{T}(itr) where {T} = union!(IdSet{T}(), itr)
 IdSet() = IdSet{Any}()
 
-copy(s::IdSet{T}) where {T} = IdSet{T}(s)
-copymutable(s::IdSet{T}) where {T} = IdSet{T}(s)
+copymutable(s::IdSet) = typeof(s)(s)
+copy(s::IdSet) = typeof(s)(s)
 
 isempty(s::IdSet) = isempty(s.dict)
 length(s::IdSet)  = length(s.dict)

base/array.jl

@@ -85,7 +85,6 @@ UInt8
 ```
 """
 eltype(::Type) = Any
-eltype(::Type{Any}) = Any
 eltype(::Type{Bottom}) = throw(ArgumentError("Union{} does not have elements"))
 eltype(x) = eltype(typeof(x))
 
@@ -117,15 +116,15 @@ asize_from(a::Array, n) = n > ndims(a) ? () : (arraysize(a,n), asize_from(a, n+1
 """
     Base.isbitsunion(::Type{T})
 
-Return whether a type is an "is-bits" Union type, meaning each type included in a Union is `isbits`.
+Return whether a type is an "is-bits" Union type, meaning each type included in a Union is `isbitstype`.
 """
 isbitsunion(u::Union) = ccall(:jl_array_store_unboxed, Cint, (Any,), u) == Cint(1)
 isbitsunion(x) = false
 
 """
     Base.bitsunionsize(U::Union)
 
-For a Union of `isbits` types, return the size of the largest type; assumes `Base.isbitsunion(U) == true`
+For a Union of `isbitstype` types, return the size of the largest type; assumes `Base.isbitsunion(U) == true`
 """
 function bitsunionsize(u::Union)
     sz = Ref{Csize_t}(0)
@@ -135,7 +134,7 @@ function bitsunionsize(u::Union)
 end
 
 length(a::Array) = arraylen(a)
-elsize(::Type{<:Array{T}}) where {T} = isbits(T) ? sizeof(T) : (isbitsunion(T) ? bitsunionsize(T) : sizeof(Ptr))
+elsize(::Type{<:Array{T}}) where {T} = isbitstype(T) ? sizeof(T) : (isbitsunion(T) ? bitsunionsize(T) : sizeof(Ptr))
 sizeof(a::Array) = Core.sizeof(a)
 
 function isassigned(a::Array, i::Int...)
@@ -178,7 +177,7 @@ the same manner as C.
 function unsafe_copyto!(dest::Array{T}, doffs, src::Array{T}, soffs, n) where T
     t1 = @_gc_preserve_begin dest
     t2 = @_gc_preserve_begin src
-    if isbits(T)
+    if isbitstype(T)
         unsafe_copyto!(pointer(dest, doffs), pointer(src, soffs), n)
     elseif isbitsunion(T)
         ccall(:memmove, Ptr{Cvoid}, (Ptr{Cvoid}, Ptr{Cvoid}, UInt),
@@ -1478,7 +1477,7 @@ function vcat(arrays::Vector{T}...) where T
     end
     arr = Vector{T}(undef, n)
     ptr = pointer(arr)
-    if isbits(T)
+    if isbitstype(T)
         elsz = Core.sizeof(T)
     elseif isbitsunion(T)
         elsz = bitsunionsize(T)
@@ -1490,7 +1489,7 @@ function vcat(arrays::Vector{T}...) where T
     for a in arrays
         na = length(a)
         nba = na * elsz
-        if isbits(T)
+        if isbitstype(T)
             ccall(:memcpy, Ptr{Cvoid}, (Ptr{Cvoid}, Ptr{Cvoid}, UInt),
                   ptr, a, nba)
         elseif isbitsunion(T)

base/arraymath.jl

@@ -94,7 +94,7 @@ function reverse(A::Array{T}; dims::Integer) where T
             end
         end
     else
-        if isbits(T) && M>200
+        if isbitstype(T) && M>200
             for i = 1:sd
                 ri = sd+1-i
                 for j=0:stride:(N-stride)

base/atomics.jl

@@ -26,10 +26,12 @@ else
                       UInt8, UInt16, UInt32, UInt64, UInt128)
 end
 const floattypes = (Float16, Float32, Float64)
-# TODO: Support Bool, Ptr
-const atomictypes = (inttypes..., floattypes...)
+const arithmetictypes = (inttypes..., floattypes...)
+# TODO: Support Ptr
+const atomictypes = (arithmetictypes..., Bool)
 const IntTypes = Union{inttypes...}
 const FloatTypes = Union{floattypes...}
+const ArithmeticTypes = Union{arithmetictypes...}
 const AtomicTypes = Union{atomictypes...}
 
 """
@@ -39,8 +41,8 @@ Holds a reference to an object of type `T`, ensuring that it is only
 accessed atomically, i.e. in a thread-safe manner.
 
 Only certain "simple" types can be used atomically, namely the
-primitive integer and float-point types. These are `Int8`...`Int128`,
-`UInt8`...`UInt128`, and `Float16`...`Float64`.
+primitive boolean, integer, and float-point types. These are `Bool`,
+`Int8`...`Int128`, `UInt8`...`UInt128`, and `Float16`...`Float64`.
 
 New atomic objects can be created from a non-atomic values; if none is
 specified, the atomic object is initialized with zero.
@@ -130,11 +132,12 @@ julia> x[]
 function atomic_xchg! end
 
 """
-    Threads.atomic_add!(x::Atomic{T}, val::T) where T
+    Threads.atomic_add!(x::Atomic{T}, val::T) where T <: ArithmeticTypes
 
 Atomically add `val` to `x`
 
-Performs `x[] += val` atomically. Returns the **old** value.
+Performs `x[] += val` atomically. Returns the **old** value. Not defined for
+`Atomic{Bool}`.
 
 For further details, see LLVM's `atomicrmw add` instruction.
 
@@ -153,11 +156,12 @@ julia> x[]
 function atomic_add! end
 
 """
-    Threads.atomic_sub!(x::Atomic{T}, val::T) where T
+    Threads.atomic_sub!(x::Atomic{T}, val::T) where T <: ArithmeticTypes
 
 Atomically subtract `val` from `x`
 
-Performs `x[] -= val` atomically. Returns the **old** value.
+Performs `x[] -= val` atomically. Returns the **old** value. Not defined for
+`Atomic{Bool}`.
 
 For further details, see LLVM's `atomicrmw sub` instruction.
 
@@ -317,7 +321,7 @@ unsafe_convert(::Type{Ptr{T}}, x::Atomic{T}) where {T} = convert(Ptr{T}, pointer
 setindex!(x::Atomic{T}, v) where {T} = setindex!(x, convert(T, v))
 
 const llvmtypes = IdDict{Any,String}(
-    Bool => "i1",
+    Bool => "i8",  # julia represents bools with 8-bits for now. # TODO: is this okay?
     Int8 => "i8", UInt8 => "i8",
     Int16 => "i16", UInt16 => "i16",
     Int32 => "i32", UInt32 => "i32",
@@ -380,13 +384,15 @@ for typ in atomictypes
                      unsafe_convert(Ptr{$typ}, x), cmp, new)
     end
 
-    for rmwop in [:xchg, :add, :sub, :and, :nand, :or, :xor, :max, :min]
+    arithmetic_ops = [:add, :sub]
+    for rmwop in [arithmetic_ops..., :xchg, :and, :nand, :or, :xor, :max, :min]
         rmw = string(rmwop)
         fn = Symbol("atomic_", rmw, "!")
         if (rmw == "max" || rmw == "min") && typ <: Unsigned
             # LLVM distinguishes signedness in the operation, not the integer type.
             rmw = "u" * rmw
         end
+        if rmwop in arithmetic_ops && !(typ <: ArithmeticTypes) continue end
         if typ <: Integer
             @eval $fn(x::Atomic{$typ}, v::$typ) =
                 llvmcall($"""

base/compiler/abstractinterpretation.jl

@@ -187,6 +187,7 @@ function abstract_call_method(method::Method, @nospecialize(sig), sparams::Simpl
     cyclei = 0
     infstate = sv
     edgecycle = false
+    method2 = method_for_inference_heuristics(method, sig, sparams, sv.params.world) # Union{Method, Nothing}
     while !(infstate === nothing)
         infstate = infstate::InferenceState
         if method === infstate.linfo.def
@@ -197,7 +198,9 @@ function abstract_call_method(method::Method, @nospecialize(sig), sparams::Simpl
                 edgecycle = true
                 break
             end
-            if topmost === nothing
+            inf_method2 = infstate.src.method_for_inference_limit_heuristics # limit only if user token match
+            inf_method2 isa Method || (inf_method2 = nothing) # Union{Method, Nothing}
+            if topmost === nothing && method2 === inf_method2
                 # inspect the parent of this edge,
                 # to see if they are the same Method as sv
                 # in which case we'll need to ensure it is convergent

base/compiler/compiler.jl

@@ -103,6 +103,12 @@ using .Sort
 
 inlining_enabled() = (JLOptions().can_inline == 1)
 coverage_enabled() = (JLOptions().code_coverage != 0)
+function inbounds_option()
+    opt_check_bounds = JLOptions().check_bounds
+    opt_check_bounds == 0 && return :default
+    opt_check_bounds == 1 && return :on
+    return :off
+end
 
 include("compiler/utilities.jl")
 include("compiler/validation.jl")

base/compiler/optimize.jl

@@ -218,6 +218,9 @@ const SLOT_USEDUNDEF    = 32 # slot has uses that might raise UndefVarError
 
 # const SLOT_CALLED      = 64
 
+const IR_FLAG_INBOUNDS = 0x01
+
+
 # known affect-free calls (also effect-free)
 const _PURE_BUILTINS = Any[tuple, svec, fieldtype, apply_type, ===, isa, typeof, UnionAll, nfields]
 
@@ -263,7 +266,7 @@ function isinlineable(m::Method, src::CodeInfo, mod::Module, params::Params, bon
     return inlineable
 end
 
-const enable_new_optimizer = RefValue(true)
+const enable_new_optimizer = RefValue(false)
 
 # converge the optimization work
 function optimize(me::InferenceState)
@@ -396,7 +399,7 @@ function optimize(me::InferenceState)
         me.src.inlineable = false
     elseif !me.src.inlineable && isa(def, Method)
         bonus = 0
-        if me.bestguess ⊑ Tuple && !isbits(widenconst(me.bestguess))
+        if me.bestguess ⊑ Tuple && !isbitstype(widenconst(me.bestguess))
             bonus = me.params.inline_tupleret_bonus
         end
         me.src.inlineable = isinlineable(def, me.src, me.mod, me.params, bonus)
@@ -706,7 +709,7 @@ end
 # since codegen optimizations of functions like `is` will depend on knowing it
 function widen_slot_type(@nospecialize(ty), untypedload::Bool)
     if isa(ty, DataType)
-        if untypedload || isbits(ty) || isdefined(ty, :instance)
+        if untypedload || isbitstype(ty) || isdefined(ty, :instance)
             return ty
         end
     elseif isa(ty, Union)
@@ -1154,6 +1157,7 @@ function inlineable(@nospecialize(f), @nospecialize(ft), e::Expr, atypes::Vector
                 f === Core.sizeof || f === isdefined ||
                 istopfunction(topmod, f, :typejoin) ||
                 istopfunction(topmod, f, :isbits) ||
+                istopfunction(topmod, f, :isbitstype) ||
                 istopfunction(topmod, f, :promote_type) ||
                 (f === Core.kwfunc && length(argexprs) == 2) ||
                 (is_inlineable_constant(val) &&
@@ -2712,7 +2716,7 @@ function merge_value_ssa!(ctx::AllocOptContext, info, key)
     # There are other cases that we can merge
     # but those require control flow analysis.
     var_has_static_undef(ctx.sv.src, key.first, key.second) && return false
-    local defkey
+    local defkey, deftyp
     for def in info.defs
         # No NewvarNode def for variables that aren't used undefined
         defex = (def.assign::Expr).args[2]
@@ -2735,6 +2739,12 @@ function merge_value_ssa!(ctx::AllocOptContext, info, key)
         else
             return @isdefined(defkey)
         end
+        new_deftyp = exprtype(defex, ctx.sv.src, ctx.sv.mod)
+        if @isdefined(deftyp) && deftyp != new_deftyp
+            return true
+        else
+            deftyp = new_deftyp
+        end
     end
 
     if defkey.second || defkey.first > ctx.sv.nargs
@@ -2759,8 +2769,18 @@ function merge_value_ssa!(ctx::AllocOptContext, info, key)
     if defkey.second
         # SSAValue def
         replace_v = SSAValue(defkey.first - 1)
+        # don't replace TypedSlots with SSAValues
+        replace_typ = exprtype(replace_v, ctx.sv.src, ctx.sv.mod)
+        if !(replace_typ ⊑ deftyp)
+            return false
+        end
     else
         replace_v = SlotNumber(defkey.first)
+        # don't lose type information from TypedSlots
+        replace_typ = exprtype(replace_v, ctx.sv.src, ctx.sv.mod)
+        if !(replace_typ ⊑ deftyp)
+            replace_v = TypedSlot(defkey.first, deftyp)
+        end
     end
     for use in info.uses
         if isdefined(use, :expr)
@@ -3077,7 +3097,7 @@ function structinfo_new(ctx::AllocOptContext, ex::Expr, vt::DataType)
             si.defs[i] = ex.args[i + 1]
         else
             ft = fieldtype(vt, i)
-            if isbits(ft)
+            if isbitstype(ft)
                 ex = Expr(:new, ft)
                 ex.typ = ft
                 si.defs[i] = ex
@@ -3614,7 +3634,7 @@ function split_struct_alloc_single!(ctx::AllocOptContext, info, key, nf, has_pre
         if !@isdefined(fld_name)
             fld_name = :struct_field
         end
-        need_preserved_root = has_preserve && !isbits(field_typ)
+        need_preserved_root = has_preserve && !isbitstype(field_typ)
         local var_slot
         if !has_def
             # If there's no direct use of the field