Replaceable jl_parse as single parser entry point

* Unify flisp parsing to always parse from string rather than file.
* Put all code-loading functions together into toplevel.c
* Remove flisp code from jl_parse_eval_all. This makes the top level
  parse-lower-eval loop independent of flisp internals.
* Keep jl_load and jl_load_file_string from julia.h unchanged for
  compatibility with the existing C API. (No julia code `ccall`s
  these anymore)
* Replaceable jl_parse as single parser entry point. `jl_parse` calls a
  C function pointer which can be replaced with jl_set_parser(). This
  makes it possible to replace all use of the flisp parser with julia
  code at runtime. Create frontend.c to contain the shim code for this
  and as a place to move non-flisp related frontend code in the
  future.

....

Move include() implementation to Julia; add Meta.parseall()

Move the implementation include into Julia, essentially replacing
jl_parse_eval_all (except for within bootstrap).

This also reunifies the versions of include() in MainInclude and Base
which have started diverging after being duplicated for improved stack
traces.

As part of this, add `Meta.parseall()` for top-level parsing. For now as
an internal function, but this will likely be made public shortly.

....

Remove include mapexpr handling from C code

mapexpr is now handled in Julia code so we can use a simplified version
of include during bootstrap.

....

General parser options for improved flexibility

This change allows general options to be passed through to the parser.

....

Apply suggestions from code review

Co-Authored-By: Jameson Nash <[email protected]>

....

Move code back into ast.c from frontend.c.

....

Julia ABI and implementation for swappable parser

This moves the parser-swapping implementation almost entirely into
Julia, leaving only a few generic pieces in C and some code which is
required to call into the Julia parser.

* Add Core.Compiler.parse as the internal entry point for parsing, and
  move parser-swapping machinery there.
* New builtin Core._apply_in_world to allow frozen-world APIs to be
  implemented in pure Julia code.
* Make filename a String for convenience (no big deal if it needs to be
  copied, compared to copying the source text itself).
* Rename some variables in parser APIs (particularly pos vs offset) for
  consistency.

....

Use latest world for calling Core.Compiler.parse (fixup! Julia ABI and implementation for swappable parser)

Restoring the world age this way doesn't seem entirely consistent with
the dynamically scoped way it's used elsewhere.

....

More flexibility in freezing parser world age

This separates the decision about whether to freeze the parser world age
from Core.Compiler.parse. The builtin _apply_in_world allows this to be
done with more flexibility from outside (for example, one might want to
do some argument conversion in the current world before calling through
to a fixed world for the parser internals.

....

Apply suggestions from code review

Co-authored-by: Jameson Nash <[email protected]>

....

Put replaceable parser shim in Core

Avoids introducing (more?) dependency between Base and Core.Compiler.

....

Simplify Core._parse

Author: c42f

base/Base.jl

@@ -365,31 +365,8 @@ for m in methods(include)
 end
 # These functions are duplicated in client.jl/include(::String) for
 # nicer stacktraces. Modifications here have to be backported there
-include(mod::Module, _path::AbstractString) = include(identity, mod, _path)
-function include(mapexpr::Function, mod::Module, _path::AbstractString)
-    path, prev = _include_dependency(mod, _path)
-    for callback in include_callbacks # to preserve order, must come before Core.include
-        invokelatest(callback, mod, path)
-    end
-    tls = task_local_storage()
-    tls[:SOURCE_PATH] = path
-    local result
-    try
-        # result = Core.include(mod, path)
-        if mapexpr === identity
-            result = ccall(:jl_load, Any, (Any, Cstring), mod, path)
-        else
-            result = ccall(:jl_load_rewrite, Any, (Any, Cstring, Any), mod, path, mapexpr)
-        end
-    finally
-        if prev === nothing
-            delete!(tls, :SOURCE_PATH)
-        else
-            tls[:SOURCE_PATH] = prev
-        end
-    end
-    return result
-end
+include(mod::Module, _path::AbstractString) = _include(identity, mod, _path)
+include(mapexpr::Function, mod::Module, _path::AbstractString) = _include(mapexpr, mod, _path)
 
 end_base_include = time_ns()
 

base/boot.jl

@@ -740,4 +740,19 @@ Unsigned(x::Union{Float32, Float64, Bool}) = UInt(x)
 Integer(x::Integer) = x
 Integer(x::Union{Float32, Float64}) = Int(x)
 
+# Binding for the julia parser, called as
+#
+#    Core._parse(text, filename, offset, options)
+#
+# Parse Julia code from the buffer `text`, starting at `offset` and attributing
+# it to `filename`. `text` may be a `String` or `svec(ptr::Ptr{UInt8},
+# len::Int)` for a raw unmanaged buffer. `options` should be one of `:atom`,
+# `:statement` or `:all`, indicating how much the parser will consume.
+#
+# `_parse` must return an `svec` containing an `Expr` and the new offset as an
+# `Int`.
+#
+# The internal jl_parse which will call into Core._parse if not `nothing`.
+eval(Core, :(_parse = nothing))
+
 ccall(:jl_set_istopmod, Cvoid, (Any, Bool), Core, true)

base/client.jl

@@ -154,8 +154,7 @@ function eval_user_input(errio, @nospecialize(ast), show_value::Bool)
 end
 
 function _parse_input_line_core(s::String, filename::String)
-    ex = ccall(:jl_parse_all, Any, (Ptr{UInt8}, Csize_t, Ptr{UInt8}, Csize_t),
-               s, sizeof(s), filename, sizeof(filename))
+    ex = Meta.parseall(s, filename=filename)
     if ex isa Expr && ex.head === :toplevel
         if isempty(ex.args)
             return nothing
@@ -439,30 +438,12 @@ end
 # MainInclude exists to hide Main.include and eval from `names(Main)`.
 baremodule MainInclude
 using ..Base
-include(mapexpr::Function, fname::AbstractString) = Base.include(mapexpr, Main, fname)
-# We inline the definition of include from loading.jl/include_relative to get one-frame stacktraces
-# for the common case of include(fname).  Otherwise we would use:
-#    include(fname::AbstractString) = Base.include(Main, fname)
+# These definitions calls Base._include rather than Base.include to get
+# one-frame stacktraces for the common case of using include(fname) in Main.
+include(mapexpr::Function, fname::AbstractString) = Base._include(mapexpr, Main, fname)
 function include(fname::AbstractString)
-    mod = Main
     isa(fname, String) || (fname = Base.convert(String, fname)::String)
-    path, prev = Base._include_dependency(mod, fname)
-    for callback in Base.include_callbacks # to preserve order, must come before Core.include
-        Base.invokelatest(callback, mod, path)
-    end
-    tls = Base.task_local_storage()
-    tls[:SOURCE_PATH] = path
-    local result
-    try
-        result = ccall(:jl_load, Any, (Any, Cstring), mod, path)
-    finally
-        if prev === nothing
-            Base.delete!(tls, :SOURCE_PATH)
-        else
-            tls[:SOURCE_PATH] = prev
-        end
-    end
-    return result
+    Base._include(identity, Main, fname)
 end
 eval(x) = Core.eval(Main, x)
 end

base/compiler/compiler.jl

@@ -113,5 +113,9 @@ include("compiler/optimize.jl") # TODO: break this up further + extract utilitie
 include("compiler/bootstrap.jl")
 ccall(:jl_set_typeinf_func, Cvoid, (Any,), typeinf_ext_toplevel)
 
+include("compiler/parsing.jl")
+Core.eval(Core, :(_parse = Compiler.fl_parse))
+
 end # baremodule Compiler
 ))
+

base/compiler/parsing.jl

@@ -0,0 +1,19 @@
+# This file is a part of Julia. License is MIT: https://julialang.org/license
+
+# Call Julia's builtin flisp-based parser. `offset` is 0-based offset into the
+# byte buffer or string.
+function fl_parse(text::Union{Core.SimpleVector,String},
+                  filename::String, offset, options)
+    if text isa Core.SimpleVector
+        # Will be generated by C entry points jl_parse_string etc
+        text, text_len = text
+    else
+        text_len = sizeof(text)
+    end
+    ccall(:jl_fl_parse, Any, (Ptr{UInt8}, Csize_t, Any, Csize_t, Any),
+          text, text_len, filename, offset, options)
+end
+
+function fl_parse(text::AbstractString, filename::AbstractString, offset, options)
+    fl_parse(String(text), String(filename), offset, options)
+end

base/loading.jl

@@ -1082,14 +1082,31 @@ The optional first argument `mapexpr` can be used to transform the included code
 it is evaluated: for each parsed expression `expr` in `code`, the `include_string` function
 actually evaluates `mapexpr(expr)`.  If it is omitted, `mapexpr` defaults to [`identity`](@ref).
 """
-function include_string(mapexpr::Function, m::Module, txt_::AbstractString, fname::AbstractString="string")
-    txt = String(txt_)
-    if mapexpr === identity
-        ccall(:jl_load_file_string, Any, (Ptr{UInt8}, Csize_t, Cstring, Any),
-            txt, sizeof(txt), String(fname), m)
-    else
-        ccall(:jl_load_rewrite_file_string, Any, (Ptr{UInt8}, Csize_t, Cstring, Any, Any),
-            txt, sizeof(txt), String(fname), m, mapexpr)
+function include_string(mapexpr::Function, mod::Module, code::AbstractString,
+                        filename::AbstractString="string")
+    loc = LineNumberNode(1, Symbol(filename))
+    try
+        ast = Meta.parseall(code, filename=filename)
+        @assert Meta.isexpr(ast, :toplevel)
+        result = nothing
+        line_and_ex = Expr(:toplevel, loc, nothing)
+        for ex in ast.args
+            if ex isa LineNumberNode
+                loc = ex
+                line_and_ex.args[1] = ex
+                continue
+            end
+            ex = mapexpr(ex)
+            # Wrap things to be eval'd in a :toplevel expr to carry line
+            # information as part of the expr.
+            line_and_ex.args[2] = ex
+            result = Core.eval(mod, line_and_ex)
+        end
+        return result
+    catch exc
+        # TODO: Now that stacktraces are more reliable we should remove
+        # LoadError and expose the real error type directly.
+        rethrow(LoadError(filename, loc.line, exc))
     end
 end
 
@@ -1124,7 +1141,28 @@ The optional first argument `mapexpr` can be used to transform the included code
 it is evaluated: for each parsed expression `expr` in `path`, the `include` function
 actually evaluates `mapexpr(expr)`.  If it is omitted, `mapexpr` defaults to [`identity`](@ref).
 """
-Base.include # defined in sysimg.jl
+Base.include # defined in Base.jl
+
+# Full include() implementation which is used after bootstrap
+function _include(mapexpr::Function, mod::Module, _path::AbstractString)
+    @_noinline_meta # Workaround for module availability in _simplify_include_frames
+    path, prev = _include_dependency(mod, _path)
+    for callback in include_callbacks # to preserve order, must come before eval in include_string
+        invokelatest(callback, mod, path)
+    end
+    code = read(path, String)
+    tls = task_local_storage()
+    tls[:SOURCE_PATH] = path
+    try
+        return include_string(mapexpr, mod, code, path)
+    finally
+        if prev === nothing
+            delete!(tls, :SOURCE_PATH)
+        else
+            tls[:SOURCE_PATH] = prev
+        end
+    end
+end
 
 """
     evalfile(path::AbstractString, args::Vector{String}=String[])

base/meta.jl

@@ -149,6 +149,15 @@ struct ParseError <: Exception
     msg::AbstractString
 end
 
+function _parse_string(text::AbstractString, filename::AbstractString,
+                       index::Integer, options)
+    if index < 1 || index > ncodeunits(text) + 1
+        throw(BoundsError(text, index))
+    end
+    ex, offset = Core._parse(text, filename, index-1, options)
+    ex, offset+1
+end
+
 """
     parse(str, start; greedy=true, raise=true, depwarn=true)
 
@@ -171,19 +180,11 @@ julia> Meta.parse("x = 3, y = 5", 5)
 """
 function parse(str::AbstractString, pos::Integer; greedy::Bool=true, raise::Bool=true,
                depwarn::Bool=true)
-    # pos is one based byte offset.
-    # returns (expr, end_pos). expr is () in case of parse error.
-    bstr = String(str)
-    # For now, assume all parser warnings are depwarns
-    ex, pos = with_logger(depwarn ? current_logger() : NullLogger()) do
-        ccall(:jl_parse_string, Any,
-              (Ptr{UInt8}, Csize_t, Int32, Int32),
-              bstr, sizeof(bstr), pos-1, greedy ? 1 : 0)
-    end
+    ex, pos = _parse_string(str, "none", pos, greedy ? :statement : :atom)
     if raise && isa(ex,Expr) && ex.head === :error
         throw(ParseError(ex.args[1]))
     end
-    return ex, pos+1 # C is zero-based, Julia is 1-based
+    return ex, pos
 end
 
 """
@@ -223,6 +224,15 @@ function parse(str::AbstractString; raise::Bool=true, depwarn::Bool=true)
     return ex
 end
 
+function parseatom(text::AbstractString, pos::Integer; filename="none")
+    return _parse_string(text, filename, pos, :atom)
+end
+
+function parseall(text::AbstractString; filename="none")
+    ex,_ = _parse_string(text, filename, 1, :all)
+    return ex
+end
+
 """
     partially_inline!(code::Vector{Any}, slot_replacements::Vector{Any},
                       type_signature::Type{<:Tuple}, static_param_values::Vector{Any},