diff --git a/src/codegen/parser.cpp b/src/codegen/parser.cpp index e1da3779b..2d7459bc8 100644 --- a/src/codegen/parser.cpp +++ b/src/codegen/parser.cpp @@ -1018,13 +1018,26 @@ static std::string getParserCommandLine(const char* fn) { AST_Module* parse_string(const char* code, FutureFlags inherited_flags) { inherited_flags &= ~(CO_NESTED | CO_FUTURE_DIVISION); + if (ENABLE_CPYTHON_PARSER) { + PyCompilerFlags cf; + cf.cf_flags = inherited_flags; + ArenaWrapper arena; + assert(arena); + const char* fn = ""; + mod_ty mod = PyParser_ASTFromString(code, fn, Py_file_input, &cf, arena); + if (!mod) + throwCAPIException(); + auto rtn = cpythonToPystonAST(mod, fn); + return rtn; + } + if (ENABLE_PYPA_PARSER || inherited_flags) { AST_Module* rtn = pypa_parse_string(code, inherited_flags); RELEASE_ASSERT(rtn, "unknown parse error (possibly: '%s'?)", strerror(errno)); return rtn; } - ASSERT(!inherited_flags, "the old cpython parser doesn't support specifying initial future flags"); + RELEASE_ASSERT(!inherited_flags, "the old cpython parser doesn't support specifying initial future flags"); int size = strlen(code); char buf[] = "pystontmp_XXXXXX"; @@ -1035,10 +1048,11 @@ AST_Module* parse_string(const char* code, FutureFlags inherited_flags) { printf("writing %d bytes to %s\n", size, tmp.c_str()); } - FILE* f = fopen(tmp.c_str(), "w"); - fwrite(code, 1, size, f); - fputc('\n', f); - fclose(f); + { + FileHandle f(tmp.c_str(), "w"); + fwrite(code, 1, size, f); + fputc('\n', f); + } AST_Module* m = parse_file(tmp.c_str(), inherited_flags); removeDirectoryIfExists(tmpdir); @@ -1050,10 +1064,9 @@ AST_Module* parse_file(const char* fn, FutureFlags inherited_flags) { Timer _t("parsing"); if (ENABLE_CPYTHON_PARSER) { - ASSERT(!inherited_flags, "unimplemented"); - FILE* fp = fopen(fn, "r"); + FileHandle fp(fn, "r"); PyCompilerFlags cf; - cf.cf_flags = 0; + cf.cf_flags = inherited_flags; ArenaWrapper arena; assert(arena); mod_ty mod = PyParser_ASTFromFile(fp, fn, Py_file_input, 0, 0, &cf, NULL, arena); @@ -1106,7 +1119,7 @@ const char* getMagic() { static std::vector _reparse(const char* fn, const std::string& cache_fn, AST_Module*& module, FutureFlags inherited_flags) { inherited_flags &= ~(CO_NESTED | CO_FUTURE_DIVISION); - FILE* cache_fp = fopen(cache_fn.c_str(), "w"); + FileHandle cache_fp(cache_fn.c_str(), "w"); if (DEBUG_PARSING) { fprintf(stderr, "_reparse('%s', '%s'), pypa=%d\n", fn, cache_fn.c_str(), ENABLE_PYPA_PARSER); @@ -1137,10 +1150,9 @@ static std::vector _reparse(const char* fn, const std::string& cache_fn, A if (ENABLE_CPYTHON_PARSER || ENABLE_PYPA_PARSER || inherited_flags) { if (ENABLE_CPYTHON_PARSER) { - ASSERT(!inherited_flags, "unimplemented"); - FILE* fp = fopen(fn, "r"); + FileHandle fp(fn, "r"); PyCompilerFlags cf; - cf.cf_flags = 0; + cf.cf_flags = inherited_flags; ArenaWrapper arena; assert(arena); mod_ty mod = PyParser_ASTFromFile(fp, fn, Py_file_input, 0, 0, &cf, NULL, arena); @@ -1159,7 +1171,7 @@ static std::vector _reparse(const char* fn, const std::string& cache_fn, A checksum = p.second; bytes_written += p.first; } else { - ASSERT(!inherited_flags, "the old cpython parser doesn't support specifying initial future flags"); + RELEASE_ASSERT(!inherited_flags, "the old cpython parser doesn't support specifying initial future flags"); FILE* parser = popen(getParserCommandLine(fn).c_str(), "r"); char buf[80]; while (true) { @@ -1188,8 +1200,6 @@ static std::vector _reparse(const char* fn, const std::string& cache_fn, A fwrite(&checksum, 1, CHECKSUM_LENGTH, cache_fp); memcpy(&file_data[checksum_start + LENGTH_LENGTH], &checksum, CHECKSUM_LENGTH); - if (cache_fp) - fclose(cache_fp); return std::move(file_data); } @@ -1221,7 +1231,8 @@ AST_Module* caching_parse_file(const char* fn, FutureFlags inherited_flags) { && cache_stat.st_mtim.tv_nsec > source_stat.st_mtim.tv_nsec))) { oss << "reading pyc file\n"; char buf[1024]; - FILE* cache_fp = fopen(cache_fn.c_str(), "r"); + + FileHandle cache_fp(cache_fn.c_str(), "r"); if (cache_fp) { while (true) { int read = fread(buf, 1, 1024, cache_fp); @@ -1239,7 +1250,6 @@ AST_Module* caching_parse_file(const char* fn, FutureFlags inherited_flags) { break; } } - fclose(cache_fp); } } diff --git a/src/core/types.h b/src/core/types.h index f3ad50006..f81f77b4a 100644 --- a/src/core/types.h +++ b/src/core/types.h @@ -803,6 +803,20 @@ class ArenaWrapper { operator PyArena*() const { return arena; } }; +class FileHandle { +private: + FILE* file; + +public: + FileHandle(const char* fn, const char* mode) : file(fopen(fn, mode)) {} + ~FileHandle() { + if (file) + fclose(file); + } + + operator FILE*() const { return file; } +}; + // similar to Java's Array.binarySearch: // return values are either: // >= 0 : the index where a given item was found diff --git a/src/runtime/capi.cpp b/src/runtime/capi.cpp index 6405022cd..5264847e9 100644 --- a/src/runtime/capi.cpp +++ b/src/runtime/capi.cpp @@ -1367,6 +1367,31 @@ static void err_input(perrdetail* err) noexcept { #endif extern "C" grammar _PyParser_Grammar; + +/* Preferred access to parser is through AST. */ +extern "C" mod_ty PyParser_ASTFromString(const char* s, const char* filename, int start, PyCompilerFlags* flags, + PyArena* arena) noexcept { + mod_ty mod; + PyCompilerFlags localflags; + perrdetail err; + int iflags = PARSER_FLAGS(flags); + + node* n = PyParser_ParseStringFlagsFilenameEx(s, filename, &_PyParser_Grammar, start, &err, &iflags); + if (flags == NULL) { + localflags.cf_flags = 0; + flags = &localflags; + } + if (n) { + flags->cf_flags |= iflags & PyCF_MASK; + mod = PyAST_FromNode(n, flags, filename, arena); + PyNode_Free(n); + return mod; + } else { + err_input(&err); + return NULL; + } +} + extern "C" mod_ty PyParser_ASTFromFile(FILE* fp, const char* filename, int start, char* ps1, char* ps2, PyCompilerFlags* flags, int* errcode, PyArena* arena) noexcept { mod_ty mod;