Skip to content

Parser fix: call fclose #988

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Oct 29, 2015
Merged

Parser fix: call fclose #988

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
bf8e0fc
Parser fix: call fclose
kmod Oct 27, 2015
cee9765
Support inherited flags in the cpython parser
kmod Oct 28, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 27 additions & 17 deletions src/codegen/parser.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1018,13 +1018,26 @@ static std::string getParserCommandLine(const char* fn) {
AST_Module* parse_string(const char* code, FutureFlags inherited_flags) {
inherited_flags &= ~(CO_NESTED | CO_FUTURE_DIVISION);

if (ENABLE_CPYTHON_PARSER) {
PyCompilerFlags cf;
cf.cf_flags = inherited_flags;
ArenaWrapper arena;
assert(arena);
const char* fn = "<string>";
mod_ty mod = PyParser_ASTFromString(code, fn, Py_file_input, &cf, arena);
if (!mod)
throwCAPIException();
auto rtn = cpythonToPystonAST(mod, fn);
return rtn;
}

if (ENABLE_PYPA_PARSER || inherited_flags) {
AST_Module* rtn = pypa_parse_string(code, inherited_flags);
RELEASE_ASSERT(rtn, "unknown parse error (possibly: '%s'?)", strerror(errno));
return rtn;
}

ASSERT(!inherited_flags, "the old cpython parser doesn't support specifying initial future flags");
RELEASE_ASSERT(!inherited_flags, "the old cpython parser doesn't support specifying initial future flags");

int size = strlen(code);
char buf[] = "pystontmp_XXXXXX";
Expand All @@ -1035,10 +1048,11 @@ AST_Module* parse_string(const char* code, FutureFlags inherited_flags) {
printf("writing %d bytes to %s\n", size, tmp.c_str());
}

FILE* f = fopen(tmp.c_str(), "w");
fwrite(code, 1, size, f);
fputc('\n', f);
fclose(f);
{
FileHandle f(tmp.c_str(), "w");
fwrite(code, 1, size, f);
fputc('\n', f);
}

AST_Module* m = parse_file(tmp.c_str(), inherited_flags);
removeDirectoryIfExists(tmpdir);
Expand All @@ -1050,10 +1064,9 @@ AST_Module* parse_file(const char* fn, FutureFlags inherited_flags) {
Timer _t("parsing");

if (ENABLE_CPYTHON_PARSER) {
ASSERT(!inherited_flags, "unimplemented");
FILE* fp = fopen(fn, "r");
FileHandle fp(fn, "r");
PyCompilerFlags cf;
cf.cf_flags = 0;
cf.cf_flags = inherited_flags;
ArenaWrapper arena;
assert(arena);
mod_ty mod = PyParser_ASTFromFile(fp, fn, Py_file_input, 0, 0, &cf, NULL, arena);
Expand Down Expand Up @@ -1106,7 +1119,7 @@ const char* getMagic() {
static std::vector<char> _reparse(const char* fn, const std::string& cache_fn, AST_Module*& module,
FutureFlags inherited_flags) {
inherited_flags &= ~(CO_NESTED | CO_FUTURE_DIVISION);
FILE* cache_fp = fopen(cache_fn.c_str(), "w");
FileHandle cache_fp(cache_fn.c_str(), "w");

if (DEBUG_PARSING) {
fprintf(stderr, "_reparse('%s', '%s'), pypa=%d\n", fn, cache_fn.c_str(), ENABLE_PYPA_PARSER);
Expand Down Expand Up @@ -1137,10 +1150,9 @@ static std::vector<char> _reparse(const char* fn, const std::string& cache_fn, A

if (ENABLE_CPYTHON_PARSER || ENABLE_PYPA_PARSER || inherited_flags) {
if (ENABLE_CPYTHON_PARSER) {
ASSERT(!inherited_flags, "unimplemented");
FILE* fp = fopen(fn, "r");
FileHandle fp(fn, "r");
PyCompilerFlags cf;
cf.cf_flags = 0;
cf.cf_flags = inherited_flags;
ArenaWrapper arena;
assert(arena);
mod_ty mod = PyParser_ASTFromFile(fp, fn, Py_file_input, 0, 0, &cf, NULL, arena);
Expand All @@ -1159,7 +1171,7 @@ static std::vector<char> _reparse(const char* fn, const std::string& cache_fn, A
checksum = p.second;
bytes_written += p.first;
} else {
ASSERT(!inherited_flags, "the old cpython parser doesn't support specifying initial future flags");
RELEASE_ASSERT(!inherited_flags, "the old cpython parser doesn't support specifying initial future flags");
FILE* parser = popen(getParserCommandLine(fn).c_str(), "r");
char buf[80];
while (true) {
Expand Down Expand Up @@ -1188,8 +1200,6 @@ static std::vector<char> _reparse(const char* fn, const std::string& cache_fn, A
fwrite(&checksum, 1, CHECKSUM_LENGTH, cache_fp);
memcpy(&file_data[checksum_start + LENGTH_LENGTH], &checksum, CHECKSUM_LENGTH);

if (cache_fp)
fclose(cache_fp);
return std::move(file_data);
}

Expand Down Expand Up @@ -1221,7 +1231,8 @@ AST_Module* caching_parse_file(const char* fn, FutureFlags inherited_flags) {
&& cache_stat.st_mtim.tv_nsec > source_stat.st_mtim.tv_nsec))) {
oss << "reading pyc file\n";
char buf[1024];
FILE* cache_fp = fopen(cache_fn.c_str(), "r");

FileHandle cache_fp(cache_fn.c_str(), "r");
if (cache_fp) {
while (true) {
int read = fread(buf, 1, 1024, cache_fp);
Expand All @@ -1239,7 +1250,6 @@ AST_Module* caching_parse_file(const char* fn, FutureFlags inherited_flags) {
break;
}
}
fclose(cache_fp);
}
}

Expand Down
14 changes: 14 additions & 0 deletions src/core/types.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -803,6 +803,20 @@ class ArenaWrapper {
operator PyArena*() const { return arena; }
};

class FileHandle {
private:
FILE* file;

public:
FileHandle(const char* fn, const char* mode) : file(fopen(fn, mode)) {}
~FileHandle() {
if (file)
fclose(file);
}

operator FILE*() const { return file; }
};

// similar to Java's Array.binarySearch:
// return values are either:
// >= 0 : the index where a given item was found
Expand Down
25 changes: 25 additions & 0 deletions src/runtime/capi.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1367,6 +1367,31 @@ static void err_input(perrdetail* err) noexcept {
#endif

extern "C" grammar _PyParser_Grammar;

/* Preferred access to parser is through AST. */
extern "C" mod_ty PyParser_ASTFromString(const char* s, const char* filename, int start, PyCompilerFlags* flags,
PyArena* arena) noexcept {
mod_ty mod;
PyCompilerFlags localflags;
perrdetail err;
int iflags = PARSER_FLAGS(flags);

node* n = PyParser_ParseStringFlagsFilenameEx(s, filename, &_PyParser_Grammar, start, &err, &iflags);
if (flags == NULL) {
localflags.cf_flags = 0;
flags = &localflags;
}
if (n) {
flags->cf_flags |= iflags & PyCF_MASK;
mod = PyAST_FromNode(n, flags, filename, arena);
PyNode_Free(n);
return mod;
} else {
err_input(&err);
return NULL;
}
}

extern "C" mod_ty PyParser_ASTFromFile(FILE* fp, const char* filename, int start, char* ps1, char* ps2,
PyCompilerFlags* flags, int* errcode, PyArena* arena) noexcept {
mod_ty mod;
Expand Down