diff --git a/from_cpython/Include/Python.h b/from_cpython/Include/Python.h index 87132fd12..ea600d09b 100644 --- a/from_cpython/Include/Python.h +++ b/from_cpython/Include/Python.h @@ -134,6 +134,13 @@ PyObject* PyGC_AddRoot(PyObject*) PYSTON_NOEXCEPT; // PyGC_AddRoot(obj); // to reduce any chances of compiler reorderings or a GC somehow happening between the assignment // to the static slot and the call to PyGC_AddRoot. +// + +// Similarly, some extension modules have static non-type objects that our GC thinks are errors +// since they are not in the heap, unless we root them. This function should be used purely as +// a temporary patching mechanism - we want to have a better way of dealing with such objects +// in the future (even if it's an invalid use of CPython APIs). +PyObject* PyGC_AddNonHeapRoot(PyObject* obj, int size) PYSTON_NOEXCEPT; // Pyston change : expose these type objects extern PyTypeObject Pattern_Type; diff --git a/from_cpython/Include/pyconfig.h.in b/from_cpython/Include/pyconfig.h.in index bb58b3375..160c33ff0 100644 --- a/from_cpython/Include/pyconfig.h.in +++ b/from_cpython/Include/pyconfig.h.in @@ -23,6 +23,7 @@ #define SIZEOF__BOOL 1 #define SIZEOF_VOID_P 8 #define SIZEOF_SIZE_T 8 +#define SIZEOF_SHORT 2 #define SIZEOF_INT 4 #define SIZEOF_LONG 8 #define SIZEOF_LONG_LONG 8 diff --git a/src/analysis/scoping_analysis.cpp b/src/analysis/scoping_analysis.cpp index a92be9fa2..e46df4f1c 100644 --- a/src/analysis/scoping_analysis.cpp +++ b/src/analysis/scoping_analysis.cpp @@ -524,6 +524,7 @@ class NameCollectorVisitor : public ASTVisitor { // bool visit_classdef(AST_ClassDef *node) override { return false; } bool visit_continue(AST_Continue* node) override { return false; } bool visit_dict(AST_Dict* node) override { return false; } + bool visit_ellipsis(AST_Ellipsis* node) override { return false; } bool visit_excepthandler(AST_ExceptHandler* node) override { return false; } bool visit_expr(AST_Expr* node) override { return false; } bool visit_extslice(AST_ExtSlice* node) override { return false; } diff --git a/src/capi/typeobject.cpp b/src/capi/typeobject.cpp index 065d03e01..38a10e3aa 100644 --- a/src/capi/typeobject.cpp +++ b/src/capi/typeobject.cpp @@ -3247,7 +3247,6 @@ extern "C" int PyType_Ready(PyTypeObject* cls) noexcept { | Py_TPFLAGS_DICT_SUBCLASS | Py_TPFLAGS_BASE_EXC_SUBCLASS | Py_TPFLAGS_TYPE_SUBCLASS; RELEASE_ASSERT((cls->tp_flags & ~ALLOWABLE_FLAGS) == 0, ""); - RELEASE_ASSERT(cls->tp_free == NULL || cls->tp_free == PyObject_Del || cls->tp_free == PyObject_GC_Del, ""); RELEASE_ASSERT(cls->tp_is_gc == NULL, ""); RELEASE_ASSERT(cls->tp_mro == NULL, ""); RELEASE_ASSERT(cls->tp_cache == NULL, ""); @@ -3278,6 +3277,19 @@ extern "C" int PyType_Ready(PyTypeObject* cls) noexcept { assert(cls->tp_name); + // Inherit some special protocols. Normally methods are automatically inherited, + // when a Python class is declared, but that may not be the case with C extensions. + if (base != NULL) { + if (cls->tp_as_number == NULL) + cls->tp_as_number = base->tp_as_number; + if (cls->tp_as_sequence == NULL) + cls->tp_as_sequence = base->tp_as_sequence; + if (cls->tp_as_mapping == NULL) + cls->tp_as_mapping = base->tp_as_mapping; + if (cls->tp_as_buffer == NULL) + cls->tp_as_buffer = base->tp_as_buffer; + } + if (cls->tp_call) { cls->tpp_call.capi_val = tppProxyToTpCall; cls->tpp_call.cxx_val = tppProxyToTpCall; diff --git a/src/codegen/parser.cpp b/src/codegen/parser.cpp index e7646b17c..5fcee55dd 100644 --- a/src/codegen/parser.cpp +++ b/src/codegen/parser.cpp @@ -384,6 +384,12 @@ AST_DictComp* read_dictcomp(BufferedReader* reader) { return rtn; } +AST_Ellipsis* read_ellipsis(BufferedReader* reader) { + AST_Ellipsis* rtn = new AST_Ellipsis(); + rtn->col_offset = -1; + rtn->lineno = -1; + return rtn; +} AST_ExceptHandler* read_excepthandler(BufferedReader* reader) { AST_ExceptHandler* rtn = new AST_ExceptHandler(); @@ -822,6 +828,8 @@ AST_expr* readASTExpr(BufferedReader* reader) { return read_dict(reader); case AST_TYPE::DictComp: return read_dictcomp(reader); + case AST_TYPE::Ellipsis: + return read_ellipsis(reader); case AST_TYPE::ExtSlice: return read_extslice(reader); case AST_TYPE::GeneratorExp: diff --git a/src/codegen/serialize_ast.cpp b/src/codegen/serialize_ast.cpp index f0de2ba6a..351f87eb1 100644 --- a/src/codegen/serialize_ast.cpp +++ b/src/codegen/serialize_ast.cpp @@ -277,6 +277,7 @@ class SerializeASTVisitor : public ASTVisitor { writeExpr(node->value); return true; } + virtual bool visit_ellipsis(AST_Ellipsis* node) { return true; } virtual bool visit_excepthandler(AST_ExceptHandler* node) { writeStmtVector(node->body); writeColOffset(node->col_offset); diff --git a/src/core/ast.cpp b/src/core/ast.cpp index 2a7ee7177..92c466b7d 100644 --- a/src/core/ast.cpp +++ b/src/core/ast.cpp @@ -2034,6 +2034,10 @@ class FlattenVisitor : public ASTVisitor { output->push_back(node); return false; } + virtual bool visit_ellipsis(AST_Ellipsis* node) { + output->push_back(node); + return false; + } virtual bool visit_excepthandler(AST_ExceptHandler* node) { output->push_back(node); return false; diff --git a/src/core/cfg.cpp b/src/core/cfg.cpp index 0c0b4de2e..58e3dd4eb 100644 --- a/src/core/cfg.cpp +++ b/src/core/cfg.cpp @@ -855,6 +855,8 @@ class CFGVisitor : public ASTVisitor { return rtn; } + AST_expr* remapEllipsis(AST_Ellipsis* node) { return node; } + AST_expr* remapExtSlice(AST_ExtSlice* node) { AST_ExtSlice* rtn = new AST_ExtSlice(); rtn->lineno = node->lineno; @@ -1183,6 +1185,9 @@ class CFGVisitor : public ASTVisitor { case AST_TYPE::DictComp: rtn = remapScopedComprehension(ast_cast(node)); break; + case AST_TYPE::Ellipsis: + rtn = remapEllipsis(ast_cast(node)); + break; case AST_TYPE::ExtSlice: rtn = remapExtSlice(ast_cast(node)); break; diff --git a/src/gc/collector.cpp b/src/gc/collector.cpp index 838e03cd8..ffa5611e7 100644 --- a/src/gc/collector.cpp +++ b/src/gc/collector.cpp @@ -243,6 +243,13 @@ extern "C" PyObject* PyGC_AddRoot(PyObject* obj) noexcept { return obj; } +extern "C" PyObject* PyGC_AddNonHeapRoot(PyObject* obj, int size) noexcept { + if (obj) { + registerNonheapRootObject(obj, size); + } + return obj; +} + void registerNonheapRootObject(void* obj, int size) { // I suppose that things could work fine even if this were true, but why would it happen? assert(global_heap.getAllocationFromInteriorPointer(obj) == NULL); diff --git a/src/runtime/complex.cpp b/src/runtime/complex.cpp index 39d309b42..3f1dc543d 100644 --- a/src/runtime/complex.cpp +++ b/src/runtime/complex.cpp @@ -31,7 +31,19 @@ extern "C" Box* createPureImaginary(double i) { } extern "C" Py_complex PyComplex_AsCComplex(PyObject* op) noexcept { - Py_FatalError("unimplemented"); + // TODO: Incomplete v.s. CPython's implementation. + Py_complex cval; + if (PyComplex_Check(op)) { + cval.real = ((BoxedComplex*)op)->real; + cval.imag = ((BoxedComplex*)op)->imag; + return cval; + } else if (op->cls == int_cls) { + cval.real = ((BoxedInt*)op)->n; + cval.imag = 0.0; + return cval; + } else { + Py_FatalError("unimplemented"); + } } extern "C" double PyComplex_RealAsDouble(PyObject* op) noexcept { @@ -255,6 +267,12 @@ Box* complexHash(BoxedComplex* self) { return boxInt(combined); } +Box* complexAbs(BoxedComplex* self) { + assert(self->cls == complex_cls); + // TODO: CPython does a lot more safety checks. + return boxFloat(sqrt(self->real * self->real + self->imag * self->imag)); +} + Box* complexStr(BoxedComplex* self) { assert(self->cls == complex_cls); return boxString(complexFmt(self->real, self->imag, 12, 'g')); @@ -364,6 +382,7 @@ void setupComplex() { complex_cls->giveAttr("__pos__", new BoxedFunction(boxRTFunction((void*)complexPos, BOXED_COMPLEX, 1))); complex_cls->giveAttr("__hash__", new BoxedFunction(boxRTFunction((void*)complexHash, BOXED_INT, 1))); + complex_cls->giveAttr("__abs__", new BoxedFunction(boxRTFunction((void*)complexAbs, BOXED_FLOAT, 1))); complex_cls->giveAttr("__str__", new BoxedFunction(boxRTFunction((void*)complexStr, STR, 1))); complex_cls->giveAttr("__repr__", new BoxedFunction(boxRTFunction((void*)complexRepr, STR, 1))); complex_cls->giveAttr("real", diff --git a/src/runtime/dict.cpp b/src/runtime/dict.cpp index c6e0dbeef..e9e9751e2 100644 --- a/src/runtime/dict.cpp +++ b/src/runtime/dict.cpp @@ -174,6 +174,11 @@ Box* dictViewItems(BoxedDict* self) { return rtn; } +// Analoguous to CPython's, used for sq_ slots. +static Py_ssize_t dict_length(PyDictObject* mp) { + return ((BoxedDict*)mp)->d.size(); +} + Box* dictLen(BoxedDict* self) { if (!isSubclass(self->cls, dict_cls)) raiseExcHelper(TypeError, "descriptor '__len__' requires a 'dict' object but received a '%s'", @@ -374,9 +379,7 @@ extern "C" PyObject* PyDict_GetItemString(PyObject* dict, const char* key) noexc } Box* dictSetitem(BoxedDict* self, Box* k, Box* v) { - // printf("Starting setitem\n"); Box*& pos = self->d[k]; - // printf("Got the pos\n"); if (pos != NULL) { pos = v; @@ -402,6 +405,22 @@ Box* dictDelitem(BoxedDict* self, Box* k) { return None; } +// Analoguous to CPython's, used for sq_ slots. +static int dict_ass_sub(PyDictObject* mp, PyObject* v, PyObject* w) noexcept { + try { + Box* res; + if (w == NULL) { + res = dictDelitem((BoxedDict*)mp, v); + } else { + res = dictSetitem((BoxedDict*)mp, v, w); + } + assert(res == None); + } catch (ExcInfo e) { + setCAPIException(e); + } + return 0; +} + extern "C" int PyDict_DelItem(PyObject* op, PyObject* key) noexcept { ASSERT(isSubclass(op->cls, dict_cls) || op->cls == attrwrapper_cls, "%s", getTypeName(op)); try { @@ -824,7 +843,11 @@ void setupDict() { dict_cls->tp_init = dict_init; dict_cls->tp_repr = dict_repr; + dict_cls->tp_as_mapping->mp_length = (lenfunc)dict_length; dict_cls->tp_as_mapping->mp_subscript = (binaryfunc)dictGetitem; + dict_cls->tp_as_mapping->mp_ass_subscript = (objobjargproc)dict_ass_sub; + + dict_cls->tp_as_sequence->sq_contains = (objobjproc)PyDict_Contains; dict_keys_cls->giveAttr( "__iter__", new BoxedFunction(boxRTFunction((void*)dictViewKeysIter, typeFromClass(dict_iterator_cls), 1))); diff --git a/src/runtime/import.cpp b/src/runtime/import.cpp index ee8b5ca83..69457db9d 100644 --- a/src/runtime/import.cpp +++ b/src/runtime/import.cpp @@ -809,7 +809,7 @@ BoxedModule* importCExtension(const std::string& full_name, const std::string& l // Let the GC know about the static variables. uintptr_t bss_start = (uintptr_t)dlsym(handle, "__bss_start"); uintptr_t bss_end = (uintptr_t)dlsym(handle, "_end"); - RELEASE_ASSERT(bss_end - bss_start < 100000, "Large BSS section detected - there maybe something wrong"); + RELEASE_ASSERT(bss_end - bss_start < 1000000, "Large BSS section detected - there maybe something wrong"); // only track void* aligned memory bss_start = (bss_start + (sizeof(void*) - 1)) & ~(sizeof(void*) - 1); bss_end -= bss_end % sizeof(void*); diff --git a/src/runtime/list.cpp b/src/runtime/list.cpp index 07be83e7d..bdec64788 100644 --- a/src/runtime/list.cpp +++ b/src/runtime/list.cpp @@ -31,6 +31,8 @@ namespace pyston { +static int list_ass_slice(PyListObject* a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject* v); + extern "C" int PyList_Append(PyObject* op, PyObject* newitem) noexcept { RELEASE_ASSERT(PyList_Check(op), ""); try { @@ -198,7 +200,7 @@ static Box* list_slice(Box* o, Py_ssize_t ilow, Py_ssize_t ihigh) noexcept { return (PyObject*)np; } -extern "C" Box* listGetitemUnboxed(BoxedList* self, int64_t n) { +static inline Box* listGetitemUnboxed(BoxedList* self, int64_t n) { assert(isSubclass(self->cls, list_cls)); if (n < 0) n = self->size + n; @@ -252,6 +254,17 @@ extern "C" Box* listGetslice(BoxedList* self, Box* boxedStart, Box* boxedStop) { return _listSlice(self, start, stop, 1, stop - start); } +// Analoguous to CPython's, used for sq_ slots. +static PyObject* list_item(PyListObject* a, Py_ssize_t i) noexcept { + try { + BoxedList* self = (BoxedList*)a; + return listGetitemUnboxed(self, i); + } catch (ExcInfo e) { + setCAPIException(e); + return NULL; + } +} + template Box* listGetitem(BoxedList* self, Box* slice) { if (S == CAPI) { try { @@ -297,6 +310,22 @@ extern "C" Box* listSetitemInt(BoxedList* self, BoxedInt* slice, Box* v) { return listSetitemUnboxed(self, slice->n, v); } +// Analoguous to CPython's, used for sq_ slots. +static int list_ass_item(PyListObject* a, Py_ssize_t i, PyObject* v) { + PyObject* old_value; + if (i < 0 || i >= Py_SIZE(a)) { + PyErr_SetString(PyExc_IndexError, "list assignment index out of range"); + return -1; + } + if (v == NULL) + return list_ass_slice(a, i, i + 1, v); + Py_INCREF(v); + old_value = a->ob_item[i]; + a->ob_item[i] = v; + Py_DECREF(old_value); + return 0; +} + extern "C" int PyList_SetItem(PyObject* op, Py_ssize_t i, PyObject* newitem) noexcept { PyObject* olditem; PyObject** p; @@ -449,7 +478,7 @@ int list_ass_ext_slice(BoxedList* self, PyObject* item, PyObject* value) { } } -Box* listSetitemSliceInt64(BoxedList* self, i64 start, i64 stop, i64 step, Box* v) { +static inline void listSetitemSliceInt64(BoxedList* self, i64 start, i64 stop, i64 step, Box* v) { RELEASE_ASSERT(step == 1, "step sizes must be 1 in this code path"); boundSliceWithLength(&start, &stop, start, stop, self->size); @@ -488,8 +517,6 @@ Box* listSetitemSliceInt64(BoxedList* self, i64 start, i64 stop, i64 step, Box* } self->size += delts; - - return None; } extern "C" Box* listSetitemSlice(BoxedList* self, BoxedSlice* slice, Box* v) { @@ -511,7 +538,14 @@ extern "C" Box* listSetitemSlice(BoxedList* self, BoxedSlice* slice, Box* v) { return None; } - return listSetitemSliceInt64(self, start, stop, step, v); + listSetitemSliceInt64(self, start, stop, step, v); + return None; +} + +// Analoguous to CPython's, used for sq_ slots. +static int list_ass_slice(PyListObject* a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject* v) { + listSetitemSliceInt64((BoxedList*)a, ilow, ihigh, 1, v); + return 0; } extern "C" Box* listSetslice(BoxedList* self, Box* boxedStart, Box* boxedStop, Box** args) { @@ -522,7 +556,8 @@ extern "C" Box* listSetslice(BoxedList* self, Box* boxedStart, Box* boxedStop, B sliceIndex(boxedStart, &start); sliceIndex(boxedStop, &stop); - return listSetitemSliceInt64(self, start, stop, 1, value); + listSetitemSliceInt64(self, start, stop, 1, value); + return None; } extern "C" Box* listSetitem(BoxedList* self, Box* slice, Box* v) { @@ -822,23 +857,33 @@ extern "C" Box* PyList_GetSlice(PyObject* a, Py_ssize_t ilow, Py_ssize_t ihigh) return listGetitemSlice(self, new BoxedSlice(boxInt(ilow), boxInt(ihigh), boxInt(1))); } -Box* listContains(BoxedList* self, Box* elt) { +static inline int list_contains_shared(BoxedList* self, Box* elt) { + assert(isSubclass(self->cls, list_cls)); + int size = self->size; for (int i = 0; i < size; i++) { Box* e = self->elts->elts[i]; bool identity_eq = e == elt; if (identity_eq) - return True; + return true; int r = PyObject_RichCompareBool(e, elt, Py_EQ); if (r == -1) throwCAPIException(); if (r) - return True; + return true; } - return False; + return false; +} + +static int list_contains(PyListObject* a, PyObject* el) { + return list_contains_shared((BoxedList*)a, el); +} + +Box* listContains(BoxedList* self, Box* elt) { + return boxBool(list_contains_shared(self, elt)); } Box* listCount(BoxedList* self, Box* elt) { @@ -1195,9 +1240,13 @@ void setupList() { list_cls->giveAttr("__hash__", None); list_cls->freeze(); - list_cls->tp_as_sequence->sq_slice = list_slice; - list_cls->tp_as_sequence->sq_length = list_length; list_cls->tp_iter = listIter; + list_cls->tp_as_sequence->sq_length = list_length; + list_cls->tp_as_sequence->sq_item = (ssizeargfunc)list_item; + list_cls->tp_as_sequence->sq_slice = list_slice; + list_cls->tp_as_sequence->sq_ass_item = (ssizeobjargproc)list_ass_item; + list_cls->tp_as_sequence->sq_ass_slice = (ssizessizeobjargproc)list_ass_slice; + list_cls->tp_as_sequence->sq_contains = (objobjproc)list_contains; CLFunction* hasnext = boxRTFunction((void*)listiterHasnextUnboxed, BOOL, 1); addRTFunction(hasnext, (void*)listiterHasnext, BOXED_BOOL); diff --git a/src/runtime/str.cpp b/src/runtime/str.cpp index d8d09780d..9cf64f544 100644 --- a/src/runtime/str.cpp +++ b/src/runtime/str.cpp @@ -1660,6 +1660,7 @@ static Box* str_slice(Box* o, Py_ssize_t i, Py_ssize_t j) { return PyString_FromStringAndSize(a->data() + i, j - i); } +// Analoguous to CPython's, used for sq_ slots. static Py_ssize_t str_length(Box* a) { return Py_SIZE(a); } @@ -2062,14 +2063,14 @@ Box* strSwapcase(BoxedString* self) { return rtn; } -Box* strContains(BoxedString* self, Box* elt) { +static inline int string_contains_shared(BoxedString* self, Box* elt) { assert(PyString_Check(self)); if (PyUnicode_Check(elt)) { int r = PyUnicode_Contains(self, elt); if (r < 0) throwCAPIException(); - return boxBool(r); + return r; } if (!PyString_Check(elt)) @@ -2083,9 +2084,16 @@ Box* strContains(BoxedString* self, Box* elt) { found_idx = self->s().find(sub->s()[0]); else found_idx = self->s().find(sub->s()); - if (found_idx == std::string::npos) - return False; - return True; + return (found_idx != std::string::npos); +} + +// Analoguous to CPython's, used for sq_ slots. +static int string_contains(PyObject* str_obj, PyObject* sub_obj) { + return string_contains_shared((BoxedString*)str_obj, sub_obj); +} + +Box* strContains(BoxedString* self, Box* elt) { + return boxBool(string_contains_shared(self, elt)); } // compares (a+a_pos, len) with (str) @@ -2284,6 +2292,17 @@ Box* strEncode(BoxedString* self, Box* encoding, Box* error) { return result; } +static PyObject* string_item(PyStringObject* self, register Py_ssize_t i) { + BoxedString* boxedString = (BoxedString*)self; + + if (i < 0 || i >= boxedString->size()) { + raiseExcHelper(IndexError, "string index out of range"); + } + + char c = boxedString->s()[i]; + return characters[c & UCHAR_MAX]; +} + template Box* strGetitem(BoxedString* self, Box* slice) { if (S == CAPI) { try { @@ -2857,10 +2876,12 @@ void setupStr() { add_operators(str_cls); str_cls->freeze(); - str_cls->tp_as_sequence->sq_slice = str_slice; - str_cls->tp_as_sequence->sq_length = str_length; str_cls->tp_iter = (decltype(str_cls->tp_iter))strIter; str_cls->tp_hash = (hashfunc)str_hash; + str_cls->tp_as_sequence->sq_length = str_length; + str_cls->tp_as_sequence->sq_item = (ssizeargfunc)string_item; + str_cls->tp_as_sequence->sq_slice = str_slice; + str_cls->tp_as_sequence->sq_contains = (objobjproc)string_contains; basestring_cls->giveAttr("__doc__", boxString("Type basestring cannot be instantiated; it is the base for str and unicode.")); diff --git a/test/testsuite b/test/testsuite index e1d16ce89..1b28a3b4c 160000 --- a/test/testsuite +++ b/test/testsuite @@ -1 +1 @@ -Subproject commit e1d16ce8922700ef6bde50589e84db3a5c095c2b +Subproject commit 1b28a3b4c941c7b93fcc0c5174147641fdeadf30