From 748621447bb544f1867f1436ea265b95caf1981f Mon Sep 17 00:00:00 2001 From: Claudiu Coman Date: Wed, 22 Feb 2017 01:21:27 +0200 Subject: [PATCH 01/11] Add LoadMembers for binding module members --- runtime/module.go | 61 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/runtime/module.go b/runtime/module.go index 36a0a579..6988b404 100644 --- a/runtime/module.go +++ b/runtime/module.go @@ -179,6 +179,67 @@ func ImportNativeModule(f *Frame, name string, members map[string]*Object) (*Obj return prev, nil } +// LoadMembers scans over all the members in module +// and populates globals with them, taking __all__ into +// account. +func LoadMembers(f *Frame, module *Object) *BaseException { + all_attr, raised := GetAttr(f, module, NewStr("__all__"), nil) + if raised != nil && !raised.isInstance(AttributeErrorType) { + return raised + } + f.RestoreExc(nil, nil) + + if raised == nil { + raised = loadMembersFromIterable(f, module, all_attr, nil) + if raised != nil { + return raised + } + return nil + } + + // Fall back on __dict__ + dict_attr, raised := GetAttr(f, module, NewStr("__dict__"), nil) + if raised != nil && !raised.isInstance(AttributeErrorType) { + return raised + } + raised = loadMembersFromIterable(f, module, dict_attr, func(key *Object) bool { + return strings.HasPrefix(toStrUnsafe(key).value, "_") + }) + if raised != nil { + return raised + } + return nil +} + +func loadMembersFromIterable(f *Frame, module, iterable *Object, filter_f func(*Object) bool) *BaseException { + iter, raised := Iter(f, iterable) + if raised != nil { + return raised + } + + member_name, raised := Next(f, iter) + for ; raised == nil; member_name, raised = Next(f, iter) { + member, raised := GetAttr(f, module, toStrUnsafe(member_name), nil) + if raised != nil { + return raised + } + if filter_f != nil { + if filter_f(member_name) { + continue + } + } + raised = f.Globals().SetItem(f, member_name, member) + if raised != nil { + return raised + } + } + if !raised.isInstance(StopIterationType) { + return raised + } + f.RestoreExc(nil, nil) + return nil +} + // newModule creates a new Module object with the given fully qualified name // (e.g a.b.c) and its corresponding Python filename. func newModule(name, filename string) *Module { From b11917275832b3472223354ddd24a3fed165890d Mon Sep 17 00:00:00 2001 From: Claudiu Coman Date: Wed, 22 Feb 2017 01:22:14 +0200 Subject: [PATCH 02/11] Add support for wildcard imports --- compiler/stmt.py | 13 ++++++++----- compiler/stmt_test.py | 14 ++++++-------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/compiler/stmt.py b/compiler/stmt.py index 98051a8d..ff3fea92 100644 --- a/compiler/stmt.py +++ b/compiler/stmt.py @@ -378,13 +378,16 @@ def visit_Import(self, node): self.block.bind_var(self.writer, asname, mod.expr) def visit_ImportFrom(self, node): - # Wildcard imports are not yet supported. + self._write_py_context(node.lineno) for alias in node.names: if alias.name == '*': - msg = 'wildcard member import is not implemented: from %s import %s' % ( - node.module, alias.name) - raise util.ParseError(node, msg) - self._write_py_context(node.lineno) + module_name = node.module + + with self.block.alloc_temp() as members, \ + self._import(module_name, module_name.count('.')) as module: + self.writer.write_checked_call1( + 'πg.LoadMembers(πF, {})', module.expr) + return if node.module.startswith(_NATIVE_MODULE_PREFIX): values = [alias.name for alias in node.names] with self._import_native(node.module, values) as mod: diff --git a/compiler/stmt_test.py b/compiler/stmt_test.py index 5c351c12..df5f25b6 100644 --- a/compiler/stmt_test.py +++ b/compiler/stmt_test.py @@ -366,14 +366,12 @@ def testImportFromFutureParseError(self): self.assertRaisesRegexp(util.ParseError, want_regexp, stmt.import_from_future, node) - def testImportWildcardMemberRaises(self): - regexp = r'wildcard member import is not implemented: from foo import *' - self.assertRaisesRegexp(util.ParseError, regexp, _ParseAndVisit, - 'from foo import *') - regexp = (r'wildcard member import is not ' - r'implemented: from __go__.foo import *') - self.assertRaisesRegexp(util.ParseError, regexp, _ParseAndVisit, - 'from __go__.foo import *') + def testImportWildcard(self): + result = _GrumpRun(textwrap.dedent("""\ + from time import * + print sleep""")) + self.assertEqual(0, result[0]) + self.assertIn(' Date: Wed, 22 Feb 2017 01:31:12 +0200 Subject: [PATCH 03/11] Fix coding style issues --- runtime/module.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/runtime/module.go b/runtime/module.go index 6988b404..08033ab0 100644 --- a/runtime/module.go +++ b/runtime/module.go @@ -183,14 +183,14 @@ func ImportNativeModule(f *Frame, name string, members map[string]*Object) (*Obj // and populates globals with them, taking __all__ into // account. func LoadMembers(f *Frame, module *Object) *BaseException { - all_attr, raised := GetAttr(f, module, NewStr("__all__"), nil) + allAttr, raised := GetAttr(f, module, NewStr("__all__"), nil) if raised != nil && !raised.isInstance(AttributeErrorType) { return raised } f.RestoreExc(nil, nil) if raised == nil { - raised = loadMembersFromIterable(f, module, all_attr, nil) + raised = loadMembersFromIterable(f, module, allAttr, nil) if raised != nil { return raised } @@ -198,11 +198,11 @@ func LoadMembers(f *Frame, module *Object) *BaseException { } // Fall back on __dict__ - dict_attr, raised := GetAttr(f, module, NewStr("__dict__"), nil) + dictAttr, raised := GetAttr(f, module, NewStr("__dict__"), nil) if raised != nil && !raised.isInstance(AttributeErrorType) { return raised } - raised = loadMembersFromIterable(f, module, dict_attr, func(key *Object) bool { + raised = loadMembersFromIterable(f, module, dictAttr, func(key *Object) bool { return strings.HasPrefix(toStrUnsafe(key).value, "_") }) if raised != nil { @@ -211,24 +211,24 @@ func LoadMembers(f *Frame, module *Object) *BaseException { return nil } -func loadMembersFromIterable(f *Frame, module, iterable *Object, filter_f func(*Object) bool) *BaseException { +func loadMembersFromIterable(f *Frame, module, iterable *Object, filterF func(*Object) bool) *BaseException { iter, raised := Iter(f, iterable) if raised != nil { return raised } - member_name, raised := Next(f, iter) - for ; raised == nil; member_name, raised = Next(f, iter) { - member, raised := GetAttr(f, module, toStrUnsafe(member_name), nil) + memberName, raised := Next(f, iter) + for ; raised == nil; memberName, raised = Next(f, iter) { + member, raised := GetAttr(f, module, toStrUnsafe(memberName), nil) if raised != nil { return raised } - if filter_f != nil { - if filter_f(member_name) { + if filterF != nil { + if filterF(memberName) { continue } } - raised = f.Globals().SetItem(f, member_name, member) + raised = f.Globals().SetItem(f, memberName, member) if raised != nil { return raised } From b95d2986040c8b09b974feddef084c27777a794b Mon Sep 17 00:00:00 2001 From: Claudiu Coman Date: Wed, 22 Feb 2017 01:38:56 +0200 Subject: [PATCH 04/11] Remove unused variable --- compiler/stmt.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/compiler/stmt.py b/compiler/stmt.py index ff3fea92..d0f5dbbb 100644 --- a/compiler/stmt.py +++ b/compiler/stmt.py @@ -383,8 +383,7 @@ def visit_ImportFrom(self, node): if alias.name == '*': module_name = node.module - with self.block.alloc_temp() as members, \ - self._import(module_name, module_name.count('.')) as module: + with self._import(module_name, module_name.count('.')) as module: self.writer.write_checked_call1( 'πg.LoadMembers(πF, {})', module.expr) return From cfcd4c3b6add6a41f4cdaa66279ea9c014cebd3d Mon Sep 17 00:00:00 2001 From: Claudiu Coman Date: Thu, 23 Feb 2017 02:22:37 +0200 Subject: [PATCH 05/11] Add tests for LoadMembers --- runtime/module_test.go | 48 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/runtime/module_test.go b/runtime/module_test.go index 5b70f57a..691fc554 100644 --- a/runtime/module_test.go +++ b/runtime/module_test.go @@ -204,6 +204,54 @@ func TestImportNativeModule(t *testing.T) { } } +func TestLoadMembers(t *testing.T) { + f := NewRootFrame() + var1 := NewStr("var1").ToObject() + var2 := NewStr("_var2").ToObject() + var3 := NewStr("var3").ToObject() + nameAttr := NewStr("__name__").ToObject() + allAttr := NewStr("__all__") + val1 := NewStr("val1").ToObject() + val2 := NewStr("val2").ToObject() + val3 := NewStr("val3").ToObject() + nameValue := NewStr("foo").ToObject() + allValue := newTestList(var1, var2).ToObject() + + allDefinedDict := newTestDict(var1, val1, var2, val2, var3, val3, nameAttr, nameValue, allAttr, allValue) + fooAllDefinedModule := &Module{Object: Object{typ: testModuleType, dict: allDefinedDict}} + allUndefinedDict := newTestDict(var1, val1, var2, val2, var3, val3, nameAttr, nameValue) + fooAllUndefinedModule := &Module{Object: Object{typ: testModuleType, dict: allUndefinedDict}} + + cases := []struct { + module *Module + wantGlobals *Dict + }{ + { + fooAllDefinedModule, + newTestDict(var1, val1, var2, val2), + }, + { + fooAllUndefinedModule, + newTestDict(var1, val1, var3, val3), + }, + } + for _, cas := range cases { + f.globals = NewDict() + raised := LoadMembers(f, cas.module.ToObject()) + if raised != nil { + t.Errorf("LoadMmembers: raised %v", raised) + } + ne := mustNotRaise(NE(f, f.Globals().ToObject(), cas.wantGlobals.ToObject())) + b, raised := IsTrue(f, ne) + if raised != nil { + panic(raised) + } + if b { + t.Errorf("LoadMembers: Globals() = %v, want %v", f.Globals(), cas.wantGlobals) + } + } +} + func TestModuleGetNameAndFilename(t *testing.T) { fun := wrapFuncForTest(func(f *Frame, m *Module) (*Tuple, *BaseException) { name, raised := m.GetName(f) From bf75ff98f8ca91e2aaa40b8b6c8de3dcd0d4f4a9 Mon Sep 17 00:00:00 2001 From: Claudiu Coman Date: Sat, 25 Feb 2017 02:29:31 +0200 Subject: [PATCH 06/11] Refactor LoadMembers --- runtime/module.go | 31 +++++++++---------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/runtime/module.go b/runtime/module.go index 08033ab0..bd892417 100644 --- a/runtime/module.go +++ b/runtime/module.go @@ -198,10 +198,7 @@ func LoadMembers(f *Frame, module *Object) *BaseException { } // Fall back on __dict__ - dictAttr, raised := GetAttr(f, module, NewStr("__dict__"), nil) - if raised != nil && !raised.isInstance(AttributeErrorType) { - return raised - } + dictAttr := module.dict.ToObject() raised = loadMembersFromIterable(f, module, dictAttr, func(key *Object) bool { return strings.HasPrefix(toStrUnsafe(key).value, "_") }) @@ -212,32 +209,22 @@ func LoadMembers(f *Frame, module *Object) *BaseException { } func loadMembersFromIterable(f *Frame, module, iterable *Object, filterF func(*Object) bool) *BaseException { - iter, raised := Iter(f, iterable) - if raised != nil { - return raised - } - - memberName, raised := Next(f, iter) - for ; raised == nil; memberName, raised = Next(f, iter) { + globals := f.Globals() + raised := seqForEach(f, iterable, func(memberName *Object) *BaseException { member, raised := GetAttr(f, module, toStrUnsafe(memberName), nil) if raised != nil { return raised } - if filterF != nil { - if filterF(memberName) { - continue - } + if filterF != nil && filterF(memberName) { + return nil } - raised = f.Globals().SetItem(f, memberName, member) + raised = globals.SetItem(f, memberName, member) if raised != nil { return raised } - } - if !raised.isInstance(StopIterationType) { - return raised - } - f.RestoreExc(nil, nil) - return nil + return nil + }) + return raised } // newModule creates a new Module object with the given fully qualified name From b97e1f3257868b89e4590715bdcec1c7f7453a0b Mon Sep 17 00:00:00 2001 From: Claudiu Coman Date: Sat, 25 Feb 2017 02:59:09 +0200 Subject: [PATCH 07/11] Refactor LoadMembers tests --- runtime/module_test.go | 36 +++++++++++++++--------------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/runtime/module_test.go b/runtime/module_test.go index 691fc554..167384a5 100644 --- a/runtime/module_test.go +++ b/runtime/module_test.go @@ -205,7 +205,6 @@ func TestImportNativeModule(t *testing.T) { } func TestLoadMembers(t *testing.T) { - f := NewRootFrame() var1 := NewStr("var1").ToObject() var2 := NewStr("_var2").ToObject() var3 := NewStr("var3").ToObject() @@ -222,32 +221,27 @@ func TestLoadMembers(t *testing.T) { allUndefinedDict := newTestDict(var1, val1, var2, val2, var3, val3, nameAttr, nameValue) fooAllUndefinedModule := &Module{Object: Object{typ: testModuleType, dict: allUndefinedDict}} - cases := []struct { - module *Module - wantGlobals *Dict - }{ + fun := wrapFuncForTest(func(f *Frame, module *Module) (*Dict, *BaseException) { + f.globals = NewDict() + raised := LoadMembers(f, module.ToObject()) + if raised != nil { + return nil, raised + } + return f.Globals(), nil + }) + cases := []invokeTestCase{ { - fooAllDefinedModule, - newTestDict(var1, val1, var2, val2), + args: wrapArgs(fooAllDefinedModule), + want: newTestDict(var1, val1, var2, val2).ToObject(), }, { - fooAllUndefinedModule, - newTestDict(var1, val1, var3, val3), + args: wrapArgs(fooAllUndefinedModule), + want: newTestDict(var1, val1, var3, val3).ToObject(), }, } for _, cas := range cases { - f.globals = NewDict() - raised := LoadMembers(f, cas.module.ToObject()) - if raised != nil { - t.Errorf("LoadMmembers: raised %v", raised) - } - ne := mustNotRaise(NE(f, f.Globals().ToObject(), cas.wantGlobals.ToObject())) - b, raised := IsTrue(f, ne) - if raised != nil { - panic(raised) - } - if b { - t.Errorf("LoadMembers: Globals() = %v, want %v", f.Globals(), cas.wantGlobals) + if err := runInvokeTestCase(fun, &cas); err != "" { + t.Error(err) } } } From 9fa283cbaa7587bbcd7b7e9f63b906b7ae673dd1 Mon Sep 17 00:00:00 2001 From: Claudiu Coman Date: Sat, 25 Feb 2017 13:55:06 +0200 Subject: [PATCH 08/11] Improve validation for LoadMembers --- runtime/module.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/runtime/module.go b/runtime/module.go index bd892417..0b28f113 100644 --- a/runtime/module.go +++ b/runtime/module.go @@ -211,6 +211,10 @@ func LoadMembers(f *Frame, module *Object) *BaseException { func loadMembersFromIterable(f *Frame, module, iterable *Object, filterF func(*Object) bool) *BaseException { globals := f.Globals() raised := seqForEach(f, iterable, func(memberName *Object) *BaseException { + if !memberName.isInstance(StrType) { + error_message := fmt.Sprintf("attribute name must be string, not '%v'", memberName.typ.Name()) + return f.RaiseType(AttributeErrorType, error_message) + } member, raised := GetAttr(f, module, toStrUnsafe(memberName), nil) if raised != nil { return raised From 3515086e4222c5d30b22623b2eefd2712c86b9a2 Mon Sep 17 00:00:00 2001 From: Claudiu Coman Date: Sat, 25 Feb 2017 13:55:42 +0200 Subject: [PATCH 09/11] Add more LoadMembers tests --- runtime/module_test.go | 54 +++++++++++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 16 deletions(-) diff --git a/runtime/module_test.go b/runtime/module_test.go index 167384a5..4a91dac6 100644 --- a/runtime/module_test.go +++ b/runtime/module_test.go @@ -205,21 +205,31 @@ func TestImportNativeModule(t *testing.T) { } func TestLoadMembers(t *testing.T) { - var1 := NewStr("var1").ToObject() - var2 := NewStr("_var2").ToObject() - var3 := NewStr("var3").ToObject() - nameAttr := NewStr("__name__").ToObject() + var1 := NewStr("var1") + var2 := NewStr("_var2") + var3 := NewStr("var3") + nameAttr := NewStr("__name__") allAttr := NewStr("__all__") - val1 := NewStr("val1").ToObject() - val2 := NewStr("val2").ToObject() - val3 := NewStr("val3").ToObject() - nameValue := NewStr("foo").ToObject() - allValue := newTestList(var1, var2).ToObject() + val1 := NewStr("val1") + val2 := NewStr("val2") + val3 := NewStr("val3") + nameValue := NewStr("foo") + allValue := newTestList(var1, var2) + invalidMembers := newTestList(NewInt(1)) + invalidMemberName := NewInt(1) + nonIterableValue := NewInt(1) + allDefinedDict := newTestDict(var1, val1, var2, val2, var3, val3, nameAttr, nameValue, allAttr, allValue) - fooAllDefinedModule := &Module{Object: Object{typ: testModuleType, dict: allDefinedDict}} + allDefinedModule := &Module{Object: Object{typ: testModuleType, dict: allDefinedDict}} allUndefinedDict := newTestDict(var1, val1, var2, val2, var3, val3, nameAttr, nameValue) - fooAllUndefinedModule := &Module{Object: Object{typ: testModuleType, dict: allUndefinedDict}} + allUndefinedModule := &Module{Object: Object{typ: testModuleType, dict: allUndefinedDict}} + allInvalidDict := newTestDict(allAttr, invalidMembers) + allInvalidModule := &Module{Object: Object{typ: testModuleType, dict: allInvalidDict}} + packageNamesInvalidDict := newTestDict(invalidMemberName, val1) + packageNamesInvalidModule := &Module{Object: Object{typ: testModuleType, dict: packageNamesInvalidDict}} + allNonIterableDict := newTestDict(allAttr, nonIterableValue) + allNonIterableModule := &Module{Object: Object{typ: testModuleType, dict: allNonIterableDict}} fun := wrapFuncForTest(func(f *Frame, module *Module) (*Dict, *BaseException) { f.globals = NewDict() @@ -231,18 +241,30 @@ func TestLoadMembers(t *testing.T) { }) cases := []invokeTestCase{ { - args: wrapArgs(fooAllDefinedModule), + args: wrapArgs(allDefinedModule), want: newTestDict(var1, val1, var2, val2).ToObject(), }, { - args: wrapArgs(fooAllUndefinedModule), + args: wrapArgs(allUndefinedModule), want: newTestDict(var1, val1, var3, val3).ToObject(), }, + { + args: wrapArgs(allInvalidModule), + wantExc: mustCreateException(AttributeErrorType, "attribute name must be string, not 'int'"), + }, + { + args: wrapArgs(packageNamesInvalidModule), + wantExc: mustCreateException(AttributeErrorType, "attribute name must be string, not 'int'"), + }, + { + args: wrapArgs(allNonIterableModule), + wantExc: mustCreateException(TypeErrorType, "'int' object is not iterable"), + }, } for _, cas := range cases { - if err := runInvokeTestCase(fun, &cas); err != "" { - t.Error(err) - } + if err := runInvokeTestCase(fun, &cas); err != "" { + t.Error(err) + } } } From 4deb42a43b831f48574567a37f42f51b1a4596a6 Mon Sep 17 00:00:00 2001 From: Claudiu Coman Date: Sat, 25 Feb 2017 14:00:04 +0200 Subject: [PATCH 10/11] Fix Go code style --- runtime/module_test.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/runtime/module_test.go b/runtime/module_test.go index 4a91dac6..5b34c5f5 100644 --- a/runtime/module_test.go +++ b/runtime/module_test.go @@ -219,7 +219,6 @@ func TestLoadMembers(t *testing.T) { invalidMemberName := NewInt(1) nonIterableValue := NewInt(1) - allDefinedDict := newTestDict(var1, val1, var2, val2, var3, val3, nameAttr, nameValue, allAttr, allValue) allDefinedModule := &Module{Object: Object{typ: testModuleType, dict: allDefinedDict}} allUndefinedDict := newTestDict(var1, val1, var2, val2, var3, val3, nameAttr, nameValue) @@ -249,22 +248,22 @@ func TestLoadMembers(t *testing.T) { want: newTestDict(var1, val1, var3, val3).ToObject(), }, { - args: wrapArgs(allInvalidModule), + args: wrapArgs(allInvalidModule), wantExc: mustCreateException(AttributeErrorType, "attribute name must be string, not 'int'"), }, { - args: wrapArgs(packageNamesInvalidModule), + args: wrapArgs(packageNamesInvalidModule), wantExc: mustCreateException(AttributeErrorType, "attribute name must be string, not 'int'"), }, { - args: wrapArgs(allNonIterableModule), + args: wrapArgs(allNonIterableModule), wantExc: mustCreateException(TypeErrorType, "'int' object is not iterable"), }, } for _, cas := range cases { - if err := runInvokeTestCase(fun, &cas); err != "" { - t.Error(err) - } + if err := runInvokeTestCase(fun, &cas); err != "" { + t.Error(err) + } } } From 0724218991e14a934a8311cc674fc7aa89fbdfde Mon Sep 17 00:00:00 2001 From: Claudiu Coman Date: Sat, 25 Feb 2017 14:08:45 +0200 Subject: [PATCH 11/11] Remove underscore from variable name --- runtime/module.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/module.go b/runtime/module.go index 0b28f113..54b4929f 100644 --- a/runtime/module.go +++ b/runtime/module.go @@ -212,8 +212,8 @@ func loadMembersFromIterable(f *Frame, module, iterable *Object, filterF func(*O globals := f.Globals() raised := seqForEach(f, iterable, func(memberName *Object) *BaseException { if !memberName.isInstance(StrType) { - error_message := fmt.Sprintf("attribute name must be string, not '%v'", memberName.typ.Name()) - return f.RaiseType(AttributeErrorType, error_message) + errorMessage := fmt.Sprintf("attribute name must be string, not '%v'", memberName.typ.Name()) + return f.RaiseType(AttributeErrorType, errorMessage) } member, raised := GetAttr(f, module, toStrUnsafe(memberName), nil) if raised != nil {