STEAM for Vietnam's enhancements

.gitignore

@@ -22,3 +22,8 @@ reports/
 \#*\#
 *.egg-info
 .idea/
+
+
+# Visual Studio Code
+.vscode/*.log
+*.code-workspace

.prospector.yaml

@@ -0,0 +1,62 @@
+mccabe:
+  disable:
+    - MC0001
+
+pep8:
+  disable:
+    - E305
+    - E306
+    - E115
+    - E116
+    - E501
+    - E722
+    - E741
+
+pycodestyle:
+  disable:
+    - E115
+    - E116
+    - E305
+    - E306
+    - E501
+    - E722
+    - E741
+
+pyflakes:
+  disable:
+    - F401
+    - F821
+    - F841
+
+pylint:
+  disable:
+    - arguments-renamed
+    - bare-except
+    - consider-using-f-string
+    - consider-using-with
+    - deprecated-module
+    - django-not-configured
+    - import-error
+    - import-outside-toplevel
+    - inconsistent-return-statements
+    - line-too-long
+    - logging-format-interpolation
+    - logging-not-lazy
+    - method-hidden
+    - multiple-imports
+    - no-else-raise
+    - no-else-return
+    - pointless-statement
+    - super-with-arguments
+    - too-many-arguments
+    - too-many-branches
+    - too-many-locals
+    - too-many-statements
+    - undefined-variable
+    - unidiomatic-typecheck
+    - unused-argument
+    - unused-import
+    - unused-variable
+    - unspecified-encoding
+    - useless-object-inheritance
+    - useless-suppression

.vscode/settings.json

@@ -0,0 +1,91 @@
+{
+  "editor.rulers": [79],
+
+  "files.exclude": {
+    "**/*.egg-info": true,
+    "**/.git": true,
+    "**/.mypy_cache": true,
+    "**/*.pyc": {"when": "$(basename).py"},
+    "**/__pycache__": true,
+    "**/.ropeproject": true
+  },
+
+  "python.analysis.diagnosticSeverityOverrides": {
+    "reportMissingImports" : "none",
+    "reportMissingModuleSource" : "none",
+    "reportUndefinedVariable" : "none"
+  },
+
+  "python.linting.enabled": true,
+
+  "python.linting.pylintEnabled": true,
+  "python.linting.pylintArgs": [
+    "--disable", "arguments-renamed",
+    "--disable", "bare-except",
+    "--disable", "broad-except",
+    "--disable", "c-extension-no-member",
+    "--disable", "consider-using-f-string",
+    "--disable", "consider-using-with",
+    "--disable", "deprecated-module",
+    "--disable", "fixme",
+    "--disable", "import-error",
+    "--disable", "import-outside-toplevel",
+    "--disable", "inconsistent-return-statements",
+    "--disable", "invalid-name",
+    "--disable", "line-too-long",
+    "--disable", "logging-format-interpolation",
+    "--disable", "logging-not-lazy",
+    "--disable", "method-hidden",
+    "--disable", "missing-class-docstring",
+    "--disable", "missing-function-docstring",
+    "--disable", "missing-module-docstring",
+    "--disable", "multiple-imports",
+    "--disable", "no-else-raise",
+    "--disable", "no-else-return",
+    "--disable", "no-self-use",
+    "--disable", "pointless-statement",
+    "--disable", "super-with-arguments",
+    "--disable", "too-few-public-methods",
+    "--disable", "too-many-arguments",
+    "--disable", "too-many-branches",
+    "--disable", "too-many-instance-attributes",
+    "--disable", "too-many-locals",
+    "--disable", "too-many-return-statements",
+    "--disable", "too-many-statements",
+    "--disable", "undefined-variable",
+    "--disable", "unidiomatic-typecheck",
+    "--disable", "unnecessary-pass",
+    "--disable", "unspecified-encoding",
+    "--disable", "unused-argument",
+    "--disable", "unused-import",
+    "--disable", "unused-variable",
+    "--disable", "useless-object-inheritance",
+    "--disable", "wrong-import-order"
+  ],
+
+  "python.linting.flake8Enabled": true,
+  "python.linting.flake8Args": [
+    "--ignore=E115,E116,E123,E128,E226,E231,E261,E265,E266,E302,E303,E305,E306,E401,E501,E722,E741,F401,F821,F841,N806"
+  ],
+
+  "python.linting.mypyEnabled": false,
+
+  "python.linting.pydocstyleEnabled": false,
+
+  "python.linting.pycodestyleEnabled": true,
+  "python.linting.pycodestyleArgs": [
+    "--ignore=E115,E116,E123,E128,E226,E261,E265,E231,E266,E302,E303,E305,E306,E401,E501,E722,E741"
+  ],
+
+  "python.linting.prospectorEnabled": true,
+
+  "python.linting.pylamaEnabled": true,
+  "python.linting.pylamaArgs": [
+    "--ignore=C901,E115,E116,E123,E128,E226,E231,E261,E265,E266,E302,E303,E305,E306,E401,E501,E0602,E722,E741,W0611,W0612"
+  ],
+
+  "python.linting.banditEnabled": true,
+  "python.linting.banditArgs": [
+    "--skip=B103,B108,B110,B311"
+  ]
+}

README.md

@@ -3,58 +3,153 @@ xqueue_watcher
 
 This is an implementation of a polling [XQueue](https://github.com/edx/xqueue) client and grader.
 
+Overview
+========
 
-Running
-=======
-
-`python -m xqueue_watcher -d [path to settings directory]`
-
-
-JSON configuration file
-=======================
-	{
-		"test-123": {
-			"SERVER": "http://127.0.0.1:18040",
-			"CONNECTIONS": 1,
-			"AUTH": ["lms", "lms"],
-			"HANDLERS": [
-				{
-					"HANDLER": "xqueue_watcher.grader.Grader",
-					"KWARGS": {
-						"grader_root": "/path/to/course/graders/",
-					}
-				}
-			]
-		}
-	}
+There are several components in a working XQueue Watcher service:
+- **XQueue Watcher**: it polls an xqueue service continually for new submissions and grades them.
+- **Submissions Handler**: when the watcher finds any new submission, it will be passed to the handler for grading. It is a generic handler that can be configured to work with different submissions through individual submission graders.
+- **Individual Submission Grader**: each exercise or homework may specify its own "grader". This should map to a file on the server that usually specifies test cases or additional processing for the student submission.
+
+Usually your server will look like this:
+```
+root/
+├── xqueue-watcher/
+│   ├── ... # xqueue-watcher repo, unchanged
+│   └── ...
+├── config/
+│   └── conf.d/
+│   │   └── my-course.json
+│   └── logging.json
+└── my-course/
+   ├── exercise1/
+   │   ├── grader.py  # - per-exercise grader
+   │   └── answer.py  # - if using JailedGrader
+   ├── ...
+   └── exercise2/
+       ├── grader.py
+       └── answer.py
+```
+Running XQueue Watcher:
+======================
+
+Usually you can run XQueue Watcher without making any changes. You should keep course-specific in another folder like shown above, so that you can update xqueue_watcher anytime.
+
+Install the requirements before running `xqueue_watcher`
+```bash
+cd xqueue-watcher/
+make requirements
+```
+
+Now you're ready to run it.
+```bash
+python -m xqueue_watcher -d [path to the config directory, eg ../config]
+```
+
+The course configuration JSON file in `conf.d` should have the following structure:
+```json
+    {
+        "test-123": {
+            "SERVER": "http://127.0.0.1:18040",
+            "CONNECTIONS": 1,
+            "AUTH": ["lms", "lms"],
+            "HANDLERS": [
+                {
+                    "HANDLER": "xqueue_watcher.grader.Grader",
+                    "KWARGS": {
+                        "grader_root": "/path/to/course/graders/",
+                    }
+                }
+            ]
+        }
+    }
+```
 
 * `test-123`: the name of the queue
 * `SERVER`: XQueue server address
 * `AUTH`: list of username, password
 * `CONNECTIONS`: how many threads to spawn to watch the queue
 * `HANDLERS`: list of callables that will be called for each queue submission
-	* `HANDLER`: callable name
-	* `KWARGS`: optional keyword arguments to apply during instantiation
+   * `HANDLER`: callable name, see below for Submissions Handler
+   * `KWARGS`: optional keyword arguments to apply during instantiation
+      * `grader_root`: path to the course directory, eg /path/to/my-course
 
+> TODO: document logging.json
 
-xqueue_watcher.grader.Grader
-========================
+Submissions Handler
+===================
+
+When xqueue_watcher detects any new submission, it will be passed to the submission handler for grading. It will instantiate a new handler based on the name configured above, with submission information retrieved
+from XQueue. There is a base grader defined in xqueue_watcher: Grader and JailedGrader (for Python, using CodeJail). If you don't use JailedGrader, you'd have to implement your own Grader by subclassing `xqueue_watcher.grader.Grader
+
+The payload from XQueue will be a JSON that usually looks like this, notice that "grader" is a required field in the "grader_payload" and must be configured accordingly in the Studio for the exercise.
+```json
+{
+    "student_info": {
+        "random_seed": 1,
+        "submission_time": "20210109222647",
+        "anonymous_student_id": "6d07814a4ece5cdda54af1558a6dfec0"
+    },
+    "grader_payload": "\n        {\"grader\": \"relative/path/to/grader.py\"}\n      ",
+    "student_response": "print \"hello\"\r\n      "
+}
+```
+
+## Custom Handler
 To implement a pull grader:
 
-Subclass xqueue_watcher.grader.Grader and override the `grade` method. Then add your grader to the config like `"handler": "my_module.MyGrader"`. The arguments for the `grade` method are:
-	* `grader_path`: absolute path to the grader defined for the current problem
-	* `grader_config`: other configuration particular to the problem
-	* `student_response`: student-supplied code
+Subclass `xqueue_watcher.grader.Grader` and override the `grade` method. Then add your grader to the config like `"handler": "my_module.MyGrader"`. The arguments for the `grade` method are:
+   * `grader_path`: absolute path to the grader defined for the current problem.
+   * `grader_config`: other configuration particular to the problem
+   * `student_response`: student-supplied code
 
+Note that `grader_path` is constructed by appending the relative path to the grader from `grader_payload` to the `grader_root` in the configuration JSON. If the handler cannot find a `grader.py` file, it would fail to grade the submission.
 
-Sandboxing
-==========
-To sandbox python, use [CodeJail](https://github.com/edx/codejail). In your handler configuration, add:
+## Grading Python submissions with JailedGrader
 
-	"CODEJAIL": {
-		"name": "python",
-		"python_bin": "/path/to/sandbox/python",
-		"user": "sandbox_username"
-	}
+`xqueue_watcher` provides a few utilities for grading python submissions, including JailedGrader for running python code in a safe environment and grading support utilities.
 
+### JailedGrader
+To sandbox python, use [CodeJail](https://github.com/edx/codejail). In your handler configuration, add:
+```json
+    "HANDLER": "xqueue_watcher.jailedgrader.JailedGrader",
+    "CODEJAIL": {
+        "name": "python",
+        "python_bin": "/path/to/sandbox/python",
+        "user": "sandbox_username"
+    }
+```
 Then, `codejail_python` will automatically be added to the kwargs for your handler. You can then import codejail.jail_code and run `jail_code("python", code...)`. You can define multiple sandboxes and use them as in `jail_code("special-python", ...)`
+
+To use JailedGrader, you also need to provide an `answer.py` file on the same folder with the `grader.py` file. The grader will run both student submission and `answer.py` and compare the output with each other.
+
+### Grading Support utilities
+There are several grading support utilities that make writing `grader.py` for python code easy. Check out
+`grader_support/gradelib.py` for the documentation.
+
+- `grader_support.gradelib.Grader`: a base class for creating a new submission grader. Not to be confused with `xqueue-watcher.grader.Grader`. You can add input checks, preprocessors and tests to a grader object.
+- `grader_support.gradelib.Test`: a base class for creating tests for a submission. Usually a submission can be graded with one or a few tests. There are also few useful test functions and classes included, like `InvokeStudentFunctionTest` , `exec_wrapped_code`, etc.
+- Preprocessors: utilities to process the raw submission before grading it. `wrap_in_string` is useful for testing code that is not wrapped in a function.
+- Input checks: sanity checks before running a submission, eg check `required_string` or `prohibited_string`
+
+Using the provided grader class, your `grader.py` would look something like this:
+```python
+from grader_support import gradelib
+grader = gradelib.Grader()
+
+# invoke student function foo with parameter []
+grader.add_test(gradelib.InvokeStudentFunctionTest('foo', []))
+```
+
+Or with a pre-processor:
+```python
+import gradelib
+
+grader = gradelib.Grader()
+
+# execute a raw student code & capture stdout
+grader.add_preprocessor(gradelib.wrap_in_string)
+grader.add_test(gradelib.ExecWrappedStudentCodeTest({}, "basic test"))
+```
+
+You can also write your own test class, processor and input checks.

grader_support/gradelib.py

@@ -69,6 +69,9 @@ def __init__(self):
         # list of functions: submission_text -> error text or None
         self._input_checks = []
 
+        # Flag: Do not run, just check input
+        self._only_check_input = False
+
         # list of functions: submission_text -> processed_submission_text.  Run
         # in the specified order.  (foldl)
         self._preprocessors = [fix_line_endings]
@@ -88,6 +91,12 @@ def input_errors(self, submission_str):
         """
         return [_f for _f in [check(submission_str) for check in self._input_checks] if _f]
 
+    def only_check_input(self):
+        return self._only_check_input
+
+    def set_only_check_input(self, value):
+        self._only_check_input = value
+
     def preprocess(self, submission_str):
         """
         submission: string
@@ -546,6 +555,17 @@ def __init__(self, fn_name, args, environment=None, output_writer=None, short_de
             short_desc = "Test: %s(%s)" % (fn_name, ", ".join(repr(a) for a in args))
         Test.__init__(self, test_fn, short_desc, detailed_desc, compare)
 
+class ExecWrappedStudentCodeTest(Test):
+    """
+    A Test that exec student code and capture the stdout result.
+    The code must be preprocessed with `wrap_in_string`
+    """
+    def __init__(self, environment=None, short_desc=None, detailed_desc=None, compare=None):
+        test_fn = exec_wrapped_code(environment)
+        if short_desc is None:
+            short_desc = "Test: %s(%s)" % (fn_name, ", ".join(repr(a) for a in args))
+        Test.__init__(self, test_fn, short_desc, detailed_desc, compare)
+
 def round_float_writer(n):
     """
     Returns an output_writer function that rounds its argument to `n` places.