diff --git a/.gitignore b/.gitignore index d63e56d..89f8268 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ # for macOS .DS_Store +examples/sandbox diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..2daa979 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,37 @@ +# Documentation: http://docs.travis-ci.com/user/languages/julia/ +language: julia + +os: + - linux + #- osx + +julia: + - 1.2 + - 1.3 + +codecov: true +coveralls: true + + +matrix: + allow_failures: + - julia: nightly + +addons: + apt: + packages: + - xvfb + - xauth + - libgtk-3-dev +notifications: + email: + false + +## uncomment the following lines to override the default test script +script: + - if [[ -a .git/shallow ]]; then git fetch --unshallow; fi + - if [[ `uname` = "Linux" ]]; then TESTCMD="xvfb-run julia"; else TESTCMD="julia"; fi + - $TESTCMD -e 'using Pkg; Pkg.build(); Pkg.test(coverage=true)' + +after_success: + - julia -e 'cd(Pkg.dir("MyPkg")); Pkg.add("Coverage"); using Coverage; Codecov.submit(Codecov.process_folder())' diff --git a/Manifest.toml b/Manifest.toml index b31df81..02a6056 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -1,22 +1,46 @@ # This file is machine-generated - editing it directly is not advised +[[ATK_jll]] +deps = ["Glib_jll", "Libdl", "Pkg"] +git-tree-sha1 = "7129d58ed99d42032cefe21bcd14171a878143d2" +uuid = "7b86fcea-f67b-53e1-809c-8f1719c154e8" +version = "2.34.1+2" + [[AbstractFFTs]] deps = ["LinearAlgebra"] -git-tree-sha1 = "380e36c66edfa099cd90116b24c1ce8cafccac40" +git-tree-sha1 = "051c95d6836228d120f5f4b984dd5aba1624f716" uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c" -version = "0.4.1" +version = "0.5.0" [[AbstractTrees]] -deps = ["Markdown", "Test"] -git-tree-sha1 = "6621d9645702c1c4e6970cc6a3eae440c768000b" +deps = ["Markdown"] +git-tree-sha1 = "940760b82db1f7090f7d16fe6e4b121cc181db45" uuid = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" -version = "0.2.1" +version = "0.3.1" [[Adapt]] -deps = ["LinearAlgebra", "Test"] -git-tree-sha1 = "53d8fec4f662088c1202530e338a11a919407f3b" +deps = ["LinearAlgebra"] +git-tree-sha1 = "82dab828020b872fa9efd3abec1152b075bc7cbf" uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" -version = "0.4.2" +version = "1.0.0" + +[[ArcadeLearningEnvironment]] +deps = ["BinDeps", "Compat", "Libdl"] +git-tree-sha1 = "b5ceb6d84ce6f7bed13bd0560c420cec4520b5a3" +uuid = "b7f77d8d-088d-5e02-8ac0-89aab2acc977" +version = "0.2.1" + +[[Arpack]] +deps = ["Arpack_jll", "Libdl", "LinearAlgebra"] +git-tree-sha1 = "2ff92b71ba1747c5fdd541f8fc87736d82f40ec9" +uuid = "7d9fca2a-8960-54d3-9f78-7d1dccf2cb97" +version = "0.4.0" + +[[Arpack_jll]] +deps = ["Libdl", "OpenBLAS_jll", "Pkg"] +git-tree-sha1 = "68a90a692ddc0eb72d69a6993ca26e2a923bf195" +uuid = "68821587-b530-5797-8361-c406ea357684" +version = "3.5.0+2" [[AxisAlgorithms]] deps = ["LinearAlgebra", "Random", "SparseArrays", "WoodburyMatrices"] @@ -25,43 +49,66 @@ uuid = "13072b0f-2c55-5437-9ae7-d433b7a33950" version = "1.0.0" [[AxisArrays]] -deps = ["Compat", "Dates", "IntervalSets", "IterTools", "Random", "RangeArrays", "Test"] -git-tree-sha1 = "2e2536e9e6f27c4f8d09d8442b61a7ae0b910c28" +deps = ["Dates", "IntervalSets", "IterTools", "RangeArrays"] +git-tree-sha1 = "d63ba0315a1d287c9467e61e932578f2fdd048e0" uuid = "39de3d68-74b9-583c-8d2d-e117c070f3a9" -version = "0.3.0" +version = "0.3.3" [[Base64]] uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" [[BinDeps]] -deps = ["Compat", "Libdl", "SHA", "URIParser"] -git-tree-sha1 = "12093ca6cdd0ee547c39b1870e0c9c3f154d9ca9" +deps = ["Libdl", "Pkg", "SHA", "URIParser", "Unicode"] +git-tree-sha1 = "66158ad56b4bf6cc8413b37d0b7bc52402682764" uuid = "9e28174c-4ba2-5203-b857-d8d62c4213ee" -version = "0.8.10" +version = "1.0.0" [[BinaryProvider]] deps = ["Libdl", "SHA"] -git-tree-sha1 = "c7361ce8a2129f20b0e05a89f7070820cfed6648" +git-tree-sha1 = "5b08ed6036d9d3f0ee6369410b830f8873d4024c" uuid = "b99e7846-7c00-51b0-8f62-c81ae34c0232" -version = "0.5.4" +version = "0.5.8" -[[BufferedStreams]] -deps = ["Compat", "Test"] -git-tree-sha1 = "5d55b9486590fdda5905c275bb21ce1f0754020f" -uuid = "e1450e63-4bb3-523b-b2a4-4ffa8c0fd77d" -version = "1.0.0" +[[Bzip2_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "92463331a641b19fc3baa427e0b76cdbd54dc05d" +uuid = "6e34b625-4abd-537c-b88f-471c36dfa7a0" +version = "1.0.6+1" -[[CSTParser]] -deps = ["Tokenize"] -git-tree-sha1 = "376a39f1862000442011390f1edf5e7f4dcc7142" -uuid = "00ebfdb7-1f24-5e51-bd34-a7502290713f" -version = "0.6.0" +[[CEnum]] +git-tree-sha1 = "62847acab40e6855a9b5905ccb99c2b5cf6b3ebb" +uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82" +version = "0.2.0" + +[[CUDAapi]] +deps = ["Libdl", "Logging"] +git-tree-sha1 = "e063efb91cfefd7e6afd92c435d01398107a500b" +uuid = "3895d2a7-ec45-59b8-82bb-cfc6a382f9b3" +version = "1.2.0" + +[[CUDAdrv]] +deps = ["CUDAapi", "Libdl", "Printf"] +git-tree-sha1 = "9ce99b5732c70e06ed97c042187baed876fb1698" +uuid = "c5f51814-7f29-56b8-a69c-e4d8f6be1fde" +version = "3.1.0" + +[[CUDAnative]] +deps = ["Adapt", "CUDAapi", "CUDAdrv", "DataStructures", "InteractiveUtils", "LLVM", "Libdl", "Logging", "Printf", "TimerOutputs"] +git-tree-sha1 = "3d6427f28430730c0e4107d8f26c4943a9a142dc" +uuid = "be33ccc6-a3ff-5ff2-a52e-74243cff1e17" +version = "2.4.0" [[Cairo]] -deps = ["BinDeps", "Colors", "Compat", "Graphics", "Homebrew", "Libdl", "Pkg", "WinRPM"] -git-tree-sha1 = "e577cb41e7e2641f4b97f1b6dbcb9a69c9288330" +deps = ["Cairo_jll", "Colors", "Fontconfig_jll", "Glib_jll", "Graphics", "Libdl", "Pango_jll"] +git-tree-sha1 = "639345ddd8d7f52bbca32a847d74a5a497c4e4bc" uuid = "159f3aea-2a34-519c-b102-8c37f9878175" -version = "0.6.0" +version = "1.0.2" + +[[Cairo_jll]] +deps = ["Bzip2_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "LZO_jll", "Libdl", "Pixman_jll", "Pkg", "X11_jll", "Xorg_libXext_jll", "Xorg_libXrender_jll", "Zlib_jll", "libpng_jll"] +git-tree-sha1 = "a414ebdc4bd60b0fda62797f47c7409670c4ad1c" +uuid = "83423d85-b0ee-5818-9007-b63ccbeb887a" +version = "1.16.0+2" [[CatIndices]] deps = ["CustomUnitRanges", "OffsetArrays", "Test"] @@ -70,28 +117,28 @@ uuid = "aafaddc9-749c-510e-ac4f-586e18779b91" version = "0.2.0" [[CodecZlib]] -deps = ["BinaryProvider", "Libdl", "Test", "TranscodingStreams"] -git-tree-sha1 = "36bbf5374c661054d41410dc53ff752972583b9b" +deps = ["BinaryProvider", "Libdl", "TranscodingStreams"] +git-tree-sha1 = "05916673a2627dd91b4969ff8ba6941bc85a960e" uuid = "944b1d66-785c-5afd-91f1-9de20f533193" -version = "0.5.2" +version = "0.6.0" [[ColorTypes]] deps = ["FixedPointNumbers", "Random"] -git-tree-sha1 = "10050a24b09e8e41b951e9976b109871ce98d965" +git-tree-sha1 = "b9de8dc6106e09c79f3f776c27c62360d30e5eb8" uuid = "3da002f7-5984-5a60-b8a6-cbb66c0b333f" -version = "0.8.0" +version = "0.9.1" [[ColorVectorSpace]] deps = ["ColorTypes", "Colors", "FixedPointNumbers", "LinearAlgebra", "SpecialFunctions", "Statistics", "StatsBase"] -git-tree-sha1 = "d626b542f9c3aa22b4f82fe3d36a0ff56fc58601" +git-tree-sha1 = "59a561934dc2ad6aba4f500aa839c0f1b8fa8314" uuid = "c3611d14-8923-5661-9e6a-0046d554d3a4" -version = "0.7.0" +version = "0.8.2" [[Colors]] -deps = ["ColorTypes", "FixedPointNumbers", "InteractiveUtils", "Printf", "Reexport", "Test"] -git-tree-sha1 = "9f0a0210450acb91c730b730a994f8eef1d3d543" +deps = ["ColorTypes", "FixedPointNumbers", "InteractiveUtils", "Printf", "Reexport"] +git-tree-sha1 = "177d8b959d3c103a6d57574c38ee79c81059c31b" uuid = "5ae59095-9a9b-59fe-a467-6f913c188581" -version = "0.9.5" +version = "0.11.2" [[CommonSubexpressions]] deps = ["Test"] @@ -101,9 +148,9 @@ version = "0.2.0" [[Compat]] deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"] -git-tree-sha1 = "84aa74986c5b9b898b0d1acaf3258741ee64754f" +git-tree-sha1 = "ed2c4abadf84c53d9e58510b5fc48912c2336fbb" uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" -version = "2.1.0" +version = "2.2.0" [[ComputationalResources]] deps = ["Test"] @@ -111,23 +158,17 @@ git-tree-sha1 = "89e7e7ed20af73d9f78877d2b8d1194e7b6ff13d" uuid = "ed09eef8-17a6-5b46-8889-db040fac31e3" version = "0.3.0" -[[Conda]] -deps = ["JSON", "VersionParsing"] -git-tree-sha1 = "9a11d428dcdc425072af4aea19ab1e8c3e01c032" -uuid = "8f4d0f93-b110-5947-807f-2305c1781a2d" -version = "1.3.0" - [[CoordinateTransformations]] deps = ["Compat", "Rotations", "StaticArrays"] git-tree-sha1 = "47f05d0b7f4999609f92e657147df000818c1f24" uuid = "150eb455-5306-5404-9cee-2592286d6298" version = "0.5.0" -[[Crayons]] -deps = ["Test"] -git-tree-sha1 = "f621b8ef51fd2004c7cf157ea47f027fdeac5523" -uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" -version = "4.0.0" +[[CuArrays]] +deps = ["AbstractFFTs", "Adapt", "CUDAapi", "CUDAdrv", "CUDAnative", "GPUArrays", "LinearAlgebra", "MacroTools", "NNlib", "Printf", "Random", "Requires", "SparseArrays", "TimerOutputs"] +git-tree-sha1 = "46b48742a84bb839e74215b7e468a4a1c6ba30f9" +uuid = "3a865a2d-5b23-5a0f-bc46-62713ec82fae" +version = "1.2.1" [[CustomUnitRanges]] deps = ["Test"] @@ -135,106 +176,195 @@ git-tree-sha1 = "0a106457a1831555857e18ac9617279c22fc393b" uuid = "dc8bdbbb-1ca9-579f-8c36-e416f6a65cce" version = "0.2.0" +[[DataAPI]] +git-tree-sha1 = "674b67f344687a88310213ddfa8a2b3c76cc4252" +uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" +version = "1.1.0" + [[DataStructures]] -deps = ["InteractiveUtils", "OrderedCollections", "Random", "Serialization", "Test"] -git-tree-sha1 = "ca971f03e146cf144a9e2f2ce59674f5bf0e8038" +deps = ["InteractiveUtils", "OrderedCollections"] +git-tree-sha1 = "b7720de347734f4716d1815b00ce5664ed6bbfd4" uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" -version = "0.15.0" +version = "0.17.9" [[Dates]] deps = ["Printf"] uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" +[[Dbus_jll]] +deps = ["Expat_jll", "Libdl", "Pkg"] +git-tree-sha1 = "72ff4d947383cc009e02d322f14c7653f7ab4b21" +uuid = "ee1fde0b-3d02-5ea6-8484-8dfef6360eab" +version = "1.12.16+0" + [[DelimitedFiles]] deps = ["Mmap"] uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab" [[DiffResults]] -deps = ["Compat", "StaticArrays"] -git-tree-sha1 = "34a4a1e8be7bc99bc9c611b895b5baf37a80584c" +deps = ["StaticArrays"] +git-tree-sha1 = "da24935df8e0c6cf28de340b958f6aac88eaa0cc" uuid = "163ba53b-c6d8-5494-b064-1a9d43ac40c5" -version = "0.0.4" +version = "1.0.2" [[DiffRules]] -deps = ["Random", "Test"] -git-tree-sha1 = "dc0869fb2f5b23466b32ea799bd82c76480167f7" +deps = ["NaNMath", "Random", "SpecialFunctions"] +git-tree-sha1 = "10dca52cf6d4a62d82528262921daf63b99704a2" uuid = "b552c78f-8df3-52c6-915a-8e097449b14b" -version = "0.0.10" +version = "1.0.0" [[Distances]] -deps = ["LinearAlgebra", "Printf", "Random", "Statistics", "Test"] -git-tree-sha1 = "a135c7c062023051953141da8437ed74f89d767a" +deps = ["LinearAlgebra", "Statistics"] +git-tree-sha1 = "23717536c81b63e250f682b0e0933769eecd1411" uuid = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7" -version = "0.8.0" +version = "0.8.2" [[Distributed]] deps = ["Random", "Serialization", "Sockets"] uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" +[[Distributions]] +deps = ["LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SpecialFunctions", "Statistics", "StatsBase", "StatsFuns"] +git-tree-sha1 = "51d184211a807ddba5f48d06098e93986759ec43" +uuid = "31c24e10-a181-5473-b8eb-7969acd0382f" +version = "0.21.9" + +[[Expat_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "e1b4190be75ba8d03ce2bec3a6cfc7a939c713ea" +uuid = "2e619515-83b5-522b-bb60-26c02a35a201" +version = "2.2.7+0" + [[FFTViews]] -deps = ["CustomUnitRanges", "FFTW", "Test"] -git-tree-sha1 = "9d7993227ca7c0fdb6b31deef193adbba11c8f4e" +deps = ["CustomUnitRanges", "FFTW"] +git-tree-sha1 = "1ba17a68391b2b3a0626ffb597c45e482cf7b5f0" uuid = "4f61f5a4-77b1-5117-aa51-3ab5ef4ef0cd" -version = "0.2.0" +version = "0.3.0" [[FFTW]] -deps = ["AbstractFFTs", "BinaryProvider", "Compat", "Conda", "Libdl", "LinearAlgebra", "Reexport", "Test"] -git-tree-sha1 = "29cda58afbf62f35b1a094882ad6c745a47b2eaa" +deps = ["AbstractFFTs", "FFTW_jll", "IntelOpenMP_jll", "Libdl", "LinearAlgebra", "MKL_jll", "Reexport"] +git-tree-sha1 = "109d82fa4b00429f9afcce873e9f746f11f018d3" uuid = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" -version = "0.2.4" +version = "1.2.0" + +[[FFTW_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "05674f209a6e3387dd103a945b0113eeb64b1a58" +uuid = "f5851436-0d7a-5f13-b9de-f02708fd171a" +version = "3.3.9+3" [[FileIO]] deps = ["Pkg"] -git-tree-sha1 = "351f001a78aa1b7ad2696e386e110b5abd071c71" +git-tree-sha1 = "74585bf1f7ed7259e166011e89f49363d7fa89a6" uuid = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" -version = "1.0.7" +version = "1.2.1" [[FillArrays]] -deps = ["LinearAlgebra", "Random", "SparseArrays", "Test"] -git-tree-sha1 = "9ab8f76758cbabba8d7f103c51dce7f73fcf8e92" +deps = ["LinearAlgebra", "Random", "SparseArrays"] +git-tree-sha1 = "de38b0253ade98340fabaf220f368f6144541938" uuid = "1a297f60-69ca-5386-bcde-b61e274b549b" -version = "0.6.3" +version = "0.7.4" [[FixedPointNumbers]] -git-tree-sha1 = "d14a6fa5890ea3a7e5dcab6811114f132fec2b4b" +git-tree-sha1 = "4aaea64dd0c30ad79037084f8ca2b94348e65eaa" uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93" -version = "0.6.1" +version = "0.7.1" [[Flux]] -deps = ["AbstractTrees", "Adapt", "CodecZlib", "Colors", "DelimitedFiles", "Juno", "LinearAlgebra", "MacroTools", "NNlib", "Pkg", "Printf", "Random", "Reexport", "Requires", "SHA", "Statistics", "StatsBase", "Tracker", "ZipFile"] -git-tree-sha1 = "08212989c2856f95f90709ea5fd824bd27b34514" +deps = ["AbstractTrees", "Adapt", "CUDAapi", "CodecZlib", "Colors", "CuArrays", "DelimitedFiles", "Juno", "LinearAlgebra", "MacroTools", "NNlib", "Pkg", "Printf", "Random", "Reexport", "SHA", "Statistics", "StatsBase", "Tracker", "ZipFile"] +git-tree-sha1 = "b5ebbd896dcd8ff19c6cb7297c4d323155b26bcf" uuid = "587475ba-b771-5e3f-ad9e-33799f191a9c" -version = "0.8.3" +version = "0.9.0" + +[[Fontconfig_jll]] +deps = ["Bzip2_jll", "Expat_jll", "FreeType2_jll", "Libdl", "Libuuid_jll", "Pkg", "Zlib_jll"] +git-tree-sha1 = "6fb165331f35ac6a778dcb96d672b98626249dec" +uuid = "a3f928ae-7b40-5064-980b-68af3947d34b" +version = "2.13.1+9" [[ForwardDiff]] -deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "InteractiveUtils", "LinearAlgebra", "NaNMath", "Random", "SparseArrays", "SpecialFunctions", "StaticArrays", "Test"] -git-tree-sha1 = "4c4d727f1b7e0092134fabfab6396b8945c1ea5b" +deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "NaNMath", "Random", "SpecialFunctions", "StaticArrays"] +git-tree-sha1 = "88b082d492be6b63f967b6c96b352e25ced1a34c" uuid = "f6369f11-7733-5829-9624-2563aa707210" -version = "0.10.3" +version = "0.10.9" + +[[FreeType2_jll]] +deps = ["Bzip2_jll", "Libdl", "Pkg", "Zlib_jll"] +git-tree-sha1 = "8e290780d75bc0f676548c3bb84c153f83d14bdc" +uuid = "d7e528f0-a631-5988-bf34-fe36492bcfd7" +version = "2.10.1+1" + +[[FriBidi_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "e479537bf8a8f060c546553c14fd0633978dda7e" +uuid = "559328eb-81f9-559d-9380-de523a88c83c" +version = "1.0.5+2" + +[[GPUArrays]] +deps = ["Adapt", "FFTW", "FillArrays", "LinearAlgebra", "Printf", "Random", "Serialization", "Test"] +git-tree-sha1 = "8d74ced24448c52b539a23d107bd2424ee139c0f" +uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7" +version = "1.0.4" + +[[GTK3_jll]] +deps = ["ATK_jll", "Cairo_jll", "Fontconfig_jll", "FreeType2_jll", "FriBidi_jll", "Glib_jll", "Graphene_jll", "HarfBuzz_jll", "Libdl", "Libepoxy_jll", "Pango_jll", "Pkg", "Wayland_jll", "Xorg_libX11_jll", "Xorg_libXcomposite_jll", "Xorg_libXcursor_jll", "Xorg_libXdamage_jll", "Xorg_libXext_jll", "Xorg_libXfixes_jll", "Xorg_libXi_jll", "Xorg_libXinerama_jll", "Xorg_libXrandr_jll", "Xorg_libXrender_jll", "adwaita_icon_theme_jll", "at_spi2_atk_jll", "gdk_pixbuf_jll", "iso_codes_jll", "xkbcommon_jll"] +git-tree-sha1 = "19f11cf4e1b25ed45adb1c26c9877ebc212aca99" +uuid = "77ec8976-b24b-556a-a1bf-49a033a670a6" +version = "3.24.11+1" + +[[Gettext_jll]] +deps = ["Libdl", "Libiconv_jll", "Pkg", "XML2_jll"] +git-tree-sha1 = "0b6b238d315698441a9bf24ea9b20ac19cc7e513" +uuid = "78b55507-aeef-58d4-861c-77aaff3498b1" +version = "0.20.1+1" + +[[Glib_jll]] +deps = ["Gettext_jll", "Libdl", "Libffi_jll", "PCRE_jll", "Pkg", "Zlib_jll"] +git-tree-sha1 = "9a7e1ad28f1313da50e5ab8568725bed5f131ad1" +uuid = "7746bdde-850d-59dc-9ae8-88ece973131d" +version = "2.59.0+1" + +[[Graphene_jll]] +deps = ["Glib_jll", "Libdl", "Pkg"] +git-tree-sha1 = "a46e501da1b4a006fb26fd03eb6d927f7d112604" +uuid = "75302f13-0b7e-5bab-a6d1-23fa92e4c2ea" +version = "1.10.0+1" [[Graphics]] -deps = ["Colors", "Compat", "NaNMath"] -git-tree-sha1 = "e3ead4211073d4117a0d2ef7d1efc5c8092c8412" +deps = ["Colors", "LinearAlgebra", "NaNMath"] +git-tree-sha1 = "a9ca650f074942cadb98d59198f93b9adc11d1ea" uuid = "a2bd30eb-e257-5431-a919-1863eab51364" -version = "0.4.0" - -[[HTTPClient]] -deps = ["Compat", "LibCURL"] -git-tree-sha1 = "161d5776ae8e585ac0b8c20fb81f17ab755b3671" -uuid = "0862f596-cf2d-50af-8ef4-f2be67dfa83f" -version = "0.2.1" +version = "1.0.1" + +[[Graphite2_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "6fad1258f2c9eacb86e8726d38c2ad6549f9d91e" +uuid = "3b182d85-2403-5c21-9c21-1e1f0cc25472" +version = "1.3.13+1" + +[[Gtk]] +deps = ["Cairo", "Cairo_jll", "Dates", "GTK3_jll", "Glib_jll", "Graphics", "Libdl", "Pkg", "Reexport", "Serialization", "Test", "Xorg_xkeyboard_config_jll", "adwaita_icon_theme_jll", "gdk_pixbuf_jll", "hicolor_icon_theme_jll"] +git-tree-sha1 = "ab2676a65345f8813be07675aa657464f98e6704" +uuid = "4c0ca9eb-093a-5379-98c5-f87ac0bbbf44" +version = "1.1.2" + +[[GymSpaces]] +deps = ["DataStructures", "Random"] +git-tree-sha1 = "c34b145dbd74d37d2ba974797823448e55599904" +uuid = "4896a6fa-884d-11e9-35a0-9763e69c01ff" +version = "0.2.0" -[[Homebrew]] -deps = ["BinDeps", "InteractiveUtils", "JSON", "Libdl", "Test", "Unicode"] -git-tree-sha1 = "f01fb2f34675f9839d55ba7238bab63ebd2e531e" -uuid = "d9be37ee-ecc9-5288-90f1-b9ca67657a75" -version = "0.7.1" +[[HarfBuzz_jll]] +deps = ["Cairo_jll", "Fontconfig_jll", "FreeType2_jll", "Gettext_jll", "Glib_jll", "Graphite2_jll", "Libdl", "Libffi_jll", "Pkg"] +git-tree-sha1 = "dbc6a8c98abe861dd928bc2b603b4ee1eed41dcd" +uuid = "2e76f6c2-a576-52d4-95c1-20adfe4de566" +version = "2.6.1+4" [[IRTools]] deps = ["InteractiveUtils", "MacroTools", "Test"] -git-tree-sha1 = "a9b1fc7745ae4745a634bbb6d1cb7fd64e37248a" +git-tree-sha1 = "72421971e60917b8cd7737f9577c4f0f87eab306" uuid = "7869d1d1-7146-5819-86e3-90919afe41df" -version = "0.2.2" +version = "0.3.0" [[IdentityRanges]] deps = ["OffsetArrays", "Test"] @@ -243,64 +373,69 @@ uuid = "bbac6d45-d8f3-5730-bfe4-7a449cd117ca" version = "0.3.0" [[ImageAxes]] -deps = ["AxisArrays", "Colors", "FixedPointNumbers", "ImageCore", "MappedArrays", "Reexport", "SimpleTraits"] -git-tree-sha1 = "8109bb9a28deca29a5b938ebfa7d0a75319d8776" +deps = ["AxisArrays", "ImageCore", "MappedArrays", "Reexport", "SimpleTraits"] +git-tree-sha1 = "75b0225bd999c4bdd83144f4b1b2437890050892" uuid = "2803e5a7-5153-5ecf-9a86-9b4c37f5f5ac" -version = "0.6.0" +version = "0.6.2" [[ImageCore]] -deps = ["ColorTypes", "Colors", "FFTW", "FixedPointNumbers", "Graphics", "MappedArrays", "OffsetArrays", "PaddedViews", "Reexport"] -git-tree-sha1 = "2fb7a0d80ab6bf0c5a8b0a189bca60311c73a605" +deps = ["Colors", "FixedPointNumbers", "Graphics", "MappedArrays", "OffsetArrays", "PaddedViews", "Reexport", "Requires"] +git-tree-sha1 = "eb3c4a3341a3ea918a93a64a5e4d856a84f89489" uuid = "a09fc81d-aa75-5fe9-8630-4744c3626534" -version = "0.8.4" +version = "0.8.10" [[ImageDistances]] -deps = ["ColorVectorSpace", "Colors", "Distances", "FixedPointNumbers", "ImageCore"] -git-tree-sha1 = "aa69ce81260bcb5e950a5e3b48ccca15447c6d8c" +deps = ["ColorVectorSpace", "Distances", "ImageCore", "LinearAlgebra", "MappedArrays", "Statistics"] +git-tree-sha1 = "cf9b02b9f5e33c768c223de6d8f7d1b6d0cf4136" uuid = "51556ac3-7006-55f5-8cb3-34580c88182d" -version = "0.2.4" +version = "0.2.7" [[ImageFiltering]] -deps = ["CatIndices", "ColorVectorSpace", "Colors", "ComputationalResources", "DataStructures", "FFTViews", "FFTW", "FixedPointNumbers", "ImageCore", "LinearAlgebra", "MappedArrays", "OffsetArrays", "Requires", "StaticArrays", "Statistics", "TiledIteration"] -git-tree-sha1 = "ea26aea659b889da49f51afe2886358990ef9732" +deps = ["CatIndices", "ColorVectorSpace", "ComputationalResources", "DataStructures", "FFTViews", "FFTW", "ImageCore", "LinearAlgebra", "MappedArrays", "OffsetArrays", "Requires", "StaticArrays", "Statistics", "TiledIteration"] +git-tree-sha1 = "06f2f3b48f859a44b045c1a0b6ff71058923414f" uuid = "6a3955dd-da59-5b1f-98d4-e7296123deb5" -version = "0.6.3" +version = "0.6.9" [[ImageMetadata]] -deps = ["AxisArrays", "ColorVectorSpace", "Colors", "FixedPointNumbers", "ImageAxes", "ImageCore", "IndirectArrays"] -git-tree-sha1 = "6522fdedc8aace8b8dc9f7adc7f6079bc51c86e2" +deps = ["AxisArrays", "ColorVectorSpace", "ImageAxes", "ImageCore", "IndirectArrays"] +git-tree-sha1 = "5bedd77cb80ce2c51889414cd2de9e300217ee8d" uuid = "bc367c6b-8a6b-528e-b4bd-a4b897500b49" -version = "0.7.1" +version = "0.9.0" [[ImageMorphology]] -deps = ["Colors", "FixedPointNumbers", "ImageCore"] -git-tree-sha1 = "2ea6e55a36166321ca735b8eaf74936180e2ad5d" +deps = ["ImageCore"] +git-tree-sha1 = "e41dd25ebf8de738a8ff4f1058a1823a7fe509cf" uuid = "787d08f9-d448-5407-9aad-5290dd7ab264" -version = "0.2.4" +version = "0.2.5" [[ImageShow]] -deps = ["Base64", "ColorTypes", "Colors", "FileIO", "FixedPointNumbers", "ImageCore", "OffsetArrays", "Requires"] -git-tree-sha1 = "c23323afc82b6b553e6b2244d531e50858ea392c" +deps = ["Base64", "FileIO", "ImageCore", "OffsetArrays", "Requires"] +git-tree-sha1 = "8544e64c9cffd0d59e49a574851ac13bb2cbe6a7" uuid = "4e3cecfd-b093-5904-9786-8bbb286a6a31" -version = "0.2.0" +version = "0.2.2" [[ImageTransformations]] -deps = ["AxisAlgorithms", "ColorTypes", "ColorVectorSpace", "Colors", "CoordinateTransformations", "FixedPointNumbers", "IdentityRanges", "ImageCore", "Interpolations", "OffsetArrays", "StaticArrays"] -git-tree-sha1 = "4cf03fc72d8877d5e58c1c72d176340ad4e28fda" +deps = ["AxisAlgorithms", "ColorVectorSpace", "CoordinateTransformations", "IdentityRanges", "ImageCore", "Interpolations", "OffsetArrays", "StaticArrays"] +git-tree-sha1 = "c4f958d2c5ca2aa11d29c1b8d50caf73602b667c" uuid = "02fcd773-0e25-5acc-982a-7f6622650795" -version = "0.8.0" +version = "0.8.1" [[Images]] -deps = ["AxisArrays", "Base64", "ColorTypes", "ColorVectorSpace", "Colors", "FileIO", "FixedPointNumbers", "Graphics", "ImageAxes", "ImageCore", "ImageDistances", "ImageFiltering", "ImageMetadata", "ImageMorphology", "ImageShow", "ImageTransformations", "IndirectArrays", "MappedArrays", "OffsetArrays", "Reexport", "SparseArrays", "StaticArrays", "Statistics", "StatsBase", "TiledIteration"] -git-tree-sha1 = "1aae59fc0e34e47dfb111113bb6e9a65d7bafe81" +deps = ["AxisArrays", "Base64", "ColorVectorSpace", "FileIO", "Graphics", "ImageAxes", "ImageCore", "ImageDistances", "ImageFiltering", "ImageMetadata", "ImageMorphology", "ImageShow", "ImageTransformations", "IndirectArrays", "MappedArrays", "OffsetArrays", "Random", "Reexport", "SparseArrays", "StaticArrays", "Statistics", "StatsBase", "TiledIteration"] +git-tree-sha1 = "de1bbdcda72f590289409e0eee66c4178135991b" uuid = "916415d5-f1e6-5110-898d-aaa5f9f070e0" -version = "0.18.0" +version = "0.20.1" [[IndirectArrays]] -deps = ["Compat", "Test"] -git-tree-sha1 = "b6e249be10a3381b2c72ac82f2d13d70067cb2bd" +git-tree-sha1 = "c2a145a145dc03a7620af1444e0264ef907bd44f" uuid = "9b13fd28-a010-5f03-acff-a1bbcff69959" -version = "0.5.0" +version = "0.5.1" + +[[IntelOpenMP_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "fb8e1c7a5594ba56f9011310790e03b5384998d6" +uuid = "1d5cc7b8-4909-519e-a0f8-d0f5ad9712d0" +version = "2018.0.3+0" [[InteractiveUtils]] deps = ["Markdown"] @@ -308,45 +443,44 @@ uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" [[Interpolations]] deps = ["AxisAlgorithms", "LinearAlgebra", "OffsetArrays", "Random", "Ratios", "SharedArrays", "SparseArrays", "StaticArrays", "WoodburyMatrices"] -git-tree-sha1 = "e1bac96b5ef3ea23b50e801b4a988ec21861a47f" +git-tree-sha1 = "e4a219889f33a8c2435c26bf1bca6f4fbf8c1c0b" uuid = "a98d9a8b-a2ab-59e6-89dd-64a1c18fca59" -version = "0.12.2" +version = "0.12.6" [[IntervalSets]] -deps = ["Compat"] -git-tree-sha1 = "9dc556002f23740de13946e8c2e41798e09a9249" +deps = ["Dates", "Statistics"] +git-tree-sha1 = "4214b48a62eb8f2c292b2ee34a508c256c0cdbc9" uuid = "8197267c-284f-5f27-9208-e0e47529a953" -version = "0.3.1" +version = "0.3.2" [[IterTools]] -deps = ["SparseArrays", "Test"] -git-tree-sha1 = "79246285c43602384e6f1943b3554042a3712056" +git-tree-sha1 = "05110a2ab1fc5f932622ffea2a003221f4782c18" uuid = "c8e1da08-722c-5040-9ed9-7db0dc04731e" -version = "1.1.1" +version = "1.3.0" -[[JSON]] -deps = ["Dates", "Distributed", "Mmap", "Sockets", "Test", "Unicode"] -git-tree-sha1 = "1f7a25b53ec67f5e9422f1f551ee216503f4a0fa" -uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" -version = "0.20.0" +[[JpegTurbo_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "b007cedc10bb017cdae3af0062fe4dd29b483c70" +uuid = "aacddb02-875f-59d6-b918-886e6ef4fbf8" +version = "2.0.1+0" [[Juno]] deps = ["Base64", "Logging", "Media", "Profile", "Test"] -git-tree-sha1 = "4e4a8d43aa7ecec66cadaf311fbd1e5c9d7b9175" +git-tree-sha1 = "30d94657a422d09cb97b6f86f04f750fa9c50df8" uuid = "e5e0dc1b-0480-54bc-9374-aad01c23163d" -version = "0.7.0" +version = "0.7.2" -[[LibCURL]] -deps = ["BinaryProvider", "Libdl"] -git-tree-sha1 = "fd5fc15f2a04608fe1435a769dbbfc7959ff1daa" -uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" -version = "0.5.2" +[[LLVM]] +deps = ["CEnum", "Libdl", "Printf", "Unicode"] +git-tree-sha1 = "1d08d7e4250f452f6cb20e4574daaebfdbee0ff7" +uuid = "929cbde3-209d-540e-8aea-75f648917ca0" +version = "1.3.3" -[[LibExpat]] -deps = ["Compat"] -git-tree-sha1 = "fde352ec13479e2f90e57939da2440fb78c5e388" -uuid = "522f3ed2-3f36-55e3-b6df-e94fee9b0c07" -version = "0.5.0" +[[LZO_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "26ff20dc58051d7eda388662b90049d192f8d0a1" +uuid = "dd4b983a-f0e5-5f8d-a1b7-129d4a5fb1ac" +version = "2.10.0+0" [[LibGit2]] uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" @@ -354,11 +488,53 @@ uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" [[Libdl]] uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" -[[Libz]] -deps = ["BufferedStreams", "Random", "Test"] -git-tree-sha1 = "d405194ffc0293c3519d4f7251ce51baac9cc871" -uuid = "2ec943e9-cfe8-584d-b93d-64dcb6d567b7" -version = "1.0.0" +[[Libepoxy_jll]] +deps = ["Libdl", "Libglvnd_jll", "Pkg", "X11_jll", "Xorg_libX11_jll"] +git-tree-sha1 = "c9346f231e9279e810099f61313fd23fc14215e5" +uuid = "42c93a91-0102-5b3f-8f9d-e41de60ac950" +version = "1.5.3+1" + +[[Libffi_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "f2c12e7da9c4f7fab577619626b772f607e67b04" +uuid = "e9f186c6-92d2-5b65-8a66-fee21dc1b490" +version = "3.2.1+0" + +[[Libgcrypt_jll]] +deps = ["Libdl", "Libgpg_error_jll", "Pkg"] +git-tree-sha1 = "9592d031aac2566fb5cf9eeb28e2d7b05db9b9ef" +uuid = "d4300ac3-e22c-5743-9152-c294e39db1e4" +version = "1.8.5+0" + +[[Libglvnd_jll]] +deps = ["Libdl", "Pkg", "X11_jll", "Xorg_glproto_jll", "Xorg_libX11_jll", "Xorg_libXext_jll"] +git-tree-sha1 = "d3e5ad7af102c4fc1b58ba52952d8cc31bc6ee5b" +uuid = "7e76a0d4-f3c7-5321-8279-8d96eeed0f29" +version = "1.2.0+1" + +[[Libgpg_error_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "3332d872d131a6ee88f10f20d7d131ece886c424" +uuid = "7add5ba3-2f88-524e-9cd5-f83b8a55f7b8" +version = "1.36.0+0" + +[[Libiconv_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "48563fe30f00c9d4a4d61891e71df389bf901142" +uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531" +version = "1.16.0+0" + +[[Libtiff_jll]] +deps = ["JpegTurbo_jll", "Libdl", "Pkg", "Zlib_jll", "Zstd_jll"] +git-tree-sha1 = "6cab365abd0108b45e1cbd234223152d88647742" +uuid = "89763e89-9b03-5906-acba-b20f662cd828" +version = "4.0.10+0" + +[[Libuuid_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "5b0aaf3f931102e18b8b4b1590fe69708d3eb7d6" +uuid = "38a345b3-de98-5d2b-a5d3-14cd9215e700" +version = "2.34.0+3" [[LinearAlgebra]] deps = ["Libdl"] @@ -367,17 +543,22 @@ uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" [[Logging]] uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" +[[MKL_jll]] +deps = ["IntelOpenMP_jll", "Libdl", "Pkg"] +git-tree-sha1 = "720629cc8cbd12c146ca01b661fd1a6cf66e2ff4" +uuid = "856f044c-d86e-5d09-b602-aeab76dc8ba7" +version = "2019.0.117+2" + [[MacroTools]] -deps = ["CSTParser", "Compat", "DataStructures", "Test"] -git-tree-sha1 = "daecd9e452f38297c686eba90dba2a6d5da52162" +deps = ["DataStructures", "Markdown", "Random"] +git-tree-sha1 = "e2fc7a55bb2224e203bbd8b59f72b91323233458" uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" -version = "0.5.0" +version = "0.5.3" [[MappedArrays]] -deps = ["Test"] -git-tree-sha1 = "923441c5ac942b60bd3a842d5377d96646bcbf46" +git-tree-sha1 = "e2a02fe7ee86a10c707ff1756ab1650b40b140bb" uuid = "dbb5928d-eab1-5f90-85c2-b9b0edb7c900" -version = "0.2.1" +version = "0.2.2" [[Markdown]] deps = ["Base64"] @@ -390,30 +571,35 @@ uuid = "e89f7d12-3494-54d1-8411-f7d8b9ae1f27" version = "0.5.0" [[Missings]] -deps = ["SparseArrays", "Test"] -git-tree-sha1 = "f0719736664b4358aa9ec173077d4285775f8007" +deps = ["DataAPI"] +git-tree-sha1 = "de0a5ce9e5289f27df672ffabef4d1e5861247d5" uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" -version = "0.4.1" +version = "0.4.3" [[Mmap]] uuid = "a63ad114-7e13-5084-954f-fe012c677804" [[NNlib]] -deps = ["Libdl", "LinearAlgebra", "Requires", "Statistics", "TimerOutputs"] -git-tree-sha1 = "0c667371391fc6bb31f7f12f96a56a17098b3de8" +deps = ["BinaryProvider", "Libdl", "LinearAlgebra", "Requires", "Statistics"] +git-tree-sha1 = "755c0bab3912ff782167e1b4b774b833f8a0e550" uuid = "872c559c-99b0-510c-b3b7-b6c96a88d5cd" -version = "0.6.0" +version = "0.6.4" [[NaNMath]] -deps = ["Compat"] -git-tree-sha1 = "ce3b85e484a5d4c71dd5316215069311135fa9f2" +git-tree-sha1 = "928b8ca9b2791081dc71a51c55347c27c618760f" uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" -version = "0.3.2" +version = "0.3.3" [[OffsetArrays]] -git-tree-sha1 = "1af2f79c7eaac3e019a0de41ef63335ff26a0a57" +git-tree-sha1 = "87d0a91efe29352d5caaa271ae3927083c096e33" uuid = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" -version = "0.11.1" +version = "0.11.4" + +[[OpenBLAS_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "3a6e5767e8ae022871c19162cc3ecd80748bd3dc" +uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" +version = "0.3.7+4" [[OrderedCollections]] deps = ["Random", "Serialization", "Test"] @@ -421,14 +607,38 @@ git-tree-sha1 = "c4c13474d23c60d20a67b217f1d7f22a40edf8f1" uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" version = "1.1.0" +[[PCRE_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "859aa38f2823d45940312575bca3c856f4a1fbab" +uuid = "2f80f16e-611a-54ab-bc61-aa92de5b98fc" +version = "8.42.0+1" + +[[PDMats]] +deps = ["Arpack", "LinearAlgebra", "SparseArrays", "SuiteSparse", "Test"] +git-tree-sha1 = "035f8d60ba2a22cb1d2580b1e0e5ce0cb05e4563" +uuid = "90014a1f-27ba-587c-ab20-58faa44d9150" +version = "0.9.10" + [[PaddedViews]] deps = ["OffsetArrays", "Test"] git-tree-sha1 = "7da3e7e1a58cffbf10177553ae95f17b92516912" uuid = "5432bcbf-9aad-5242-b902-cca2824c8663" version = "0.4.2" +[[Pango_jll]] +deps = ["Cairo_jll", "Fontconfig_jll", "FreeType2_jll", "FriBidi_jll", "Glib_jll", "HarfBuzz_jll", "Libdl", "Pkg"] +git-tree-sha1 = "99ee35d5298b2615c800becfea2b3dc4fdb803c5" +uuid = "36c8627f-9965-5494-a995-c6b170f724f3" +version = "1.42.4+7" + +[[Pixman_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "cdddc5aab6ae8e3034dbf3c264e636fbf19fdee0" +uuid = "30392449-352a-5448-841d-b1acce4e97dc" +version = "0.38.4+1" + [[Pkg]] -deps = ["Dates", "LibGit2", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"] +deps = ["Dates", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" [[Printf]] @@ -439,6 +649,12 @@ uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" deps = ["Printf"] uuid = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79" +[[QuadGK]] +deps = ["DataStructures", "LinearAlgebra"] +git-tree-sha1 = "dc84e810393cfc6294248c9032a9cdacc14a3db4" +uuid = "1fd47b50-473d-5c70-9696-f719f8f3bcdc" +version = "2.3.1" + [[REPL]] deps = ["InteractiveUtils", "Markdown", "Sockets"] uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" @@ -448,10 +664,9 @@ deps = ["Serialization"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" [[RangeArrays]] -deps = ["Compat"] -git-tree-sha1 = "d925adfd5b01cb46fde89dc9548d167b3b136f4a" +git-tree-sha1 = "b9039e93773ddcfc828f12aadf7115b4b4d225f5" uuid = "b3c3ace0-ae52-54e7-9d0b-2c1406fd6b9d" -version = "0.3.1" +version = "0.3.2" [[Ratios]] deps = ["Compat"] @@ -466,16 +681,22 @@ uuid = "189a3867-3050-52da-a836-e630ba90ab69" version = "0.2.0" [[Requires]] -deps = ["Test"] -git-tree-sha1 = "f6fbf4ba64d295e146e49e021207993b6b48c7d1" +deps = ["UUIDs"] +git-tree-sha1 = "999513b7dea8ac17359ed50ae8ea089e4464e35e" uuid = "ae029012-a4dd-5104-9daa-d747884805df" -version = "0.5.2" +version = "1.0.0" + +[[Rmath]] +deps = ["BinaryProvider", "Libdl", "Random", "Statistics"] +git-tree-sha1 = "2bbddcb984a1d08612d0c4abb5b4774883f6fa98" +uuid = "79098fc4-a85e-5d69-aa6a-4863f24498fa" +version = "0.6.0" [[Rotations]] -deps = ["LinearAlgebra", "Random", "StaticArrays", "Statistics", "Test"] -git-tree-sha1 = "dfb3ceb177a59f25fee4e2f26c1aeb92b73d3a0e" +deps = ["LinearAlgebra", "StaticArrays", "Statistics"] +git-tree-sha1 = "d5f83867093db7319a9366d55f29280ecae9bcda" uuid = "6038ab10-8711-5258-84ad-4b1120ba62dc" -version = "0.11.1" +version = "0.13.0" [[SHA]] uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" @@ -488,10 +709,10 @@ deps = ["Distributed", "Mmap", "Random", "Serialization"] uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383" [[SimpleTraits]] -deps = ["InteractiveUtils", "MacroTools", "Test"] -git-tree-sha1 = "c0a542b8d5e369b179ccd296b2ca987f6da5da0a" +deps = ["InteractiveUtils", "MacroTools"] +git-tree-sha1 = "2bdf3b6300a9d66fe29ee8bb51ba100c4df9ecbc" uuid = "699a6c99-e7fa-54fc-8d76-47d257e15c1d" -version = "0.8.0" +version = "0.9.1" [[Sockets]] uuid = "6462fe0b-24de-5631-8697-dd941f90decc" @@ -507,26 +728,36 @@ deps = ["LinearAlgebra", "Random"] uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" [[SpecialFunctions]] -deps = ["BinDeps", "BinaryProvider", "Libdl", "Test"] -git-tree-sha1 = "0b45dc2e45ed77f445617b99ff2adf0f5b0f23ea" +deps = ["BinDeps", "BinaryProvider", "Libdl"] +git-tree-sha1 = "3bdd374b6fd78faf0119b8c5d538788dbf910c6e" uuid = "276daf66-3868-5448-9aa4-cd146d93841b" -version = "0.7.2" +version = "0.8.0" [[StaticArrays]] deps = ["LinearAlgebra", "Random", "Statistics"] -git-tree-sha1 = "db23bbf50064c582b6f2b9b043c8e7e98ea8c0c6" +git-tree-sha1 = "5a3bcb6233adabde68ebc97be66e95dcb787424c" uuid = "90137ffa-7385-5640-81b9-e52037218182" -version = "0.11.0" +version = "0.12.1" [[Statistics]] deps = ["LinearAlgebra", "SparseArrays"] uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" [[StatsBase]] -deps = ["DataStructures", "LinearAlgebra", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics"] -git-tree-sha1 = "8a0f4b09c7426478ab677245ab2b0b68552143c7" +deps = ["DataAPI", "DataStructures", "LinearAlgebra", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics"] +git-tree-sha1 = "c53e809e63fe5cf5de13632090bc3520649c9950" uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" -version = "0.30.0" +version = "0.32.0" + +[[StatsFuns]] +deps = ["Rmath", "SpecialFunctions"] +git-tree-sha1 = "79982835d2ff3970685cb704500909c94189bde9" +uuid = "4c63d2b9-4356-54db-8cca-17b64c39e42c" +version = "0.9.3" + +[[SuiteSparse]] +deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"] +uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9" [[Test]] deps = ["Distributed", "InteractiveUtils", "Logging", "Random"] @@ -539,27 +770,22 @@ uuid = "06e1c1a7-607b-532d-9fad-de7d9aa2abac" version = "0.2.3" [[TimerOutputs]] -deps = ["Crayons", "Printf", "Test", "Unicode"] -git-tree-sha1 = "b80671c06f8f8bae08c55d67b5ce292c5ae2660c" +deps = ["Printf"] +git-tree-sha1 = "311765af81bbb48d7bad01fb016d9c328c6ede03" uuid = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f" -version = "0.5.0" - -[[Tokenize]] -git-tree-sha1 = "0de343efc07da00cd449d5b04e959ebaeeb3305d" -uuid = "0796e94c-ce3b-5d07-9a54-7f471281c624" -version = "0.5.4" +version = "0.5.3" [[Tracker]] deps = ["Adapt", "DiffRules", "ForwardDiff", "LinearAlgebra", "MacroTools", "NNlib", "NaNMath", "Printf", "Random", "Requires", "SpecialFunctions", "Statistics", "Test"] -git-tree-sha1 = "327342fec6e09f68ced0c2dc5731ed475e4b696b" +git-tree-sha1 = "86929a5811dca5ce76c65a1d3fecda92d90c2e49" uuid = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" -version = "0.2.2" +version = "0.2.6" [[TranscodingStreams]] deps = ["Random", "Test"] -git-tree-sha1 = "a25d8e5a28c3b1b06d3859f30757d43106791919" +git-tree-sha1 = "7c53c35547de1c5b9d46a4797cf6d8253807108c" uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" -version = "0.9.4" +version = "0.9.5" [[URIParser]] deps = ["Test", "Unicode"] @@ -574,32 +800,314 @@ uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [[Unicode]] uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" -[[VersionParsing]] -deps = ["Compat"] -git-tree-sha1 = "c9d5aa108588b978bd859554660c8a5c4f2f7669" -uuid = "81def892-9a0e-5fdd-b105-ffc91e053289" -version = "1.1.3" - -[[WinRPM]] -deps = ["BinDeps", "Compat", "HTTPClient", "LibExpat", "Libdl", "Libz", "URIParser"] -git-tree-sha1 = "2a889d320f3b77d17c037f295859fe570133cfbf" -uuid = "c17dfb99-b4f7-5aad-8812-456da1ad7187" -version = "0.4.2" +[[Wayland_jll]] +deps = ["Expat_jll", "Libdl", "Libffi_jll", "Pkg", "XML2_jll"] +git-tree-sha1 = "0e7ae7fab73d43299b6b0332716dc2d0149f6b03" +uuid = "a2964d1f-97da-50d4-b82a-358c7fce9d89" +version = "1.17.0+0" + +[[Wayland_protocols_jll]] +deps = ["Libdl", "Pkg", "Wayland_jll"] +git-tree-sha1 = "d53533e9cbc71f1274b20f4af76c4f13e6e4a8a0" +uuid = "2381bf8a-dfd0-557d-9999-79630e7b1b91" +version = "1.18.0+1" [[WoodburyMatrices]] -deps = ["LinearAlgebra", "Random", "SparseArrays", "Test"] -git-tree-sha1 = "21772c33b447757ec7d3e61fcdfb9ea5c47eedcf" +deps = ["LinearAlgebra", "SparseArrays"] +git-tree-sha1 = "bbb9f7fd6fbdd9582e77c0b698312c543de5eb71" uuid = "efce3f68-66dc-5838-9240-27a6d6f5f9b6" -version = "0.4.1" +version = "0.5.0" + +[[X11_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "dbded9081d84be23bd31108e60ecac84f4e82e33" +uuid = "546b0b6d-9ca3-5ba2-8705-1bc1841d8479" +version = "1.6.8+4" + +[[XML2_jll]] +deps = ["Libdl", "Libiconv_jll", "Pkg", "Zlib_jll"] +git-tree-sha1 = "ed5603a695aefe3e9e404fc7b052e02cc72cfab6" +uuid = "02c8fc9c-b97f-50b9-bbe4-9be30ff0a78a" +version = "2.9.9+1" + +[[XSLT_jll]] +deps = ["Libdl", "Libgcrypt_jll", "Pkg", "XML2_jll"] +git-tree-sha1 = "77d0086a876532cb4fae005d4c9ecc8236dcdceb" +uuid = "aed1982a-8fda-507f-9586-7b0439959a61" +version = "1.1.33+0" + +[[Xorg_compositeproto_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "264e520f4c754e87e2609519cb994fe4c0999acd" +uuid = "0af4abc2-9bda-511f-85a5-daebf69421ba" +version = "0.4.0+0" + +[[Xorg_damageproto_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "eadb7238c6bf4abf8ba6644a132bcfbe9d051e2a" +uuid = "17eb5352-d50b-5fdc-b767-c749cd790e65" +version = "1.2.1+0" + +[[Xorg_fixesproto_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "101dcd548d6021b8d3cdaeb66c65c0d85faf1bcb" +uuid = "cf2f014d-5496-555f-b295-889ac9dfddaa" +version = "5.0.0+0" + +[[Xorg_glproto_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "16cb7bf73d86119bffe90d3ae7dd17b12281c887" +uuid = "41595b7c-9afa-5287-863f-896d3380052a" +version = "1.4.17+0" + +[[Xorg_inputproto_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "a941739b553f589cd264dc16fdeaf5300af30f51" +uuid = "84d6cd60-beca-5f49-93c5-789031781a2d" +version = "2.3.2+0" + +[[Xorg_kbproto_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "d0991fb63a9c663108bc5adc2ebdf358be6a289f" +uuid = "060dd47b-79ec-5ba1-a7b2-f4f2f7dcdd0f" +version = "1.0.7+0" + +[[Xorg_libX11_jll]] +deps = ["Libdl", "Pkg", "Xorg_inputproto_jll", "Xorg_kbproto_jll", "Xorg_libxcb_jll", "Xorg_util_macros_jll", "Xorg_xextproto_jll", "Xorg_xproto_jll", "Xorg_xtrans_jll"] +git-tree-sha1 = "a48b5faf2a95933d1249c889a2893f7701935bcb" +uuid = "4f6342f7-b3d2-589e-9d20-edeb45f2b2bc" +version = "1.6.8+0" + +[[Xorg_libXau_jll]] +deps = ["Libdl", "Pkg", "Xorg_xproto_jll"] +git-tree-sha1 = "8cc829c6223f11f06b56108c45e6b1aad13b1fdc" +uuid = "0c0b7dd1-d40b-584c-a123-a41640f87eec" +version = "1.0.9+0" + +[[Xorg_libXcomposite_jll]] +deps = ["Libdl", "Pkg", "Xorg_compositeproto_jll", "Xorg_libXfixes_jll", "Xorg_util_macros_jll"] +git-tree-sha1 = "58989e19a0dd9c9bc5e8ec00400d03441787cad5" +uuid = "3c9796d7-64a0-5134-86ad-79f8eb684845" +version = "0.4.5+0" + +[[Xorg_libXcursor_jll]] +deps = ["Libdl", "Pkg", "Xorg_libXfixes_jll", "Xorg_libXrender_jll", "Xorg_util_macros_jll"] +git-tree-sha1 = "0632e71a9d45f56a84b2cb517773918f3d54ddf3" +uuid = "935fb764-8cf2-53bf-bb30-45bb1f8bf724" +version = "1.2.0+0" + +[[Xorg_libXdamage_jll]] +deps = ["Libdl", "Pkg", "Xorg_damageproto_jll", "Xorg_libXfixes_jll"] +git-tree-sha1 = "484c2d81b1e47ddbdf92beed693951b719c057ff" +uuid = "0aeada51-83db-5f97-b67e-184615cfc6f6" +version = "1.1.5+0" + +[[Xorg_libXdmcp_jll]] +deps = ["Libdl", "Pkg", "Xorg_util_macros_jll", "Xorg_xproto_jll"] +git-tree-sha1 = "1cb581254dd38106e9cfb2c6aa1ed675547fd2e4" +uuid = "a3789734-cfe1-5b06-b2d0-1dd0d9d62d05" +version = "1.1.3+0" + +[[Xorg_libXext_jll]] +deps = ["Libdl", "Pkg", "Xorg_libX11_jll", "Xorg_util_macros_jll", "Xorg_xextproto_jll"] +git-tree-sha1 = "08b8a06261eec5fbe45d4077a993a224af2880e3" +uuid = "1082639a-0dae-5f34-9b06-72781eeb8cb3" +version = "1.3.4+0" + +[[Xorg_libXfixes_jll]] +deps = ["Libdl", "Pkg", "Xorg_fixesproto_jll", "Xorg_libX11_jll", "Xorg_util_macros_jll"] +git-tree-sha1 = "d4194996acacdd25f4f86a40ea6b9e497d4c1604" +uuid = "d091e8ba-531a-589c-9de9-94069b037ed8" +version = "5.0.3+0" + +[[Xorg_libXi_jll]] +deps = ["Libdl", "Pkg", "Xorg_inputproto_jll", "Xorg_libXext_jll", "Xorg_libXfixes_jll", "Xorg_util_macros_jll"] +git-tree-sha1 = "411b9503eb1dd7af3605e51a4cf7e4850a75f20e" +uuid = "a51aa0fd-4e3c-5386-b890-e753decda492" +version = "1.7.10+0" + +[[Xorg_libXinerama_jll]] +deps = ["Libdl", "Pkg", "Xorg_libXext_jll", "Xorg_util_macros_jll", "Xorg_xineramaproto_jll"] +git-tree-sha1 = "123ffa8c714ee630dde2256ad91d02322e079074" +uuid = "d1454406-59df-5ea1-beac-c340f2130bc3" +version = "1.1.4+0" + +[[Xorg_libXrandr_jll]] +deps = ["Libdl", "Pkg", "Xorg_libXext_jll", "Xorg_libXrender_jll", "Xorg_randrproto_jll", "Xorg_util_macros_jll"] +git-tree-sha1 = "122d70832769124a15f2a1623307f4637179b1b3" +uuid = "ec84b674-ba8e-5d96-8ba1-2a689ba10484" +version = "1.5.2+0" + +[[Xorg_libXrender_jll]] +deps = ["Libdl", "Pkg", "Xorg_libX11_jll", "Xorg_renderproto_jll"] +git-tree-sha1 = "07abd690ddd09818237a0a21359fd6ef449e820d" +uuid = "ea2f1a96-1ddc-540d-b46f-429655e07cfa" +version = "0.9.10+0" + +[[Xorg_libXtst_jll]] +deps = ["Libdl", "Pkg", "Xorg_inputproto_jll", "Xorg_libXext_jll", "Xorg_libXfixes_jll", "Xorg_libXi_jll", "Xorg_recordproto_jll", "Xorg_util_macros_jll"] +git-tree-sha1 = "2ca123092ee2b2d79c9316e4ecf0551209f3660e" +uuid = "b6f176f1-7aea-5357-ad67-1d3e565ea1c6" +version = "1.2.3+0" + +[[Xorg_libpthread_stubs_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "5c2806e6e08b093ca2071e53602651f4a5e079da" +uuid = "14d82f49-176c-5ed1-bb49-ad3f5cbd8c74" +version = "0.1.0+0" + +[[Xorg_libxcb_jll]] +deps = ["Libdl", "Pkg", "XSLT_jll", "Xorg_libXau_jll", "Xorg_libXdmcp_jll", "Xorg_libpthread_stubs_jll", "Xorg_util_macros_jll", "Xorg_xcb_proto_jll"] +git-tree-sha1 = "b3d001a3df5ec00fd677f0ae674127983218ac72" +uuid = "c7cfdc94-dc32-55de-ac96-5a1b8d977c5b" +version = "1.13.0+0" + +[[Xorg_libxkbfile_jll]] +deps = ["Libdl", "Pkg", "Xorg_libX11_jll", "Xorg_util_macros_jll"] +git-tree-sha1 = "272e91c38e82596f90cfd0f4ea24ff397bbef260" +uuid = "cc61e674-0454-545c-8b26-ed2c68acab7a" +version = "1.1.0+0" + +[[Xorg_randrproto_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "5c2f995614980bedaed7de97995f9a5e0f857bef" +uuid = "0e394dc1-71ae-5c65-abe5-8749687e42d3" +version = "1.5.0+0" + +[[Xorg_recordproto_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "0192d9bc3f2bfdd90191e055c412c3fd3faf4b1d" +uuid = "a7b44dc7-5846-51fe-a1cd-6b242f0ec0a2" +version = "1.14.2+0" + +[[Xorg_renderproto_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "36dbfa838fc4faf019a6957f2eb8c1b5f31bb500" +uuid = "21e99dc2-7dba-5609-a726-b181bd3bbb6c" +version = "0.11.1+0" + +[[Xorg_util_macros_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "a99f6275c9b2a9862d2cc4e714aad806c2daecc1" +uuid = "7c09cfe3-afe2-5798-bcc9-d6b7fecfdca5" +version = "1.19.2+0" + +[[Xorg_xcb_proto_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "1506cde098df4b6dedcc082bf9386a5808ee6dea" +uuid = "c2e9c405-c068-5e7b-9b35-084fd074cae4" +version = "1.13.0+0" + +[[Xorg_xextproto_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "0a1da65671cfc3b20926081b668c7dd098dd9c76" +uuid = "d13bc2ba-d276-5c6f-8a1c-29ed04aab5d0" +version = "7.3.0+0" + +[[Xorg_xineramaproto_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "9f331b569a18a4fbf66ee8f60cb4018c73faabb4" +uuid = "6a3da44c-33b1-5374-838f-bf0fbf92c29b" +version = "1.2.1+0" + +[[Xorg_xkbcomp_jll]] +deps = ["Libdl", "Pkg", "Xorg_libxkbfile_jll", "Xorg_util_macros_jll"] +git-tree-sha1 = "55944512cd801174f94dcd19479c435acd6785e4" +uuid = "35661453-b289-5fab-8a00-3d9160c6a3a4" +version = "1.4.2+0" + +[[Xorg_xkeyboard_config_jll]] +deps = ["Libdl", "Pkg", "Xorg_xkbcomp_jll"] +git-tree-sha1 = "416f9257ada9365f53766394af77134bf594e1a2" +uuid = "33bec58e-1273-512f-9401-5d533626f822" +version = "2.27.0+1" + +[[Xorg_xproto_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "fdb0d8bae06762e24dade427e7914d199cdd721e" +uuid = "46797783-dccc-5433-be59-056c4bde8513" +version = "7.0.31+0" + +[[Xorg_xtrans_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "b18f01c3b50be7bde08be131934112db3137f28b" +uuid = "c5fb5394-a638-5e4d-96e5-b29de1b5cf10" +version = "1.4.0+0" [[ZipFile]] -deps = ["BinaryProvider", "Libdl", "Printf"] -git-tree-sha1 = "580ce62b6c14244916cc28ad54f8a2e2886f843d" +deps = ["Libdl", "Printf", "Zlib_jll"] +git-tree-sha1 = "5de8320a46812da1a8ca98b16a8a4546d44efa62" uuid = "a5390f91-8eb1-5f08-bee0-b1d1ffed6cea" -version = "0.8.3" +version = "0.9.0" + +[[Zlib_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "5618a43055eb09377edca21d19d0e99bce24a9c3" +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.2.11+7" + +[[Zstd_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "aa97e3e240e86010da61254b0045b7175ead0659" +uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" +version = "1.4.4+0" [[Zygote]] -deps = ["DiffRules", "FillArrays", "ForwardDiff", "IRTools", "InteractiveUtils", "LinearAlgebra", "MacroTools", "NNlib", "NaNMath", "Random", "Requires", "SpecialFunctions", "Statistics"] -git-tree-sha1 = "520d65c5e5554473863d738bde053f5f6769d3be" +deps = ["DiffRules", "FFTW", "FillArrays", "ForwardDiff", "IRTools", "InteractiveUtils", "LinearAlgebra", "MacroTools", "NNlib", "NaNMath", "Random", "Requires", "SpecialFunctions", "Statistics", "ZygoteRules"] +git-tree-sha1 = "ca4dfa4de0a0e2c1da6c8c67d3b9af99645b57fc" uuid = "e88e6eb3-aa80-5325-afca-941959d7151f" -version = "0.3.2" +version = "0.4.6" + +[[ZygoteRules]] +deps = ["MacroTools"] +git-tree-sha1 = "b3b4882cc9accf6731a08cc39543fbc6b669dca8" +uuid = "700de1a5-db45-46bc-99cf-38207098b444" +version = "0.2.0" + +[[adwaita_icon_theme_jll]] +deps = ["Libdl", "Pkg", "hicolor_icon_theme_jll"] +git-tree-sha1 = "245d4d5884e54d171c9683e1d30213358ddeb648" +uuid = "b437f822-2cd6-5e08-a15c-8bac984d38ee" +version = "3.33.92+3" + +[[at_spi2_atk_jll]] +deps = ["ATK_jll", "Libdl", "Pkg", "XML2_jll", "Xorg_libX11_jll", "at_spi2_core_jll"] +git-tree-sha1 = "c896019bced4ebbf7e507e66fa403ec4409a1937" +uuid = "de012916-1e3f-58c2-8f29-df3ef51d412d" +version = "2.34.1+0" + +[[at_spi2_core_jll]] +deps = ["Dbus_jll", "Glib_jll", "Libdl", "Pkg", "Xorg_libXtst_jll"] +git-tree-sha1 = "db3118da77a00c375c04ff3df08daa4045324b44" +uuid = "0fc3237b-ac94-5853-b45c-d43d59a06200" +version = "2.34.0+0" + +[[gdk_pixbuf_jll]] +deps = ["Glib_jll", "JpegTurbo_jll", "Libdl", "Libtiff_jll", "Pkg", "X11_jll", "Xorg_libX11_jll", "libpng_jll"] +git-tree-sha1 = "566ebf4b2c9efd81b35eb914c325026831077f39" +uuid = "da03df04-f53b-5353-a52f-6a8b0620ced0" +version = "2.38.2+6" + +[[hicolor_icon_theme_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "4f8addaf9ca8492b0ba3f289587c983ba395dd60" +uuid = "059c91fe-1bad-52ad-bddd-f7b78713c282" +version = "0.17.0+1" + +[[iso_codes_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "46515ba7486a978a3f6418ce060a3407ed837e1e" +uuid = "bf975903-5238-5d20-8243-bc370bc1e7e5" +version = "4.3.0+1" + +[[libpng_jll]] +deps = ["Libdl", "Pkg", "Zlib_jll"] +git-tree-sha1 = "30c6baf453f2603faedb846289f955d307ddf3b2" +uuid = "b53b4c65-9356-5827-b1ea-8c7a1a84506f" +version = "1.6.37+2" + +[[xkbcommon_jll]] +deps = ["Libdl", "Pkg", "Wayland_jll", "Wayland_protocols_jll", "Xorg_libxcb_jll", "Xorg_xkeyboard_config_jll"] +git-tree-sha1 = "cbed604f26643b998f29b74b2a2f73298cb07606" +uuid = "d8fb68d0-12a3-5cfd-a85a-d49703b185fd" +version = "0.9.1+1" diff --git a/Project.toml b/Project.toml index 924b3df..35b3612 100644 --- a/Project.toml +++ b/Project.toml @@ -4,10 +4,15 @@ authors = ["Tejan Karmali ", "Neethu Mariya Joy <>"] version = "0.1.0" [deps] +ArcadeLearningEnvironment = "b7f77d8d-088d-5e02-8ac0-89aab2acc977" Cairo = "159f3aea-2a34-519c-b102-8c37f9878175" Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c" +Gtk = "4c0ca9eb-093a-5379-98c5-f87ac0bbbf44" +GymSpaces = "4896a6fa-884d-11e9-35a0-9763e69c01ff" Images = "916415d5-f1e6-5110-898d-aaa5f9f070e0" -Requires = "ae029012-a4dd-5104-9daa-d747884805df" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" diff --git a/README.md b/README.md index 5447a7b..71db308 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,14 @@ Gym environments in Julia **`Gym.jl` requires Julia v1.1** # Installation +First, we need to install a dependency `GymSpaces.jl`, which is currently unregistered (hopefully not for long). + +```julia +julia> ] add https://github.com/kraftpunk97/GymSpaces.jl +``` +Next we proceed to install `Gym.jl` ```julia -julia> ] add https://github.com/FluxML/Gym.jl +julia> ] add https://github.com/kraftpunk97/Gym.jl#toytext ``` ## Usage @@ -27,6 +33,7 @@ while i <= length(actions) && !done end ``` ## Currently available environments -* CartPole -* Pendulum -* Continuous_MountainCar +* Continous Control Problems +* Algorithmic Environments +* Atari Environments +* Toy Text Problems diff --git a/examples/Algorithmic/Q-learning.jl b/examples/Algorithmic/Q-learning.jl new file mode 100644 index 0000000..4b9f52f --- /dev/null +++ b/examples/Algorithmic/Q-learning.jl @@ -0,0 +1,64 @@ +using Gym +using StatsBase: sample, Weights +using DataStructures: CircularBuffer +using Statistics: mean + +env = make("Copy-v0") +reset!(env) + +max_episodes = 75000 +α = 3f-1 +γ = 9f-1 +ϵ = 1f-1 +ϵ_schedule = 10000 +# Copy is considered solved when the average reward for 100 continuous episodes >=25 +goal = 25f0 + +num_dims = [length(space) for space in env.action_space.spaces] +num_actions = prod(num_dims) +num_states = length(env.observation_space) + +Q = zeros(Float32, num_states, num_actions) +P = zeros(Float32, num_actions) + +function policy(state) + fill!(P, ϵ/num_actions) + P[argmax(Q[state[1], :])] += 1 - ϵ + return sample(1:num_actions, Weights(P)) +end + +function actioninttotup(int) + @assert int <= prod(num_dims) + res = [] + int -= 1 + for dim in reverse(num_dims) + push!(res, (int%dim)+1) + int = div(int, dim) + end + return reverse(res) +end + +total_reward = CircularBuffer{Float32}(100) +for e in 1:max_episodes + global α, γ, ϵ_schedule, ϵ, goal + if e % 50 == 0 + println("Episode: $e | Average reward of the last 100 episodes: $(mean(total_reward))") + if mean(total_reward) > goal + println("Copy Solved at episode = $e") + break + end + end + e % ϵ_schedule == 0 && (ϵ /= 2) + current_state = reset!(env) + done = false + push!(total_reward, 0f0) + while !done + a = policy(current_state) + decoded_a = actioninttotup(a) + next_state, reward, done, _ = step!(env, decoded_a) + #println("a = $a\tcurrent_state[1] = $(current_state[1])\tnext_state[1] = $(next_state[1])") + Q[current_state[1], a] += α * ((reward + γ * maximum(Q[next_state[1], :])) - Q[current_state[1], a]) + current_state = next_state + total_reward[end] += reward + end +end diff --git a/examples/CartPole/DQN.jl b/examples/CartPole/DQN.jl index a3f1203..8c76e48 100644 --- a/examples/CartPole/DQN.jl +++ b/examples/CartPole/DQN.jl @@ -1,14 +1,13 @@ -using Flux, Gym +using Flux, Gym, Printf using Flux.Optimise: Optimiser using Flux.Tracker: data using Statistics: mean using DataStructures: CircularBuffer using Distributions: sample -using Printf #using CuArrays # Load game environment -env = make("CartPole-v0", :human_pane) +env = make("CartPole-v0") reset!(env) # ----------------------------- Parameters ------------------------------------- @@ -40,7 +39,7 @@ loss(x, y) = Flux.mse(model(x |> gpu), y) opt = Optimiser(ADAM(η), InvDecay(η_decay)) -# ----------------------------- Helper Functions ------------------------------- +# ----------------------------- HelperFunctions ------------------------------- get_ϵ(e) = max(ϵ_MIN, min(ϵ, 1f0 - log10(e * ϵ_DECAY))) @@ -80,19 +79,15 @@ function replay() end function episode!(env, train=true, draw=false) - done = false - total_reward = 0f0 - while !done + reset!(env) + while !game_over(env) draw && render!(env) s = env._env.state - a = action(s, train) + a = action(s, trainable(env)) s′, r, done, _ = step!(env, a) - total_reward += r - train && remember(s, a, r, s′, done) + trainable(env) && remember(s, a, r, s′, done) draw && sleep(0.01) end - - total_reward end # ------------------------------ Training -------------------------------------- @@ -102,10 +97,9 @@ scores = CircularBuffer{Float32}(100) while true global e - reset!(env) - total_reward = episode!(env) - push!(scores, total_reward) - print("Episode: $e | Score: $total_reward ") + episode!(env) + push!(scores, env.total_reward) + print("Episode: $e | Score: $(env.total_reward) ") last_100_mean = mean(scores) print("Last 100 episodes mean score: $(@sprintf "%6.2f" last_100_mean)") if last_100_mean > 195 @@ -118,13 +112,14 @@ while true end # -------------------------------- Testing ------------------------------------- - +#= ee = 1 while true global ee reset!(env) - total_reward = episode!(env, false, true) + total_reward = episode!(env, false, false) println("Episode: $ee | Score: $total_reward") ee += 1 end +=# diff --git a/src/Envs/algorithmic/AlgorithmicEnv.jl b/src/Envs/algorithmic/AlgorithmicEnv.jl new file mode 100644 index 0000000..a6ddd1d --- /dev/null +++ b/src/Envs/algorithmic/AlgorithmicEnv.jl @@ -0,0 +1,165 @@ +using GymSpaces: TupleSpace, Discrete + +#= +Algorithmic environments have the following traits in common: + +- A 1-d "input tape" or 2-d "input grid" of characters +- A target string which is a deterministic function of the input characters + +Agents control a read head that moves over the input tape. Observations consist +of the single character currently under the read head. The read head may fall +off the end of the tape in any direction. When this happens, agents will observe +a special blank character (with index=0) until they get back in bounds. + +Actions consist of 3 sub-actions: + - Direction to move the read head (left or right, plus up and down for 2-d envs) + - Whether to write to the output tape + - Which character to write (ignored if the above sub-action is 0) + +An episode ends when: + - The agent writes the full target string to the output tape. + - The agent writes an incorrect character. + - The agent runs out the time limit. (Which is fairly conservative.) + +Reward schedule: + write a correct character: +1 + write a wrong character: -.5 + run out the clock: -1 + otherwise: 0 + +In the beginning, input strings will be fairly short. After an environment has +been consistently solved over some window of episodes, the environment will +increase the average length of generated strings. Typical env specs require +leveling up many times to reach their reward threshold. +=# + +abstract type AlgorithmicEnv <: AbstractEnv end +abstract type TapeAlgorithmicEnv <: AlgorithmicEnv end +abstract type GridAlgorithmicEnv <: AlgorithmicEnv end + +include("vis/algorithmicenv.jl") + +#= +Called between episodes. Update our running record of episode rewards and, +if appropriate, 'level up' minimum input length. +=# +function check_level_up!(env::AlgorithmicEnv) + push!(env.reward_shortfalls, env.episode_total_reward - length(env.target)) + if length(env.reward_shortfalls) == env.reward_shortfalls.capacity && + minimum(env.reward_shortfalls) >= env.MIN_REWARD_SHORTFALL_FOR_PROMOTION && + env.min_length < 30 + env.min_length += 1 + env.reward_shortfalls = CircularBuffer{Float32}(10) + end +end + +function seed!(env::AlgorithmicEnv, seed::Unsigned) + env.seed = MersenneTwister(seed) + return nothing +end + +function reset!(env::AlgorithmicEnv) + check_level_up!(env) + env.last_action = nothing + env.last_reward = 0 + env.read_head_position = env.READ_HEAD_START + env.write_head_position = 1 + env.episode_total_reward = 0f0 + env.time = 0 + len = rand(env.seed, 0:2) + env.min_length + generate_input_data!(env, len) + target_from_input_data!(env) + _get_obs(env) +end + + +function step!(env::AlgorithmicEnv, action) + @assert action ∈ env.action_space "$(action) is not a valid action for this environment." + env.last_action = action + inp_act = action[1:1] + out_act = action[2:2] + pred = action[3:3] + done = false + reward = 0f0 + env.time += 1 # Clock ticks one time + @assert 1 <= env.write_head_position # Ensure that we have someplace to write to + if out_act[1] == 2 + correct = false # initialising 'correct' + try + correct = pred[1] == env.target[env.write_head_position] + catch BoundsError + @warn "It looks like you're calling step() even though this " * + "environement has already returned done=true. You should " * + "always call reset!() once you recieve done=true. Any " * + "further steps are undefined behaviour." + correct = false + end + reward += correct ? 1f0 : -5f-1 # Calculate reward. + !correct && (done = true) # Bail immediately if wrong character is written. + + env.write_head_position += 1 + env.write_head_position > length(env.target) && (done = true) + end + move!(env, inp_act) + # If an agent takes more than this many timesteps, end the episode + # immediately and return negative reward + timelimit = time_limit(env) + if env.time > timelimit + reward = 0reward - 1f0 + done = true + end + obs = _get_obs(env) + env.last_reward = reward + env.episode_total_reward += reward + return (obs, reward, done, Dict()) +end + +generate_input_data!(env::TapeAlgorithmicEnv, size_) = + env.input_data = [rand(env.seed, 1:env.base) for _ in 1:size_] +generate_input_data!(env::GridAlgorithmicEnv, size_) = + env.input_data = [[rand(env.seed, 1:env.base) for _ in 1:env.rows] for __ in 1:size_] + + +function move!(env::TapeAlgorithmicEnv, movement) + named = env.MOVEMENTS[movement[1]] + env.read_head_position += named == :right ? 1 : -1 +end + +function move!(env::GridAlgorithmicEnv, movement) + named = env.MOVEMENTS[movement[1]] + x, y = env.read_head_posiion + if named == :left + x -= 1 + elseif named == :right + x += 1 + elseif named == :up + y -= 1 + elseif named == :down + y += 1 + else + throw(ArgumentError("Unrecognized direction: $(named)")) + end + env.read_head_posiion = [x, y] +end + + +# return character under read_head/pos. If the read_head/pos is out-of-bounds, then return null(0). +_get_obs(env::AlgorithmicEnv) = _get_obs(env, env.read_head_position) +function _get_obs(env::TapeAlgorithmicEnv, pos::Integer) + try + return [env.input_data[pos]] + catch BoundsError + return [env.base + 1] + end +end + +function _get_obs(env::GridAlgorithmicEnv, pos::Array) + x, y = pos + try + return [env.input_data[x][y]] + catch BoundsError + return [env.base + 1] + end +end + +time_limit(env::TapeAlgorithmicEnv) = length(env.input_data) + length(env.target) + 4 diff --git a/src/Envs/algorithmic/Copy.jl b/src/Envs/algorithmic/Copy.jl new file mode 100644 index 0000000..570ca05 --- /dev/null +++ b/src/Envs/algorithmic/Copy.jl @@ -0,0 +1,60 @@ +""" +Task is to copy content from the input tape to +the output tape. http://arxiv.org/abs/1511.07275 +""" + +using DataStructures: CircularBuffer +using Random +include("AlgorithmicEnv.jl") + +RealOrNothing = Union{Float32, Nothing} + +mutable struct CopyEnv <: TapeAlgorithmicEnv + MIN_REWARD_SHORTFALL_FOR_PROMOTION::Float32 + base::Int8 # Number of distinct characters + episode_total_reward::RealOrNothing # Cumulative reward earned this episode + + # Running tally of reward shortfalls. eg If there were 10 points to earn + # and we got 8, we'd append -2 + reward_shortfalls::CircularBuffer{Float32} + + charmap::Array{Char, 1} # Array of all the characters used while rendering + min_length::Int8 # Minimum length of the input tape + action_space::TupleSpace + observation_space::Discrete + + MOVEMENTS::NTuple # Possible directions to move in. + READ_HEAD_START::Int8 # Starting position of the read head + + target::Array{N, 1} where N <: Union{Char, Integer} # target tape + input_data::Array{N, 1} where N <: Union{Char, Integer} + time::Int8 + read_head_position::Int8 # current position of the read head. + write_head_position::Int8 + last_action + last_reward::Float32 + seed::MersenneTwister +end + +function CopyEnv(base::Int=5, chars::Bool=true) + starting_char = chars ? 'A' : '0' + CopyEnv(-1f0, # MIN_REWARD_SHORTFALL_FOR_PROMOTION + base, + 0f0, # episode_total_reward + CircularBuffer{Float32}(10), # reward_shortfalls + push!([starting_char+i for i=0:base-1], ' '), # charmap + 2, # starting min_length + TupleSpace([Discrete(2), Discrete(2), Discrete(base)]), # action_space + Discrete(base+1), # observation_space + (:left, :right), 1, # MOVEMENTS and READ_HEAD_START + Array{(chars ? Char : Int8), 1}(), # target + Array{(chars ? Char : Int8), 1}(), # input_data + 0, # time + 1, # read_head_posiion + 1, # write_head_position + nothing, # last_action + 0.0, # last_reward + MersenneTwister()) # seed +end + +target_from_input_data!(env::CopyEnv) = env.target = env.input_data diff --git a/src/Envs/algorithmic/DuplicatedInput.jl b/src/Envs/algorithmic/DuplicatedInput.jl new file mode 100644 index 0000000..0980067 --- /dev/null +++ b/src/Envs/algorithmic/DuplicatedInput.jl @@ -0,0 +1,67 @@ +using Random +using DataStructures: CircularBuffer +include("AlgorithmicEnv.jl") + +""" +Task is to return every nth character from the input tape. +http://arxiv.org/abs/1511.07275 +""" +mutable struct DuplicatedInputEnv <: TapeAlgorithmicEnv + MIN_REWARD_SHORTFALL_FOR_PROMOTION::Float32 + base::Int + episode_total_reward::Float32 + reward_shortfalls::CircularBuffer{Float32} + charmap::Array{Char, 1} + min_length::Int + action_space::TupleSpace + observation_space::Discrete + + MOVEMENTS::NTuple + READ_HEAD_START::Int + + duplication::Int + + target::Array{Int8, 1} # target tape + input_data::Array{Int8, 1} + time::Int8 + read_head_position::Int8 # current position of the read head. + write_head_position::Int8 + last_action + last_reward::Float32 + seed::MersenneTwister +end + + +DuplicatedInputEnv(base::Int=5, duplication::Int=2) = + DuplicatedInputEnv(-1f0, # MIN_REWARD_SHORTFALL_FOR_PROMOTION + base, + 0f0, # episode_total_reward + CircularBuffer{Float32}(10), # reward_shortfalls + push!(['A'+i for i=0:base-1], ' '), # charmap + 2, # starting min_length + TupleSpace([Discrete(2), Discrete(2), Discrete(base)]), # action_space + Discrete(base+1), # observation_space + (:left, :right), 1, # MOVEMENTS and READ_HEAD_START + duplication, + Int8[], + Int8[], + 0, 1, 1, nothing, 0.0, + MersenneTwister()) + + +function generate_input_data!(env::DuplicatedInputEnv, size_) + res = [] + size_ < env.duplication && + (size_ = env.duplication) + for i=1:div(size_, env.duplication) + char = rand(env.seed, 1:env.base) + for _ in 1:env.duplication + push!(res, char) + end + end + env.input_data = res +end + + +target_from_input_data!(env::DuplicatedInputEnv) = + env.target = [env.input_data[i] for i=1:env.duplication:length(env.input_data)] diff --git a/src/Envs/algorithmic/RepeatCopy.jl b/src/Envs/algorithmic/RepeatCopy.jl new file mode 100644 index 0000000..469b0ad --- /dev/null +++ b/src/Envs/algorithmic/RepeatCopy.jl @@ -0,0 +1,49 @@ +using Random +using DataStructures: CircularBuffer +include("AlgorithmicEnv.jl") + +""" +Task is to copy content multiple times from the input tape to +the output tape. http://arxiv.org/abs/1511.07275 +""" +mutable struct RepeatCopyEnv <: TapeAlgorithmicEnv + MIN_REWARD_SHORTFALL_FOR_PROMOTION::Float32 + base::Int + episode_total_reward::Float32 + reward_shortfalls::CircularBuffer{Float32} + charmap::Array{Char, 1} + min_length::Int + action_space::TupleSpace + observation_space::Discrete + + MOVEMENTS::NTuple + READ_HEAD_START::Int + + target::Array{Int8, 1} # target tape + input_data::Array{Int8, 1} + time::Int8 + read_head_position::Int8 # current position of the read head. + write_head_position::Int8 + last_action + last_reward::Float32 + seed::MersenneTwister +end + +RepeatCopyEnv(base::Int=5) = + RepeatCopyEnv(-1f-1, # MIN_REWARD_SHORTFALL_FOR_PROMOTION + base, + 0f0, # episode_total_reward + CircularBuffer{Float32}(50), # reward_shortfalls + push!(['A'+i for i=0:base-1], ' '), # charmap + 2, # starting min_length + TupleSpace([Discrete(2), Discrete(2), Discrete(base)]), # action_space + Discrete(base+1), # observation_space + (:left, :right), 1, # MOVEMENTS and READ_HEAD_START + Int8[], + Int8[], 0, 1, 1, nothing, 0.0, + MersenneTwister()) + +target_from_input_data!(env::RepeatCopyEnv) = + env.target = vcat(env.input_data, + [char for char in reverse(env.input_data)], + env.input_data) diff --git a/src/Envs/algorithmic/Reverse.jl b/src/Envs/algorithmic/Reverse.jl new file mode 100644 index 0000000..f0426b3 --- /dev/null +++ b/src/Envs/algorithmic/Reverse.jl @@ -0,0 +1,45 @@ +using Random +using DataStructures: CircularBuffer +include("AlgorithmicEnv.jl") + +""" +Task is to reverse content over the input tape. +http://arxiv.org/abs/1511.07275 +""" +mutable struct ReverseEnv <: TapeAlgorithmicEnv + MIN_REWARD_SHORTFALL_FOR_PROMOTION::Float32 + base::Int + episode_total_reward::Float32 + reward_shortfalls::CircularBuffer{Float32} + charmap::Array{Char, 1} + min_length::Int + action_space::TupleSpace + observation_space::Discrete + + MOVEMENTS::NTuple + READ_HEAD_START::Int + + target::Array{Int8, 1} # target tape + input_data::Array{Int8, 1} + time::Int8 + read_head_position::Int8 # current position of the read head. + write_head_position::Int8 + last_action + last_reward::Float32 + seed::MersenneTwister +end + +ReverseEnv(base::Int=2) = + ReverseEnv(-1f-1, # MIN_REWARD_SHORTFALL_FOR_PROMOTION + base, # last + 0f0, # episode_total_reward + CircularBuffer{Float32}(50), # reward_shortfalls + push!(['A'+i for i=0:base-1], ' '), # charmap + 1, # starting min_length + TupleSpace([Discrete(2), Discrete(2), Discrete(base)]), # action_space + Discrete(base+1), # observation_space + (:left, :right), 1, # MOVEMENTS and READ_HEAD_START + Int8[], Int8[], 0, 1, 1, nothing, 0.0, MersenneTwister()) + +target_from_input_data!(env::ReverseEnv) = + env.target = [char for char in reverse(env.input_data)] diff --git a/src/Envs/algorithmic/ReversedAddition.jl b/src/Envs/algorithmic/ReversedAddition.jl new file mode 100644 index 0000000..6e63fc0 --- /dev/null +++ b/src/Envs/algorithmic/ReversedAddition.jl @@ -0,0 +1,54 @@ +using Random +using DataStructures: CircularBuffer +include("AlgorithmicEnv.jl") + +mutable struct ReversedAdditionEnv <: GridAlgorithmicEnv + MIN_REWARD_SHORTFALL_FOR_PROMOTION::Float32 + base::Int + episode_total_reward::Float32 + reward_shortfalls::CircularBuffer{Float32} + charmap::Array{Char, 1} + min_length::Int + action_space::TupleSpace + observation_space::Discrete + + MOVEMENTS::NTuple + READ_HEAD_START::Array{Int8, 1} + + rows::Int + + target + input_data + time + read_head_position + write_head_position + last_action + last_reward + seed::MersenneTwister +end + +ReversedAdditionEnv(base::Int=3; rows::Int=2) = + ReversedAdditionEnv(-1f0, # MIN_REWARD_SHORTFALL_FOR_PROMOTION + base, # last + 0f0, # episode_total_reward + CircularBuffer{Float32}(10), # reward_shortfalls + push!(['0'+i for i=0:base-1], ' '), # charmap + 2, # starting min_length + TupleSpace([Discrete(2), Discrete(2), Discrete(base)]), # action_space + Discrete(base+1), # observation_space + (:left, :right, :up, :down), [1, 1], # MOVEMENTS and READ_HEAD_START + rows, + Int8[], Int8[], 0, [1, 1], [1, 1], nothing, 0.0, MersenneTwister()) + +function target_from_input_data!(env::ReversedAdditionEnv) + curry = 0 + env.target = [] + for digits in env.input_data + total = sum(digits) + curry + push!(env.target, total % env.base) + curry = div(total, env.base) + end + curry > 0 && (push!(env.target, curry)) +end + +time_limit(env::ReversedAdditionEnv) = length(env.input_data) * 4 diff --git a/src/Envs/algorithmic/vis/algorithmicenv.jl b/src/Envs/algorithmic/vis/algorithmicenv.jl new file mode 100644 index 0000000..818f8e7 --- /dev/null +++ b/src/Envs/algorithmic/vis/algorithmicenv.jl @@ -0,0 +1,109 @@ +include("utils.jl") + +function Ctx(env::AlgorithmicEnv, mode::Symbol=:human) + if mode == :human + HumanCtx() + elseif mode == :ansi + ANSICtx() + elseif mode == :no_render + NoCtx() + else + error("Unrecognized mode in Ctx(): $(mode)") + end +end + +function colorize_bold(string::Union{AbstractString, Char}, color::Symbol) + colormap = Dict( + :gray => 30, # highlights to black + :red => 31, + :green => 32, + :yellow => 34, + :magenta => 35, + :cyan => 36, + :white => 37) + + num = colormap[color] + num += 10 # For highlighting + return "\x1b[$num;1m" * string * "\x1b[0m" +end + +_get_str_target(env::AlgorithmicEnv, pos) = + pos < 1 || length(env.target) < pos ? " " : env.charmap[env.target[pos]] + +function render_observation(env::TapeAlgorithmicEnv) + x = env.read_head_position + x_str = "Observation Tape : " + for i=-1:length(env.input_data)+2 + x_str *= i == x ? colorize_bold(env.charmap[_get_obs(env, i)[1]], :green) : + env.charmap[_get_obs(env, i)[1]] + end + x_str *= "\n" + return x_str +end + +function render_observation(env::GridAlgorithmicEnv) + x = env.read_head_position + label = "Observation Grid : " + x_str = "" + for j=0:env.rows + if j != 0 + for _=1:length(label) + x_str *= " " + end + end + for i=0:length(env.input_data) + println("Okay for i=$i and j=$j, _get_obs=$(_get_obs(env, [i, j]))") + x_str *= i == x[1] && j == x[2] ? colorize_bold(env.charmap[_get_obs(env, [i, j])[1]], :green) : + env.charmap[_get_obs(env, [i, j])[1]] + end + x_str *= "\n" + end + x_str = label * x_str + return x_str +end + +function drawcanvas!(env::AlgorithmicEnv) + inp = "Total length of input instance: $(length(env.input_data)), step: $(env.time)\n" + output = "" + output *= inp + x = env.read_head_position + y = env.write_head_position + action = env.last_action + !isnothing(action) && + ((inp_act, out_act, pred) = action) + equal_sign = "" + for i=1:length(inp)+1 + equal_sign *= '=' + end + output *= equal_sign * '\n' + y_str = "Output Tape : " + target_str = "Targets : " + !isnothing(action) && + (pred_str = env.charmap[pred]) + x_str = render_observation(env) + for i=-1:length(env.target)+2 + target_str *= _get_str_target(env, i) + if i < y - 1 + y_str *= _get_str_target(env, i) + elseif i == (y - 1) + if !isnothing(action) && out_act == 2 + color = pred == env.target[i] ? :green : :red + y_str *= colorize_bold(pred_str, color) + else + y_str *= _get_str_target(env, i) + end + end + end + output *= x_str * y_str * '\n' * target_str * "\n\n" + + if !isnothing(action) + output *= "Current reward : $(env.last_reward)\n" + output *= "Cumulative reward : $(env.episode_total_reward)\n" + output *= "Action : Tuple(move over input: $(String(env.MOVEMENTS[inp_act])),\n" + output *= " write to the output tape: $(out_act==2)\n" + output *= " prediction: $pred_str)\n" + else + output *= "\n\n\n\n\n" + end + return output +end diff --git a/src/Envs/algorithmic/vis/utils.jl b/src/Envs/algorithmic/vis/utils.jl new file mode 100644 index 0000000..a6b65a8 --- /dev/null +++ b/src/Envs/algorithmic/vis/utils.jl @@ -0,0 +1,9 @@ +struct HumanCtx <: AbstractCtx end +struct ANSICtx <: AbstractCtx end +struct NoCtx <: AbstractCtx end + +render!(env::AbstractEnv, ctx::ANSICtx) = drawcanvas!(env) +render!(env::AbstractEnv, ctx::NoCtx) = nothing +function render!(env::AbstractEnv, ctx::HumanCtx) + print(drawcanvas!(env)) +end diff --git a/src/Envs/atari/Atari.jl b/src/Envs/atari/Atari.jl new file mode 100644 index 0000000..df516e8 --- /dev/null +++ b/src/Envs/atari/Atari.jl @@ -0,0 +1,154 @@ +using GymSpaces: Box, Discrete +using ArcadeLearningEnvironment + +setLoggerMode!(:error) # Only log errors + +struct AtariEnv <: AbstractEnv + game_path::String + ale::Ptr{Nothing} + action_set::Array{UInt, 1} + action_space::Discrete + observation_space::Box + frameskip::Union{Int, UnitRange{Int}} + obs_type::Symbol +end + +include("vis/utils.jl") + +function AtariEnv( ; game_path::String, obs_type::Symbol=:ram, frameskip::Union{Int, UnitRange{Int}}=2:5, + repeat_action_probability::AbstractFloat=0f0, full_action_space::Bool=false) + + @assert obs_type ∈ (:ram, :color, :grey) "Invalid observation type. Choose either :ram, :image or :grey" + @assert isa(repeat_action_probability, AbstractFloat) || isa(repeat_action_probability, Integer) "Invalid repeat_action_probability" + + ale = ALE_new() + loadROM(ale, game_path) # This is will also check if the loaded file is valid or not. + action_set = full_action_space ? getLegalActionSet(ale) : getMinimalActionSet(ale) + #action_set .+= 1 # Converting to one indexed array + action_space = full_action_space ? Discrete(getLegalActionSize(ale)) : Discrete(getMinimalActionSize(ale)) + + setFloat(ale, "repeat_action_probability", repeat_action_probability) + + screen_width, screen_height = getScreenWidth(ale), getScreenHeight(ale) + + observation_space = obs_type==:ram ? Box(0, 255, (128,), UInt8) : Box(0, 255, (screen_height, screen_width, 3), UInt8) + + AtariEnv(game_path, ale, action_set, action_space, observation_space, frameskip, obs_type) +end + +function seed!(env::AtariEnv, seed::Unsigned) + seed = seed % 2^31 + setInt(env.ale, "random_seed", seed) + loadROM(env.ale, env.game_path) + return nothing +end + + + +function step!(env::AtariEnv, action) + @assert action ∈ env.action_space "Action $action is invalid." + reward = 0f0 + + num_steps = isa(env.frameskip, UnitRange) ? rand(env.frameskip) : env.frameskip + + for _ in 1:num_steps + reward += act(env.ale, env.action_set[action]) + end + ob = _get_obs(env) + return ob, reward, ArcadeLearningEnvironment.game_over(env.ale), Dict(:ale_lives => lives(env.ale)) +end + + +function get_preprocessed_RGB(env::AtariEnv) + # getScreenRGB returns an `Array{UInt8, 1}`, which needs to be converted to + # an `Array{UInt8, 3}`. + screen_grab = getScreenRGB(env.ale) + w, h = getScreenWidth(env.ale), getScreenHeight(env.ale) + + screen_grab = reshape(screen_grab, 3, w, h) + + # This step is neccessay because the image we recieve is row major + screen_grab = permutedims(screen_grab, [3, 2, 1]) + return screen_grab +end + +function get_preprocessed_greyscale(env::AtariEnv) + screen_grab = getScreenGrayscale(env.ale) ./ 255f0 + w, h = getScreenWidth(env.ale), getScreenHeight(env.ale) + + screen_grab = reshape(screen_grab, Int(w), Int(h)) + + return permutedims(screen_grab, [2, 1]) +end + +_get_obs(env::AtariEnv) = env.obs_type == :ram ? getRAM(env.ale) : + (env.obs_type == :color ? get_preprocessed_RGB(env) : + get_preprocessed_greyscale(env)) + + +function reset!(env::AtariEnv) + reset_game(env.ale) + _get_obs(env) +end + + +function clone_state(env::AtariEnv) + state_ref = cloneState(env.ale) + state = encodeState(state_ref) + deleteState(state_ref) + return state +end + +function restore_state(env::AtariEnv, state::Array{Int8, 1}) + state_ref = decodeState(state) + restoreState(env.ale, state_ref) + deleteState(state_ref) +end + + +ACTION_MEANING = [ + (:NOOP,), + (:FIRE,), + (:UP,), + (:RIGHT,), + (:LEFT,), + (:DOWN,), + (:UP, :RIGHT), + (:UP, :LEFT), + (:DOWN, :RIGHT), + (:DOWN, :LEFT), + (:UP, :FIRE), + (:RIGHT, :FIRE), + (:LEFT, :FIRE), + (:DOWN, :FIRE), + (:UP, :RIGHT, :FIRE), + (:UP, :LEFT, :FIRE), + (:DOWN, :RIGHT, :FIRE), + (:DOWN, :LEFT, :FIRE) +] + +get_action_meanings(env::AtariEnv) = [ACTION_MEANING[i] for i in env.action_set] + + +function get_keys_to_action(env::AtariEnv) + KEYWORD_TO_KEY = Dict( + :UP => Int('w'), + :DOWN => Int('s'), + :LEFT => Int('a'), + :RIGHT => Int('d'), + :FIRE => Int(' ') + ) + + keystoaction = Dict() + + for (action_id, action_meaning) ∈ enumerate(get_action_meanings(env)) + keys_ = [key for (keyword, key) ∈ collect(zip(keys(KEYWORD_TO_KEY), values(KEYWORD_TO_KEY))) + if keyword ∈ action_meaning] |> sort |> Tuple + keystoaction[keys_] = action_id + end + keystoaction +end + + +Base.show(io::IO, env::AtariEnv) = print(io, "AtariEnv($(env.game_path), $(env.obs_type))") +export get_keys_to_action, restore_state, clone_state diff --git a/src/Envs/atari/vis/utils.jl b/src/Envs/atari/vis/utils.jl new file mode 100644 index 0000000..f3d6536 --- /dev/null +++ b/src/Envs/atari/vis/utils.jl @@ -0,0 +1,59 @@ +using Cairo, Gtk, Colors + +struct AtariCairoCtx <: AbstractCtx end +struct AtariRGBCtx <: AbstractCtx end +struct NoCtx <: AbstractCtx end +struct AtariGtkCtx <: AbstractCtx + canvas::GtkCanvas + win::GtkWindowLeaf +end + +function Ctx(env::AtariEnv, mode::Symbol=:no_render) + if mode == :human_pane + AtariCairoCtx() + + elseif mode == :human_window + canvas = @GtkCanvas() + win = GtkWindow(canvas, "AtariEnv") + show(canvas) + visible(win, false) + signal_connect(win, "delete-event") do widget, event + ccall((:gtk_widget_hide_on_delete, Gtk.libgtk), Bool, (Ptr{GObject},), win) + end + + AtariGtkCtx(canvas, win) + elseif mode == :rgb + + AtariRGBCtx() + elseif mode == :no_render + NoCtx() + else + error("Unrecognized mode in Ctx(): $(mode)") + end +end + +function drawcanvas(env::AtariEnv) + screen_grab = getScreenRGB(env.ale) + w, h = getScreenWidth(env.ale), getScreenHeight(env.ale) + + # Converting RGB values to 32 bit color... + r_screen_grab = screen_grab[1:3:end] * 0x00010000 + g_screen_grab = screen_grab[2:3:end] * 0x00000100 + b_screen_grab = screen_grab[3:3:end] + + rgb_array = reshape(r_screen_grab .+ b_screen_grab .+ g_screen_grab, w, h) |> transpose |> Array +end + +render!(env::AbstractEnv, ctx::NoCtx) = nothing +render!(env::AtariEnv, ctx::AtariCairoCtx) = env |> drawcanvas |> CairoRGBSurface |> display + +function render!(env::AtariEnv, ctx::AtariGtkCtx) + !visible(ctx.win) && visible(ctx.win, true) + @guarded draw(ctx.canvas) do widget + context = getgc(ctx.canvas) + image(context, CairoRGBSurface(drawcanvas(env)), + 0, 0, getScreenWidth(env.ale), getScreenHeight(env.ale)) + end +end + +render!(env::AtariEnv, ctx::AtariRGBCtx) = get_preprocessed_RGB(env) diff --git a/src/Envs/classic_control/Acrobot.jl b/src/Envs/classic_control/Acrobot.jl new file mode 100644 index 0000000..ba012d4 --- /dev/null +++ b/src/Envs/classic_control/Acrobot.jl @@ -0,0 +1,216 @@ +using Random +using GymSpace: Box, Discrete + +""" +Copyright 2013, RLPy http://acl.mit.edu/RLPy +Credit: Alborz Geramifard, Robert H. Klein, Christoph Dann, + William Dabney, Jonathan P. How +License: "BSD 3-Clause" +Author: "Christoph Dann " +""" +mutable struct AcrobotEnv <: AbstractEnv + viewer + observation_space::Box + action_space::Discrete + state + dt::Float32 + LINK_LENGTH_1::Float32 # [m] + LINK_LENGTH_2::Float32 # [m] + LINK_MASS_1::Float32 # [kg] mass of link 1 + LINK_MASS_2::Float32 # [kg] mass of link 2 + LINK_COM_POS_1::Float32 # [m] position of the center of mass of link 1 + LINK_COM_POS_2::Float32 # [m] position of the center of mass of link 2 + LINK_MOI::Float32 # moments of intertia for both links + MAX_VEL_1::Float32 + MAX_VEL_2::Float32 + AVAIL_TORQUE::Array{Float32, N} where N + torque_noise_max::Float32 + book_or_nips::String + seed::MersenneTwister +end + +include("vis/acrobot.jl") + +function AcrobotEnv() + dt = 2f-1 + LINK_LENGTH_1 = 1f0 + LINK_LENGTH_2 = 1f0 + LINK_MASS_1 = 1f0 + LINK_MASS_2 = 1f0 + LINK_COM_POS_1 = 5f-1 + LINK_COM_POS_2 = 5f-1 + LINK_MOI = 1f0 + + MAX_VEL_1 = Float32(4 * pi) + MAX_VEL_2 = Float32(9 * pi) + + AVAIL_TORQUE = [-1f0, 0f0, 1f0] + torque_noise_max = 0f0 + + # use dynamics equations from the nips paper or the book + book_or_nips = "book" + # action_arrow = nothing + # domain_fig = nothing + # actions_num = 3 + + viewer = nothing + high = [1f0, 1f0, 1f0, 1f0, MAX_VEL_1, MAX_VEL_2] #// + low = -high + observation_space = Box(low, high, Float32) + action_space = Discrete(3) + state = nothing + seed = MersenneTwister() + + AcrobotEnv(viewer, observation_space, action_space, state, dt, LINK_LENGTH_1, + LINK_LENGTH_2, LINK_MASS_1, LINK_MASS_2, LINK_COM_POS_1, LINK_COM_POS_2, + LINK_MOI, MAX_VEL_1, MAX_VEL_2, AVAIL_TORQUE, torque_noise_max, book_or_nips, seed) +end + +function seed!(env::AcrobotEnv, seed::Unsigned) + env.seed = MersenneTwister(seed) + return nothing +end + +function reset!(env::AcrobotEnv) + env.state = param(rand(env.seed, Float32, 4) * 2f-1 .- 1f-1) + return _get_ob(env) +end + +#= +function custom_reset!(env::AcrobotEnv, args=[-0.04616826f0, -0.04653378f0, 0.07321809f0, -0.04461966f0]) + env.state = param(args) + return _get_ob(env) +end +=# +get_torque(env::AcrobotEnv, action) = env.AVAIL_TORQUE[action] + +function step!(env::AcrobotEnv, action) + @assert action ∈ env.action_space "$(action) is not in the action space. Enter a valid action." + s = env.state + torque = get_torque(env, action) + + # Add noise to the force action + env.torque_noise_max > 0 && (torque += env.torque_noise_max * (2rand(env.seed, Float32) - 1)) + + # Now, augment the state with our force action so it can be passed to _dsdt + s_augmented = vcat(s, torque) + + + ns = rk4(env, _dsdt, s_augmented, [0, env.dt]) + # only care about final timestep of integration returned by integrator + ns = ns[end, :] + + ns = vcat(wrap(ns[1:2], -π, π), bound(ns[3:4], -env.MAX_VEL_1, env.MAX_VEL_2)) # omit action + env.state = ns + terminal = _terminal(env) + reward = !all(terminal) ? -1f0 : 0f0 + + return _get_obs(env), reward, terminal, Dict() +end + + +function _get_obs(env::AcrobotEnv) + s = env.state + return vcat(cos.(s[1:1]), sin.(s[1:1]), cos.(s[2:2]), sin.(s[2:2]), s[3:4]) +end + + +function _terminal(env::AcrobotEnv) + s = env.state + return cos(s[1]) - cos(s[2] + s[1]) > 1f0 +end + + +function _dsdt(env::AcrobotEnv, s_augmented, t) + m1 = env.LINK_MASS_1 + m2 = env.LINK_MASS_2 + l1 = env.LINK_LENGTH_1 + lc1 = env.LINK_COM_POS_1 + lc2 = env.LINK_COM_POS_2 + I1 = env.LINK_MOI + I2 = env.LINK_MOI + g = 98f-1 + a = s_augmented[end:end] + s = s_augmented[1:end-1] + theta1 = s[1:1] + theta2 = s[2:2] + dtheta1 = s[3:3] + dtheta2 = s[4:4] + + d1 = m1 * lc1^2 .+ m2 * (l1^2 .+ lc2^2 .+ 2l1 * lc2 * cos.(theta2)) .+ I1 .+ I2 + + d2 = m2 * (lc2^2 .+ l1 * lc2 * cos.(theta2)) .+ I2 + + phi2 = m2 * lc2 * g * cos.(theta1 .+ theta2 .- pi/2f0) + + phi1 = - m2 * l1 * lc2 * dtheta2.^2 .* sin.(theta2) .- + 2 * m2 * l1 * lc2 * dtheta2 .* dtheta1 .* sin.(theta2) .+ + (m1 * lc1 + m2 * l1) * g * cos.(theta1 .- Float32(pi) / 2) .+ phi2 + + if env.book_or_nips == "nips" + # the following line is consistent with the description in the paper + ddtheta2 = (a .+ d2./d1 .* phi1 .- phi2) ./ + (m2 .* lc2^2 .+ I2 .- d2.^2 ./d1) + else + # the following line is consistent with the java impelemtation and the + # book + ddtheta2 = (a .+ d2 ./ d1 .* phi1 .- m2 .* l1 .* lc2 .* dtheta1.^2 .* sin.(theta2) .-phi2) ./ + (m2 .* lc2^2 .+ I2 .- d2.^2 ./ d1) + end + ddtheta1 = -(d2 .* ddtheta2 .+ phi1) ./ d1 + + return vcat(dtheta1, dtheta2, ddtheta1, ddtheta2, 0f1) +end + +#= +x : value to be wrapped\n +m : minimum possible value in range\n +M : maximum possible value in range\n +Wraps ``x`` so m <= x <= M; but unlike ``bound()`` which truncates, +``wrap()`` wraps x around the coordinate system defined by m, M.\n +For example, m = -180, M = 180 (degrees), x = 360 --> returns 0. +=# +function wrap(x, m, M) + diff = M - m + while all(x .> M) + x .-= diff + end + while all(x .< m) + x .+= diff + end + return x +end + +# returns m <= x <= M +bound(x, m, M) = min.(max.(x, m), M) + +#= +y0 : initial state vector +t : sample times +derivs : returns the derivative of the system and has the + signature ``dy = derivs(yi, ti)`` +args : additional arguments passed to the derivative function +kwargs : additional keyword arguments passed to the derivative function +=# +function rk4(env::AcrobotEnv, derivs, y0, t, args...; kwargs...) + yout = param(zeros(Float32, length(t), length(y0))) + yout.data[1, :] = y0.data + + for i=1:(length(t)-1) + thist = t[i] + dt = t[i+1] - thist + dt2 = dt / 2f0 + y0 = yout[i, :] + + k1 = derivs(env, y0, thist, args...; kwargs...) + k2 = derivs(env, y0 .+ dt2 * k1, thist .+ dt2, args...; kwargs...) + k3 = derivs(env, y0 .+ dt2 * k2, thist .+ dt2, args...; kwargs...) + k4 = derivs(env, y0 .+ dt * k3, thist .+ dt, args...; kwargs...) + + yout.data[i+1, :] = (y0 .+ dt / 6f0 * (k1 .+ 2 * k2 .+ 2 * k3 .+ k4)).data + + end + return yout +end + +show(io::IO, env::AcrobotEnv) = print(io, "AcrobotEnv") diff --git a/src/Envs/classic_control/CartPole.jl b/src/Envs/classic_control/CartPole.jl index 6b0ea52..f1b4808 100644 --- a/src/Envs/classic_control/CartPole.jl +++ b/src/Envs/classic_control/CartPole.jl @@ -1,4 +1,5 @@ -using .Space: Box, Discrete +using Random +using GymSpaces: Box, Discrete mutable struct CartPoleEnv <: AbstractEnv gravity::Float32 @@ -18,8 +19,8 @@ mutable struct CartPoleEnv <: AbstractEnv observation_space::Box viewer state - steps_beyond_done + seed::MersenneTwister end include("vis/cartpole.jl") @@ -53,10 +54,11 @@ function CartPoleEnv() state = nothing steps_beyond_done = nothing + seed = MersenneTwister() CartPoleEnv( gravity, masscart, masspole, total_mass, length, polemass_length, force_mag, τ, kinematics_integrator, θ_threshold_radians, x_threshold, - action_space, observation_space, viewer, state, steps_beyond_done) + action_space, observation_space, viewer, state, steps_beyond_done, seed) end function step!(env::CartPoleEnv, action) @@ -105,7 +107,7 @@ function step!(env::CartPoleEnv, action) end function reset!(env::CartPoleEnv) - env.state = rand(Float32, 4) * 1f-1 .- 5f-2 + env.state = rand(env.seed, Float32, 4) * 1f-1 .- 5f-2 if isdefined(Main, :CuArrays) env.state = env.state |> gpu @@ -117,3 +119,10 @@ function reset!(env::CartPoleEnv) end Base.show(io::IO, env::CartPoleEnv) = print(io, "CartPoleEnv") + +_get_obs(env::CartPoleEnv) = env.state + +function seed!(env::CartPoleEnv, seed::Unsigned) + env.seed = MersenneTwister(seed) + return nothing +end diff --git a/src/Envs/classic_control/Continuous-MountainCar.jl b/src/Envs/classic_control/Continuous-MountainCar.jl index 066f535..f630d86 100644 --- a/src/Envs/classic_control/Continuous-MountainCar.jl +++ b/src/Envs/classic_control/Continuous-MountainCar.jl @@ -1,4 +1,5 @@ -using .Space: Box +using Random +using GymSpaces: Box mutable struct Continuous_MountainCarEnv <: AbstractEnv min_action::Float32 @@ -12,8 +13,11 @@ mutable struct Continuous_MountainCarEnv <: AbstractEnv state action_space::Box observation_space::Box + seed::MersenneTwister end +include("vis/mountaincar.jl") + function Continuous_MountainCarEnv() min_action = -1f0 max_action = 1f0 @@ -24,8 +28,9 @@ function Continuous_MountainCarEnv() power = 15f-4 action_space = Box(min_action, max_action, (1,), Float32) observation_space = Box([min_position, -max_speed], [max_position, max_speed], Float32) + seed = MersenneTwister() Continuous_MountainCarEnv(min_action, max_action, min_position, max_position, - max_speed, goal_position, power, nothing, action_space, observation_space) + max_speed, goal_position, power, nothing, action_space, observation_space, seed) end function step!(env::Continuous_MountainCarEnv, action) @@ -54,7 +59,14 @@ function step!(env::Continuous_MountainCarEnv, action) end function reset!(env::Continuous_MountainCarEnv) - env.state = param([2f-1rand(Float32) - 6f-1, 0f0]) + env.state = param([2f-1rand(env.seed, Float32) - 6f-1, 0f0]) end -Base.show(io::IO, env::Continuous_MountainCarEnv) = print(io, "Continuous-MountainCarEnv") +Base.show(io::IO, env::Continuous_MountainCarEnv) = print(io, "Continuous-MountainCarEnv")s + +_get_obs(env::Continuous_MountainCarEnv) = env.state + +function seed!(env::ContinousMountainCarEnv, seed::Unsigned) + env.seed = MersenneTwister(seed) + return nothing +end diff --git a/src/Envs/classic_control/MountainCar.jl b/src/Envs/classic_control/MountainCar.jl new file mode 100644 index 0000000..b93ec87 --- /dev/null +++ b/src/Envs/classic_control/MountainCar.jl @@ -0,0 +1,77 @@ +using Random +using GymSpaces: Discrete, Box + +mutable struct MountainCarEnv <: AbstractEnv + min_position::Float32 + max_position::Float32 + max_speed::Float32 + goal_position::Float32 + + force::Float32 + gravity::Float32 + + low::Array{Float32, 1} + high::Array{Float32, 1} + + action_space::Discrete + observation_space::Box + state + seed::MersenneTwister +end + +include("vis/mountaincar.jl") + +function MountainCarEnv() + min_position = -1.2f0 + max_position = 0.6f0 + max_speed = 0.07f0 + goal_position = 0.5f0 + + force = 0.001f0 + gravity = 0.0025f0 + + low = [min_position, -max_speed] + high = [max_position, max_speed] + + action_space = Discrete(5) + observation_space = Box(low, high, Float32) + state + + seed = MersenneTwister() + MountainCarEnv(min_position, max_position, max_speed, goal_position, + force, gravity, + low, high, + action_space, observation_space, nothing, seed) +end + +function step!(env::MountainCarEnv, action) + @assert action ∈ env.action_space "$(action) is unavailable for this environment." + + position, velocity = env.state[1:1], env.state[2:2] + v = velocity .+ (action.-2)*env.force .+ cos.(3position)*(-env.gravity) + velocity_ = clamp.(v, -env.max_speed, env.max_speed) + x = position .+ velocity_ + position_ = clamp.(x, env.min_position, env.max_position) + if all(position_ .== env.min_position) && all(velocity_ .< 0) + velocity_ = 0f0velocity_ + end + + done = all(position_ .≥ env.goal_position) + reward = -1f0 + + env.state = vcat(position_, velocity_) + return env.state, reward, done, Dict() +end + +function reset!(env::MountainCarEnv) + env.state = param([2f-1rand(env.seed, Float32) - 6f-1, 0f0]) +end + +Base.show(io::IO, env::MountainCarEnv) = print(io, "MountainCarEnv") + +_get_obs(env::MountainCarEnv) = env.state + +function seed!(env::MountainCarEnv, seed::Unsigned) + env.seed = MersenneTwister(seed)\ + return nothing +end diff --git a/src/Envs/classic_control/Pendulum.jl b/src/Envs/classic_control/Pendulum.jl index 79e83b1..7cdd6d5 100644 --- a/src/Envs/classic_control/Pendulum.jl +++ b/src/Envs/classic_control/Pendulum.jl @@ -1,4 +1,5 @@ -using .Space: Box +using Random +using GymSpaces: Box mutable struct PendulumEnv <: AbstractEnv max_speed::Float32 @@ -9,6 +10,7 @@ mutable struct PendulumEnv <: AbstractEnv state action_space::Box observation_space::Box + seed::MersenneTwister end include("vis/pendulum.jl") @@ -65,3 +67,7 @@ end pi = Float32(π) angle_normalize(x) = ((x + pi) % 2pi + 2pi) % 2pi - pi + +function seed!(env::PendulumEnv, seed::Unsigned) + env.seed = MersenneTwister(seed) +end diff --git a/src/Envs/classic_control/vis/acrobot.jl b/src/Envs/classic_control/vis/acrobot.jl new file mode 100644 index 0000000..7ea23c5 --- /dev/null +++ b/src/Envs/classic_control/vis/acrobot.jl @@ -0,0 +1,108 @@ +include("utils.jl") + +struct AcrobotDrawParams <: AbstractDrawParams end + +function Ctx(env::AcrobotEnv, mode::Symbol=:no_render) + if mode == :human_pane + draw_parameters = AcrobotDrawParams() + viewer = CairoRGBSurface(500, 500) # screen_size = 500 x 500 + + CairoCtx(draw_parameters, viewer) + elseif mode == :human_window + draw_parameters = AcrobotDrawParams() + viewer = CairoRGBSurface(draw_parameters.screen_width, draw_parameters.screen_height) + + canvas = @GtkCanvas() + canvas.backcc = CairoContext(viewer) + win = GtkWindow(canvas, "Acrobot", + draw_parameters.screen_width, draw_parameters.screen_height; resizable=false) + show(canvas) + visible(win, false) + signal_connect(win, "delete-event") do widget, event + ccall((:gtk_widget_hide_on_delete, Gtk.libgtk), Bool, (Ptr{GObject},), win) + end + + GtkCtx(draw_parameters, canvas, win) + elseif mode == :rgb + draw_parameters = AcrobotDrawParams() + viewer = CairoRGBSurface(draw_parameters.screen_width, draw_parameters.screen_height) + + RGBCtx(draw_parameters, viewer) + elseif mode == :no_render + NoCtx() + else + error("Unrecognized mode in Ctx(): $(mode)") + end +end + +function drawcanvas!(env::AcrobotEnv, viewer::CairoContext, parameters::AcrobotDrawParams) + # Set background + set_source_rgb(viewer, 1, 1, 1) + rectangle(viewer, 0, 0, 500, 500) # screen_size = 500 x 500 + fill(viewer) + + # Draw goal + set_source_rgb(viewer, 0, 0, 0) + move_to(viewer, 0, 136) + line_to(viewer, 500, 136) + stroke(viewer) + close_path(viewer) + + # First translation and rotation + translate(viewer, 250, 250) # Bring the origin to the center of the screen + theta1 = -env.state[1].data + rotate(viewer, theta1) + + # Draw the first link + set_source_rgb(viewer, 0, 0.8, 0.8) + set_line_width(viewer, 19) + move_to(viewer, 0, 0) + line_to(viewer, 0, 113) # Link length translated into the view coordinates + stroke(viewer) + set_line_width(viewer, 1) # Reset line width to avoid unexpected surprises + + # Draw the first joint + set_source_rgb(viewer, 0.8, 0.8, 0) + circle(viewer, 0, 0, 10) + fill(viewer) + + # Second translation and rotation + translate(viewer, 0, 113) + theta2 = -env.state[2].data + rotate(viewer, theta2) + + # Draw the second link + set_source_rgb(viewer, 0, 0.8, 0.8) + set_line_width(viewer, 19) + move_to(viewer, 0, 0) + line_to(viewer, 0, 113) + stroke(viewer) + close_path(viewer) + set_line_width(viewer, 1) + + # Draw the second joint + set_source_rgb(viewer, 0.8, 0.8, 0) + circle(viewer, 0, 0, 10) + fill(viewer) + + # Undo all translations and rotations + rotate(viewer, -theta2) + translate(viewer, 0, -113) + rotate(viewer, -theta1) + translate(viewer, -250, -250) + + #= + # Calibration circle + set_source_rgb(viewer, 0, 0, 0) + calibration_radius = 10 + circle(viewer, calibration_radius, calibration_radius, calibration_radius) + set_line_width(viewer, 1) + stroke(viewer) + move_to(viewer, calibration_radius, calibration_radius) + line_to(viewer, 2calibration_radius, calibration_radius) + close_path(viewer) + set_line_width(viewer, 1) + stroke(viewer) + set_line_width(viewer, 1) + =# +end diff --git a/src/Envs/classic_control/vis/cartpole.jl b/src/Envs/classic_control/vis/cartpole.jl index 3a3f19d..1654771 100644 --- a/src/Envs/classic_control/vis/cartpole.jl +++ b/src/Envs/classic_control/vis/cartpole.jl @@ -1,3 +1,5 @@ +include("utils.jl") + obs(env::CartPoleEnv, ::Nothing) = obs(env, zeros(4)) obs(env::CartPoleEnv, (x, x̄, θ, θ̄)) = Dict("x" => x, "theta"=>θ) @@ -27,11 +29,10 @@ CartPoleDrawParams() = 30f0 # cart_height ) -Ctx(env::CartPoleEnv, mode::Symbol = :human_window) = Ctx(env, Val(mode)) - -function Ctx(::CartPoleEnv, ::Val{:human_pane}) - draw_params = CartPoleDrawParams() - viewer = CairoRGBSurface(draw_params.screen_width, draw_params.screen_height) +function Ctx(env::CartPoleEnv, mode::Symbol=:no_render) + if mode == :human_pane + draw_params = CartPoleDrawParams() + viewer = CairoRGBSurface(draw_params.screen_width, draw_params.screen_height) CairoCtx(draw_params, viewer) end @@ -50,7 +51,12 @@ end ccall((:gtk_widget_hide_on_delete, Gtk.libgtk), Bool, (Ptr{GObject},), win) end - GtkCtx(draw_params, canvas, win) + RGBCtx(draw_params, viewer) + elseif mode == :no_render + NoCtx() + else + error("Unrecognized mode in Ctx(): $(mode)") + end end function Ctx(::CartPoleEnv, ::Val{:rgb}) diff --git a/src/Envs/classic_control/vis/mountaincar.jl b/src/Envs/classic_control/vis/mountaincar.jl new file mode 100644 index 0000000..acb6c4f --- /dev/null +++ b/src/Envs/classic_control/vis/mountaincar.jl @@ -0,0 +1,113 @@ +include("utils.jl") + +height_ = (xs) -> 45f-2Float32(sin(3*xs)) + 55f-2 +convert_x = (xs, parameters) -> (xs - -12f-1)*parameters.scale # min_position = -12f-1 +convert_y = (ys, parameters) -> (parameters.screen_height - ys*parameters.scale) + +struct MountainCarDrawParams <: AbstractDrawParams + screen_height::UInt32 + screen_width::UInt32 + scale::Float32 + world_width::Float32 + car_width::Float32 + car_height::Float32 +end + +function MountainCarDrawParams() + + MountainCarDrawParams( + 400, # screen_height + 600, # screen_width + 600/18f-1, # scale + 18f-1, # world_width + 40f0, # car_width + 20f0, # car_height + ) +end + +function Ctx(env::MountainCarEnv, mode::Symbol=:no_render) + if mode == :human_pane + draw_parameters = MountainCarDrawParams() + viewer = CairoRGBSurface(draw_parameters.screen_width, draw_parameters.screen_height) + + CairoCtx(draw_parameters, viewer) + elseif mode == :human_window + draw_parameters = MountainCarDrawParams() + viewer = CairoRGBSurface(draw_parameters.screen_width, draw_parameters.screen_height) + + canvas = @GtkCanvas() + canvas.backcc = CairoContext(viewer) + win = GtkWindow(canvas, "MountainCar", + draw_parameters.screen_width, draw_parameters.screen_height; resizable=false) + show(canvas) + visible(win, false) + signal_connect(win, "delete-event") do widget, event + ccall((:gtk_widget_hide_on_delete, Gtk.libgtk), Bool, (Ptr{GObject},), win) + end + + GtkCtx(draw_parameters, canvas, win) + elseif mode == :rgb + draw_parameters = MountainCarDrawParams() + viewer = CairoRGBSurface(draw_parameters.screen_width, draw_parameters.screen_height) + + RGBCtx(draw_parameters, viewer) + elseif mode == :no_render + NoCtx() + else + error("Unrecognized mode in Ctx(): $(mode)") + end +end + +function drawcanvas!(env::MountainCarEnv, viewer::CairoContext, parameters::MountainCarDrawParams) + xs = collect(Float32.(range(-12f-1, 6f-1, length=100))) + ys = convert_y.(height_.(xs), [parameters]) + xs = convert_x.(xs, [parameters]) + set_source_rgb(viewer, 1, 1, 1) + rectangle(viewer, 0, 0, parameters.screen_width, parameters.screen_height) + fill(viewer) + + # Track + set_source_rgb(viewer, 0, 0, 0) + move_to(viewer, xs[1], ys[1]) + for i=2:length(xs) + line_to(viewer, xs[i], ys[i]) + end + stroke(viewer) + close_path(viewer) + + #Goalpost + goal_coordinates = Pair(convert_x(env.goal_position, parameters), # x_cordinate + convert_y(height_(env.goal_position), parameters)) # y_coordinate + set_source_rgb(viewer, 1, 0, 0) + move_to(viewer, goal_coordinates.first, goal_coordinates.second) + line_to(viewer, goal_coordinates.first, goal_coordinates.second-50) + set_line_width(viewer, 8) + stroke(viewer) + set_line_width(viewer, 1) # Resetting line width + close_path(viewer) + + # Car + cartx = convert_x(env.state[1].data, parameters) + carty = convert_y(height_(env.state[1].data), parameters) + set_source_rgb(viewer, 0, 0, 0) + translate(viewer, cartx, carty) + #rotate(viewer, cos(3state[1])) + rectangle(viewer, parameters.car_width/2, 0, -parameters.car_width, -parameters.car_height) # cartx-car_width/2 + fill(viewer) + + # Wheels + set_source_rgb(viewer, 0.5, 0.5, 0.5) + circle(viewer, parameters.car_width/4, -5, 5) + circle(viewer, -parameters.car_width/4, -5, 5) + fill(viewer) + translate(viewer, -cartx, -carty) # Resetting the cursor + + #= + # Calibration circle + set_source_rgb(viewer, 0, 0, 0) + calibration_radius = 10 + circle(viewer, calibration_radius, calibration_radius, calibration_radius) + set_line_width(viewer, 1) + stroke(viewer) + =# +end diff --git a/src/Envs/classic_control/vis/pendulum.jl b/src/Envs/classic_control/vis/pendulum.jl index 6424339..3d428c5 100644 --- a/src/Envs/classic_control/vis/pendulum.jl +++ b/src/Envs/classic_control/vis/pendulum.jl @@ -1,3 +1,5 @@ +include("utils.jl") + obs(env::PendulumEnv, ::Nothing) = obs(env, [0.0, 0.0]) obs(env::PendulumEnv, (θ, θ̄)) = Flux.data(θ) @@ -22,7 +24,10 @@ PendulumDrawParams() = 5f-2 # axle_radius ) -Ctx(env::PendulumEnv, mode::Symbol = :human_window) = Ctx(env, Val(mode)) +function Ctx(env::PendulumEnv, mode::Symbol=:no_render) + if mode == :human_pane + draw_params = PendulumDrawParams() + viewer = CairoRGBSurface(draw_params.screen_width, draw_params.screen_height) function Ctx(::PendulumEnv, ::Val{:human_pane}) draw_params = PendulumDrawParams() @@ -35,14 +40,11 @@ end draw_params = PendulumDrawParams() viewer = CairoRGBSurface(draw_params.screen_width, draw_params.screen_height) - canvas = @GtkCanvas() - canvas.backcc = CairoContext(viewer) - win = GtkWindow(canvas, "Pendulum", - draw_params.screen_width, draw_params.screen_height; resizable=false) - show(canvas) - visible(win, false) - signal_connect(win, "delete-event") do widget, event - ccall((:gtk_widget_hide_on_delete, Gtk.libgtk), Bool, (Ptr{GObject},), win) + RGBCtx(draw_params, viewer) + elseif mode == :no_render + return + else + error("Unrecognized mode in Ctx(): $(mode)") end GtkCtx(draw_params, canvas, win) @@ -65,6 +67,9 @@ function drawcanvas!(env::PendulumEnv, viewer::CairoContext, params::PendulumDra translate_dist = Pair(params.screen_width/2, params.screen_height/2) translate(viewer, translate_dist.first, translate_dist.second) + # Rotate + rotate(viewer, env.state[2] * env.dt) + # Arm Start Circle set_source_rgb(viewer, 8f-1, 3f-1, 3f-1) move_to(viewer, 0, 0) @@ -87,9 +92,7 @@ function drawcanvas!(env::PendulumEnv, viewer::CairoContext, params::PendulumDra circle(viewer, 0, 0, params.scale * params.axle_radius) fill(viewer) - # Rotate - rotate(viewer, env.state[2] * env.dt) - - # Undo translation + # Undo translation and rotation + rotate(viewer, -env.state[2] * env.dt) translate(viewer, -translate_dist.first, -translate_dist.second) end diff --git a/src/Envs/vis/utils.jl b/src/Envs/classic_control/vis/utils.jl similarity index 63% rename from src/Envs/vis/utils.jl rename to src/Envs/classic_control/vis/utils.jl index 8a0060d..356dfa4 100644 --- a/src/Envs/vis/utils.jl +++ b/src/Envs/classic_control/vis/utils.jl @@ -1,6 +1,5 @@ -using Cairo, Images, Colors +using Cairo, Gtk, Images, Colors -abstract type AbstractCtx end abstract type AbstractDrawParams end struct CairoCtx <: AbstractCtx @@ -13,6 +12,15 @@ struct RGBCtx <: AbstractCtx viewer::Cairo.CairoSurfaceBase{UInt32} end +struct GtkCtx <: AbstractCtx + params::AbstractDrawParams + canvas::GtkCanvas + win::GtkWindowLeaf +end + +struct NoCtx <: AbstractCtx +end + function render!(env::AbstractEnv, ctx::CairoCtx) drawcanvas!(env, CairoContext(ctx.viewer), ctx.params) display(ctx.viewer) @@ -26,21 +34,12 @@ function render!(env::AbstractEnv, ctx::RGBCtx) return rgb_arr end -@init @require Gtk="4c0ca9eb-093a-5379-98c5-f87ac0bbbf44" begin - using Gtk - - struct GtkCtx <: AbstractCtx - params::AbstractDrawParams - canvas::GtkCanvas - win::GtkWindowLeaf - end - - function render!(env::AbstractEnv, ctx::GtkCtx) - !visible(ctx.win) && visible(ctx.win, true) - @guarded draw(ctx.canvas) do widget - drawcanvas!(env, getgc(ctx.canvas), ctx.params) - end - reveal(ctx.canvas, true) - return ctx.canvas - end +function render!(env::AbstractEnv, ctx::GtkCtx) + !visible(ctx.win) && visible(ctx.win, true) + @guarded draw(ctx.canvas) do widget + drawcanvas!(env, getgc(ctx.canvas), ctx.params) + end + reveal(ctx.canvas, true) end + +render!(env::AbstractEnv, ctx::NoCtx) = nothing diff --git a/src/Envs/env_wrapper.jl b/src/Envs/env_wrapper.jl index 78c22e9..c3e4074 100644 --- a/src/Envs/env_wrapper.jl +++ b/src/Envs/env_wrapper.jl @@ -1,49 +1,62 @@ -import Flux.testmode! +using Random abstract type AbstractEnv end +abstract type AbstractCtx end -include("vis/utils.jl") - -IntOrNothing = Union{Int, Nothing} +IntOrNothing = Union{Integer, Nothing} RealOrNothing = Union{Real, Nothing} mutable struct EnvWrapper done::Bool total_reward::RealOrNothing - steps::Int + steps_since_reset::Int + total_steps::Int train::Bool reward_threshold::RealOrNothing max_episode_steps::IntOrNothing + action_space::GymSpaces.AbstractSpace + observation_space::GymSpaces.AbstractSpace _env::AbstractEnv _ctx::AbstractCtx + reset_called_first::Bool end EnvWrapper(env::AbstractEnv, ctx::AbstractCtx, train::Bool=true; reward_threshold=nothing, max_episode_steps=nothing) = -EnvWrapper(false, 0, 0, train, reward_threshold, max_episode_steps, env, ctx) +EnvWrapper(false, 0, 0, 0, train, reward_threshold, max_episode_steps, env.action_space, + env.observation_space, env, ctx, false) function step!(env::EnvWrapper, a) + @assert env.reset_called_first "Cannot call step!(::EnvWrapper, ::Any) before calling reset!(::EnvWrapper)" s′, r, done, dict = step!(env._env, a) env.total_reward = env.total_reward .+ r - env.steps += 1 + env.steps_since_reset += 1 + env.total_steps += 1 env.done = done if !isnothing(env.max_episode_steps) - env.done |= env.steps ≥ env.max_episode_steps + env.done |= env.steps_since_reset ≥ env.max_episode_steps end - return s′, r, done, dict + return s′, r, env.done, dict end function reset!(env::EnvWrapper) + env.reset_called_first = true env.done = false env.total_reward = 0 - env.steps = 0 + env.steps_since_reset= 0 reset!(env._env) end +function seed!(env::EnvWrapper, seed::Int=42) + seed!(env._env, UInt(seed)) + println("Resetting...") + reset!(env) +end + render!(env::EnvWrapper, ctx::AbstractCtx) = render!(env._env, ctx) render!(env::EnvWrapper) = render!(env, env._ctx) -Ctx(env::EnvWrapper, mode::Symbol = :human_window) = Ctx(env._env, mode) +Ctx(env::EnvWrapper) = Ctx(env._env, env.render_mode) """ Returns the observational state of the environment. The original state can @@ -65,4 +78,4 @@ function testmode!(env::EnvWrapper, val::Bool=true) end trainable(env::EnvWrapper) = env.train -game_over(env::EnvWrapper) = env.done \ No newline at end of file +is_over(env::EnvWrapper) = env.done diff --git a/src/Envs/registration.jl b/src/Envs/registration.jl index cfb778b..161ef63 100644 --- a/src/Envs/registration.jl +++ b/src/Envs/registration.jl @@ -21,7 +21,7 @@ struct EnvSpec entry_point::String trials::Int - reward_threshold::Union{Int, Nothing} + reward_threshold::Union{Float32, Nothing} nondeterministic::Bool tags::Dict{String, Any} max_episode_steps::Union{Int, Nothing} @@ -41,12 +41,12 @@ function EnvSpec(id, entry_point; trials=100, reward_threshold=nothing, end function _make(spec::EnvSpec, render_mode::Symbol; kwargs...) - _kwargs = deepcopy(spec.kwargs) - merge!(_kwargs, Dict(kwargs)) + kwargs_ = deepcopy(spec.kwargs) + merge!(kwargs_, Dict(kwargs)) env_var = load(spec.entry_point, spec.id) ctx_var = load(spec.entry_point, :Ctx) - env = Base.invokelatest(env_var) + env = Base.invokelatest(env_var; kwargs_...) ctx = Base.invokelatest(ctx_var, env, render_mode) env, ctx, spec.reward_threshold, spec.max_episode_steps end @@ -79,11 +79,15 @@ function _make(reg::EnvRegistry, id_string, render_mode::Symbol; kwargs...) spec = get(reg.env_specs, id_string, nothing) isnothing(spec) && - (throw(ErrorException("Environment $(id_string) not found in the registry. Please ensure that you've spelled the name correctly."))) + (throw(ArgumentError("Environment $(id_string) not found in the registry. Please ensure that you've spelled the name correctly."))) _make(spec, render_mode; kwargs...) end +function _spec_list(reg::EnvRegistry) + return keys(reg.env_specs) +end + registry = EnvRegistry() """ @@ -105,7 +109,14 @@ Optional keyword arguments: include("env_wrapper.jl") -function make(id_string, mode=:human_pane, train=true; kwargs...) +function make(id_string, mode=:no_render, train=true; kwargs...) env, ctx, rt, max_ep_steps = _make(registry, id_string, mode; kwargs...) EnvWrapper(env, ctx, train; reward_threshold=rt, max_episode_steps=max_ep_steps) end + +""" + speclist() + +Returns the list of all available environments in the gym. +""" +speclist() = _spec_list(registry) diff --git a/src/Envs/registry.jl b/src/Envs/registry.jl index 527411d..71ea4a9 100644 --- a/src/Envs/registry.jl +++ b/src/Envs/registry.jl @@ -24,3 +24,206 @@ register("MountainCarContinuous-v0", "/classic_control/Continuous-MountainCar.jl", max_episode_steps=999, reward_threshold=90.0) + +register("MountainCar-v0", + :MountainCarEnv, + "/classic_control/MountainCar.jl", + max_episode_steps=200, + reward_threshold=110.0) + +register("Acrobot-v0", + :AcrobotEnv, + "/classic_control/Acrobot.jl", + max_episode_steps=500, + reward_threshold=100.0) + + +# Algorithmic environments +#=============================================================================# + +register("Copy-v0", + :CopyEnv, + "/algorithmic/Copy.jl", + max_episode_steps=200, + reward_threshold=25.0) + +register("RepeatCopy-v0", + :RepeatCopyEnv, + "/algorithmic/RepeatCopy.jl", + max_episode_steps=200, + reward_threshold=75.0) + +register("ReversedAddition-v0", + :ReversedAdditionEnv, + "/algorithmic/ReversedAddition.jl", + max_episode_steps=200, + reward_threshold=25.0) + +register("DuplicatedInput-v0", + :DuplicatedInputEnv, + "/algorithmic/DuplicatedInput.jl", + max_episode_steps=200, + reward_threshold=25.0) + +register("Reverse-v0", + :ReverseEnv, + "/algorithmic/Reverse.jl", + max_episode_steps=200, + reward_threshold=25.0) + + +# Atari environments +#=============================================================================# + +games_list = ["adventure", "air_raid", "alien", "amidar", "assault", "asterisk", "atlantis", + "bank_heist", "battle_zone", "beam_rider", "berzerk", "bowling", "boxing", "breakout", "carnival", + "centipede", "chopper_command", "crazy_climber", "defender", "demon_attack", "double_dunk", + "elevator_action", "enduro", "fishing_derby", "freeway", "frostbite", "gopher", "gravitar", + "hero", "ice_hockey", "jamesbond", "journey_escape", "kangaroo", "krull", "kung_fu_master", + "montezuma_revenge", "ms_pacman", "name_this_game", "pheonix", "pitfall", "pong", "pooyan", + "private_eye", "qbert", "riverraid", "road_runner", "robotank", "seaquest", "skiing", + "solaris", "space_invaders", "star_gunner", "tennis", "time_pilot", "tutankham", + "up_n_down", "venture", "video_pinball", "wizard_of_wor", "yars_revenge", "zaxxon"] + +for game ∈ games_list + for obs_type ∈ (:ram, :color, :grey) + # space_invaders should yield SpaceInvaders-v0 and SpaceInvaders-ram-v0 + name = game |> (g) -> split(g, '_') .|> titlecase |> join + obs_type == :ram && (name = "$(name)-ram") + obs_type == :grey && (name = "$(name)-grey") + + # ElevatorAction-ram-v0 seems to yield slightly + # non-deterministic observations about 10% of the time. We + # should track this down eventually, but for now we just + # mark it as nondeterministic. + nondeterministic = game=="elevator_action" && obs_type==:ram + + register("$(name)-v0", + :AtariEnv, + "/atari/Atari.jl", + max_episode_steps = 10000, + nondeterministic = nondeterministic, + kwargs = Dict( + :game_path => game, + :obs_type => obs_type, + :repeat_action_probability => 0.25f0 + ) + ) + + register("$(name)-v4", + :AtariEnv, + "/atari/Atari.jl", + max_episode_steps = 100000, + nondeterministic = nondeterministic, + kwargs = Dict( + :game_path => game, + :obs_type => obs_type, + ) + ) + + frameskip = game=="space_invaders" ? 3 : 4 + + register("$(name)Deterministic-v0", + :AtariEnv, + "/atari/Atari.jl", + max_episode_steps = 100000, + nondeterministic = nondeterministic, + kwargs = Dict( + :game_path => game, + :obs_type => obs_type, + :frameskip => frameskip + ) + ) + + register("$(name)Deterministic-v4", + :AtariEnv, + "/atari/Atari.jl", + max_episode_steps = 100000, + nondeterministic = nondeterministic, + kwargs = Dict( + :game_path => game, + :obs_type => obs_type, + :frameskip => frameskip + ) + ) + + register("$(name)NoFrameskip-v0", + :AtariEnv, + "/atari/Atari.jl", + max_episode_steps = frameskip * 100000, + nondeterministic = nondeterministic, + kwargs = Dict( + :game_path => game, + :obs_type => obs_type, + :frameskip => 1, + :repeat_action_probability => 25f-2 + ) + ) + + + # No frameskip. (Atari has no entropy source, so these are + # deterministic environments.) + register("$(name)NoFrameskip-v4", + :AtariEnv, + "/atari/Atari.jl", + max_episode_steps = frameskip * 100000, + nondeterministic = nondeterministic, + kwargs = Dict( + :game_path => game, + :obs_type => obs_type, + :frameskip => 1 + ) + ) + end +end + + +# Toy Text +#==============================================================================# + +register("Blackjack-v0", + :BlackjackEnv, + "/toytext/blackjack.jl") + +register("KellyCoinflip-v0", + :KellyCoinFlipEnv, + "/toytext/kellycoinflip.jl", + reward_threshold=246.61) + +register("FrozenLake-v0", + :FrozenLakeEnv, + "/toytext/frozen_lake.jl", + kwargs = Dict( + :map_name => :fourxfour + ), + max_episode_steps=100, + reward_threshold=0.78) + +register("FrozenLake8x8-v0", + :FrozenLakeEnv, + "/toytext/frozen_lake.jl", + kwargs = Dict( + :map_name => :eightxeight + ), + max_episode_steps=200, + reward_threshold=0.99) + +register("CliffWalking-v0", + :CliffWalkingEnv, + "/toytext/cliffwalking.jl") + +register("Taxi-v2", + :TaxiEnv, + "/toytext/taxi.jl", + reward_threshold=8, + max_episode_steps=200) + +register("GuessingGame-v0", + :GuessingGameEnv, + "/toytext/guessing_game.jl", + max_episode_steps=200) + +register("HotterColder-v0", + :HotterColderEnv, + "/toytext/hotter_colder.jl", + max_episode_steps=200) diff --git a/src/Envs/toytext/blackjack.jl b/src/Envs/toytext/blackjack.jl new file mode 100644 index 0000000..61fbfae --- /dev/null +++ b/src/Envs/toytext/blackjack.jl @@ -0,0 +1,103 @@ +using GymSpaces: Discrete, TupleSpace, AbstractSpace +using Random + +# 1 = Ace, 2-10 = Number cards, Jack/Queen/King = 10 +deck = vcat(collect(1:10), [10, 10, 10]) + +@inline draw_card(seed::MersenneTwister) = rand(seed, deck) +@inline draw_hand(seed::MersenneTwister) = [draw_card(seed), draw_card(seed)] +@inline usable_ace(hand) = 1 ∈ hand && sum(hand) + 10 ≤ 21 +@inline sum_hand(hand) = usable_ace(hand) ? sum(hand) + 10 : sum(hand) +@inline is_bust(hand) = sum_hand(hand) > 21 +@inline score(hand) = is_bust(hand) ? 0 : sum_hand(hand) +@inline isnatural(hand) = sort(hand) == [1, 10] + +mutable struct BlackjackEnv <: AbstractEnv + action_space::AbstractSpace + observation_space::AbstractSpace + natural::Bool + dealer::Array + player::Array + seed::MersenneTwister +end + +include("vis/blackjack.jl") + +"""Simple blackjack environment + +Blackjack is a card game where the goal is to obtain cards that sum to as +near as possible to 21 without going over. They're playing against a fixed +dealer. +Face cards (Jack, Queen, King) have point value 10. +Aces can either count as 11 or 1, and it's called 'usable' at 11. +This game is placed with an infinite deck (or with replacement). +The game starts with each (player and dealer) having one face up and one +face down card. + +The player can request additional cards (hit=1) until they decide to stop +(stick=0) or exceed 21 (bust). + +After the player sticks, the dealer reveals their facedown card, and draws +until their sum is 17 or greater. If the dealer goes bust the player wins. + +If neither player nor dealer busts, the outcome (win, lose, draw) is +decided by whose sum is closer to 21. The reward for winning is +1, +drawing is 0, and losing is -1. + +The observation of a 3-tuple of: the players current sum, +the dealer's one showing card (1-10 where 1 is ace), +and whether or not the player holds a usable ace (0 or 1). + +This environment corresponds to the version of the blackjack problem +described in Example 5.1 in Reinforcement Learning: An Introduction +by Sutton and Barto. +http://incompleteideas.net/book/the-book-2nd.html +""" +function BlackjackEnv(natural=false) + # natural = flag to payout on a "natural" blackjack win, like casion rules + action_space = Discrete(2) + observation_space = TupleSpace([ + Discrete(32), + Discrete(11), + Discrete(2) + ]) + seed = MersenneTwister() + BlackjackEnv(action_space, observation_space, natural, [], [], seed) +end + +@inline _get_obs(env::BlackjackEnv) = (sum_hand(env.player), env.dealer[1], usable_ace(env.player)) + +function reset!(env::BlackjackEnv) + env.dealer = draw_hand(env.seed) + env.player = draw_hand(env.seed) + get_obs(env) +end + +function step!(env::BlackjackEnv, action) + @assert action ∈ env.action_space "Invalid action" + + if action == 1 # hit: add a card to the players hand and return + push!(env.player, draw_card(env.seed)) + done, reward = is_bust(env.player) ? (true, -1) : (false, 0) + + else # stick: play out the dealers hand, and score + done = true + while sum_hand(env.dealer) < 17 + push!(env.dealer, draw_card(env.seed)) + end + reward = Float32(cmp(score(env.player), score(env.dealer))) + if env.natural && isnatural(env.player) && reward == 1f0 + reward = 1.5f0 + end + end + return _get_obs(env), reward, done, Dict{Nothing, Nothing}() +end + +function drawcanvas!(env::BlackjackEnv) + return _get_obs(env) +end + +function seed!(env::BlackjackEnv, seed::Unsigned) + env.seed = MersenneTwister(seed) + return nothing +end diff --git a/src/Envs/toytext/cliffwalking.jl b/src/Envs/toytext/cliffwalking.jl new file mode 100644 index 0000000..da7918e --- /dev/null +++ b/src/Envs/toytext/cliffwalking.jl @@ -0,0 +1,101 @@ +using GymSpaces: Discrete + +include("discrete.jl") + +mutable struct CliffWalkingEnv <: DiscreteEnv + shape + start_state_index + + _cliff + discenv_obj + action_space::Discrete + observation_space::Discrete +end + +include("vis/cliffwalking.jl") + +UP = 1 +RIGHT = 2 +DOWN = 3 +LEFT = 4 + +@inline ind2sub(index, shape) = Int[((index-1) % shape[1])+1, ceil(index/shape[1])] + +function CliffWalkingEnv() + shape = [4, 12] + start_state_index = 4 + + nS = prod(shape) + nA = 4 + + _cliff = zeros(Bool, shape...) + _cliff[4, 2:11] .= true + + P = Dict() + + """Determine the outcome for an action. Transition Prob is always 1.0 + returns a tuple of form `(1.0, new_state, reward, done)`""" + function calculate_transition_prob(current, delta) + new_position = current .+ delta + limit_coordinates!(new_position) + new_state = shape[1] * (new_position[2] - 1) + new_position[1] + _cliff[new_position...] && + return [(1.0, start_state_index, -100, false)] + terminal_state = shape + is_done = all(new_position .== terminal_state) + return [(1.0, new_state, -1, is_done)] + end + + """Don't fall off...""" + function limit_coordinates!(coord) + coord[1] = clamp(coord[1], 1, shape[1]) + coord[2] = clamp(coord[2], 1, shape[2]) + end + + P = Dict() + for s=1:nS + position = ind2sub(s, shape) + P[s] = Dict(a => [] for a=1:nA) + P[s][UP] = calculate_transition_prob(position, [-1, 0]) + P[s][RIGHT] = calculate_transition_prob(position, [0, 1] ) + P[s][DOWN] = calculate_transition_prob(position, [1, 0] ) + P[s][LEFT] = calculate_transition_prob(position, [0, -1]) + end + + isd = zeros(Float32, nS) + isd[start_state_index] = 1.0 + + discenv_obj = DiscreteEnvObj(nS, nA, P, isd) + action_space = Discrete(nA) + observation_space = Discrete(nS) + CliffWalkingEnv(shape, start_state_index, _cliff, discenv_obj, action_space, observation_space) +end + +function drawcanvas!(env::CliffWalkingEnv) + output = "" + for y ∈ 1:env.shape[1] + output_line = "" + for x ∈ 1:env.shape[2] + s = env.shape[1]*(x-1) + y + if env.s == s + output_line *= " x " + elseif (y, x) == (4, 12) + output_line *= " T " + elseif env._cliff[y, x] + output_line *= " C " + else + output_line *= " o " + end + + if x == 1 + output_line = lstrip(output_line) + end + if x == 12 + output_line = rstrip(output_line) + output *= '\n' + end + end + output *= output_line + end + return output +end diff --git a/src/Envs/toytext/discrete.jl b/src/Envs/toytext/discrete.jl new file mode 100644 index 0000000..08d8410 --- /dev/null +++ b/src/Envs/toytext/discrete.jl @@ -0,0 +1,72 @@ +using Random + +abstract type DiscreteEnv <: AbstractEnv end + +function categorical_sample(prob_n, seed) + csprob_n = cumsum(prob_n) + return argmax(csprob_n .> rand(seed, Float32)) +end + +mutable struct DiscreteEnvObj + P + isd + lastaction + nS + nA + + s + seed::MersenneTwister +end + +function DiscreteEnvObj(nS, nA, P, isd) + seed = MersenneTwister() + + s = categorical_sample(isd, seed) + DiscreteEnvObj(P, isd, nothing, nS, nA, s, seed) +end + +# All DiscreteEnv must have a `discenv_obj` field... +function Base.getproperty(env::DiscreteEnv, sym::Symbol) + if sym == :P + return env.discenv_obj.P + elseif sym == :isd + return env.discenv_obj.isd + elseif sym == :lastaction + return env.discenv_obj.lastaction + elseif sym == :nS + return env.discenv_obj.nS + elseif sym == :nA + return env.discenv_obj.nA + elseif sym == :s + return env.discenv_obj.s + elseif sym == :seed + return env.discenv_obj.seed + else + return Base.getfield(env, sym) + end +end + +function seed!(env::DiscreteEnv, seed::Unsigned) + env.seed = MersenneTwister(seed) + return nothing +end + +function reset!(env::DiscreteEnv) + env.discenv_obj.s = categorical_sample(env.isd, env.seed) + env.discenv_obj.lastaction = nothing + return env.s +end + +function step!(env::DiscreteEnv, action) + @assert action ∈ env.action_space "Invalid action" + transitions = env.discenv_obj.P[env.s][action] + i = categorical_sample([t[1] for t ∈ transitions], env.seed) + p, s, r, d = transitions[i] + env.discenv_obj.s = s + env.discenv_obj.lastaction = action + return s, r, d, Dict(:prob => p) +end + +function _get_obs(env::DiscreteEnv) + return env.discenv_obj.s +end diff --git a/src/Envs/toytext/frozen_lake.jl b/src/Envs/toytext/frozen_lake.jl new file mode 100644 index 0000000..90612f9 --- /dev/null +++ b/src/Envs/toytext/frozen_lake.jl @@ -0,0 +1,176 @@ +using StatsBase: sample +using GymSpaces: Discrete + +include("discrete.jl") + +const LEFT = 1 +const DOWN = 2 +const RIGHT = 3 +const UP = 4 + +mutable struct FrozenLakeEnv <: DiscreteEnv + desc::Array{Char, 2} + nrow::Int + ncol::Int + reward_range + discenv_obj::DiscreteEnvObj + action_space::Discrete + observation_space::Discrete +end + +include("vis/frozen_lake.jl") + +MAPS = Dict( + :fourxfour => [ + "SFFF", + "FHFH", + "FFFH", + "HFFG" + ], + + :eightxeight => [ + "SFFFFFFF", + "FFFFFFFF", + "FFFHFFFF", + "FFFFFHFF", + "FFFHFFFF", + "FHHFFFHF", + "FHFFHFHF", + "FFFHFFFG" + ] +) + +""" + generate_random_map(size::Integer, prob::AbstractFloat) + +Generates a random valid map of size `size` and `prob` determines the probability +of a tile being frozen. +""" +function generate_random_map(shape=8, p=8f-1) + valid = false + + # BFS to check that it's a valid path... + function is_valid(arr, r=1, c=1) + if arr[r, c] == 'G' + return true end + + tmp = arr[r, c] + arr[r, c] = '#' + + if r+1 ≤ shape && !(arr[r+1, c] ∈ "#H") + if is_valid(arr, r+1, c) + arr[r, c] = tmp + return true end end + + if c+1 ≤ shape && !(arr[r, c+1] ∈ "#H") + if is_valid(arr, r, c+1) + arr[r, c] = tmp + return true end end + + if r-1 ≥ 1 && !(arr[r-1, c] ∈ "#H") + if is_valid(arr, r-1, c) + arr[r, c] = tmp + return true end end + + if c-1 ≥ 1 && !(arr[r, c-1] ∈ "#H") + if is_valid(arr, r, c-1) + arr[r, c] = tmp + return true end end + + arr[r, c] = tmp + return false + end + + res = Array{Char, 2}(undef, shape, shape) + while !valid + p = min(1, p) + res = StatsBase.sample(['F', 'H'], ProbabilityWeights(Float32[p, 1-p]), (shape, shape), replace=true) + res[1, 1] = 'S' + res[end, end] = 'G' + valid = is_valid(res) + end + return [join(res[x, :]) for x in 1:size(res, 1)] +end + +function FrozenLakeEnv(desc_::AbstractArray{<:AbstractString, 1}, is_slippery::Bool) + desc = Char.(vcat([Int.(collect(row))' for row in desc_]...)) + + nrow, ncol = size(desc) + + reward_range = [0, 1] + + nA = 4 + nS = nrow * ncol + + isd = Float64.(desc .== 'S') + isd ./= sum(isd) + isd = reshape(isd, length(isd)) + + P = Dict(s => Dict(a => [] for a=1:nA) for s=1:nS) + + @inline to_s(row, col) = (col-1)*nrow + row + + function inc(row, col, a) + if a == LEFT + col = max(col-1, 1) + elseif a == RIGHT + col = min(col+1, ncol) + elseif a == DOWN + row = min(row+1, nrow) + elseif a == UP + row = max(row-1, 1) + end + return (row, col) + end + + for row=1:nrow + for col=1:ncol + s = to_s(row, col) + for a=1:4 + li = P[s][a] + letter = desc[row, col] + if letter ∈ "GH" + push!(li, (1.0, s, 0, true)) + else + if is_slippery + for b ∈[((a-2)%4)+1, a, ((a%4)+1)] + newrow, newcol = inc(row, col, b) + newstate = to_s(newrow, newcol) + newletter = desc[newrow, newcol] + done = newletter == 'G' || newletter == 'H' + rew = Float32(newletter == 'G') + push!(li, (1f0/3f0, newstate, rew, done)) + end + else + newrow, newcol = inc(row, col, a) + newstate = to_s(newrow, newcol) + newletter = desc[newrow, newcol] + done = newletter == 'G' || newletter == 'H' + rew = Float32(newletter == 'G') + push!(li, (1f0, newstate, rew, done)) + end + end + end + end + end + discenv_obj = DiscreteEnvObj(nS, nA, P, isd) + action_space = Discrete(nA) + observation_space = Discrete(nS) + FrozenLakeEnv(desc, nrow, ncol, reward_range, discenv_obj, action_space, observation_space) +end + +function FrozenLakeEnv(; map_name::Union{Symbol,Nothing}=nothing, is_slippery::Bool=true) + desc = isnothing(map_name) ? generate_random_map() : MAPS[map_name] + FrozenLakeEnv(desc, is_slippery) +end + +function drawcanvas!(env::FrozenLakeEnv) + actionarr = ["Left", "Down", "Right", "Up"] + outfile = "" + row, col = ((env.s-1)÷env.ncol) + 1, (env.s-1)%env.ncol + 1 + desc = ["$char" for char in env.desc] + desc[col, row] = colorize(desc[row, col], :red, highlight=true) + outfile *= isnothing(env.lastaction) ? "\n" : " ($(actionarr[env.lastaction]))\n" + outfile *= join([join(desc[row, :], "") for row in 1:env.nrow], "\n") + return outfile +end diff --git a/src/Envs/toytext/guessing_game.jl b/src/Envs/toytext/guessing_game.jl new file mode 100644 index 0000000..3e0ed08 --- /dev/null +++ b/src/Envs/toytext/guessing_game.jl @@ -0,0 +1,77 @@ +using Random +using GymSpaces: Box, Discrete + +mutable struct GuessingGameEnv <: AbstractEnv + range + bounds + action_space + observation_space + number + guess_count + guess_max + observation + seed::MersenneTwister +end + +include("vis/guessing_game.jl") + +function GuessingGameEnv() + seed = MersenneTwister() + range = 1000 + bounds = 10000 + + action_space = Box([-bounds], [bounds], Float32) + observation_space = Discrete(4) + + number = 2range * rand(Float32) - range + guess_count = 0 + guess_max = 200 + observation = 0 + GuessingGameEnv(range, bounds, action_space, observation_space, number, + guess_count, guess_max, observation, seed) +end + +function seed!(env::GuessingGameEnv, seed::Unsigned) + env.seed = MersenneTwister(seed) + return nothing +end + +function reset!(env::GuessingGameEnv) + env.number = 2env.range * rand(Float32) - env.range + env.guess_count = 0 + env.observation = 0 + return env.observation +end + +function step!(env::GuessingGameEnv, action) + @assert action ∈ env.action_space + + if action < env.number + env.observation = 1 + elseif action == env.number + env.observation = 2 + elseif action == env.number + env.observation = 3 + end + + reward = 0 + done = false + + if (env.number - env.range * 0.01) < action < (env.number + env.range * 0.01) + reward = 1 + done = true + end + + env.guess_count += 1 + if env.guess_count ≥ env.guess_max + done = true + end + + return env.observation, reward, done, Dict(:number => env.number, :guesses => env.guess_count) +end + +function drawcanvas!(env::GuessingGameEnv) + return env.observation +end + +_get_obs(env::GuessingGameEnv) = env.observation diff --git a/src/Envs/toytext/hotter_colder.jl b/src/Envs/toytext/hotter_colder.jl new file mode 100644 index 0000000..f920c01 --- /dev/null +++ b/src/Envs/toytext/hotter_colder.jl @@ -0,0 +1,70 @@ +using Random +using GymSpaces: Box, Discrete + +mutable struct HotterColderEnv <: AbstractEnv + range + bounds + action_space + observation_space + number + guess_count + guess_max + observation + seed::MersenneTwister +end + +include("vis/hotter_colder.jl") + +function HotterColderEnv() + seed = MersenneTwister() + range = 1000 + bounds = 2000 + + action_space = Box([-bounds], [bounds], Float32) + observation_space = Discrete(4) + + number = 2range * rand(Float32) - range + guess_count = 0 + guess_max = 200 + observation = 0 + HotterColderEnv(range, bounds, action_space, observation_space, number, guess_count, + guess_max, observation, seed) +end + + +function seed!(env::HotterColderEnv, seed::Unsigned) + env.seed = MersenneTwister(seed) + return nothing +end + +function reset!(env::HotterColderEnv) + env.number = 2range * rand(Float32) - range + env.guess_count = 0 + env.observation = 0 + return env.observation +end + +function step!(env::HotterColderEnv, action) + @assert action ∈ env.action_space + + if action < env.number + env.observation = 1 + elseif action == env.number + env.observation = 2 + elseif action == env.number + env.observation = 3 + end + + reward = ((min(action, env.number) + env.bounds) / (max(action, env.number) + env.bounds)) ^ 2 + + env.guess_count += 1 + done = env.guess_count ≥ env.guess_max + + return env.observation, reward[1], done, Dict(:number => env.number, :guesses => env.guess_count) +end + +function drawcanvas!(env::HotterColderEnv) + return env.observation +end + +_get_obs(env::HotterColderEnv) = env.observation diff --git a/src/Envs/toytext/kellycoinflip.jl b/src/Envs/toytext/kellycoinflip.jl new file mode 100644 index 0000000..340a0f1 --- /dev/null +++ b/src/Envs/toytext/kellycoinflip.jl @@ -0,0 +1,64 @@ +using Random +using Distributions: beta +using GymSpaces: Discrete, TupleSpace, Box + +@inline flip(edge, seed::MersenneTwister) = rand(seed, Float32) < edge ? 1 : -1 + +mutable struct KellyCoinFlipEnv <: AbstractEnv + action_space::Discrete + observation_space::TupleSpace + reward_range + wealth + seed::MersenneTwister + rounds + initial_wealth + edge + max_wealth + max_rounds +end + +include("vis/kellycoinflip.jl") + +function KellyCoinFlipEnv(;initial_wealth=25f0, edge=6f-1, max_wealth=250f0, max_rounds=300) + action_space = Discrete(Int(max_wealth * 100)) # Betting in penny increments + observation_space = TupleSpace([ + Box(0, max_wealth, (1, ), Float32), + Discrete(max_rounds + 1) + ]) + reward_range = (0, max_wealth) + wealth = initial_wealth + seed = MersenneTwister() + rounds = max_rounds + KellyCoinFlipEnv(action_space, observation_space, reward_range, wealth, seed, + rounds, initial_wealth, max_wealth, max_rounds) +end + +function reset!(env::KellyCoinFlipEnv) + env.rounds = env.max_rounds + env.wealth = env.initial_wealth + return _get_obs(env) +end + +_get_obs(env::KellyCoinFlipEnv) = [env.wealth], env.rounds + +function step!(env::KellyCoinFlipEnv, action) + bet_in_dollars = min(action/100.0, env.wealth) + env.rounds -= 1 + + coinflip = flip(env.edge, env.seed) + env.wealth = min(env.max_wealth, env.wealth + coinflip * bet_in_dollars) + + done = env.wealth < 0.01 || env.wealth == env.max_wealth || !Bool(env.rounds) + reward = done ? env.wealth : 0f0 + + return _get_obs(env), reward, done, Dict() +end + +function drawcanvas!(env::KellyCoinFlipEnv) + return "Current wealth: $(env.wealth); Rounds left: $(env.rounds)" +end + +function seed!(env::KellyCointFlip, seed::Unsigned) + env.seed = MersenneTwister(seed) + return nothing +end diff --git a/src/Envs/toytext/taxi.jl b/src/Envs/toytext/taxi.jl new file mode 100644 index 0000000..ace4838 --- /dev/null +++ b/src/Envs/toytext/taxi.jl @@ -0,0 +1,186 @@ +using GymSpaces: Discrete + +include("discrete.jl") + +MAP = [ + "+---------+", + "|R: | : :G|", + "| : : : : |", + "| : : : : |", + "| | : | : |", + "|Y| : |B: |", + "+---------+", +] + +mutable struct TaxiEnv <: DiscreteEnv + desc + locs + discenv_obj::DiscreteEnvObj + action_space::Discrete + observation_space::Discrete +end + +include("vis/taxi.jl") + +""" +The Taxi Problem +from "Hierarchical Reinforcement Learning with the MAXQ Value Function Decomposition" +by Tom Dietterich + +Description: +There are four designated locations in the grid world indicated by R(ed), B(lue), G(reen), and Y(ellow). When the episode starts, the taxi starts off at a random square and the passenger is at a random location. The taxi drive to the passenger's location, pick up the passenger, drive to the passenger's destination (another one of the four specified locations), and then drop off the passenger. Once the passenger is dropped off, the episode ends. + +Observations: +There are 500 discrete states since there are 25 taxi positions, 5 possible locations of the passenger (including the case when the passenger is the taxi), and 4 destination locations. + +Actions: +There are 6 discrete deterministic actions: + - 1: move south + - 2: move north + - 3: move east + - 4: move west + - 5: pickup passenger + - 6: dropoff passenger + +Rewards: +There is a reward of -1 for each action and an additional reward of +20 for delievering the passenger. There is a reward of -10 for executing actions "pickup" and "dropoff" illegally. + + +Rendering: + - blue: passenger + - magenta: destination + - yellow: empty taxi + - green: full taxi + - other letters (R, G, B and Y): locations for passengers and destinations + +actions: + - 1: south + - 2: north + - 3: east + - 4: west + - 5: pickup + - 6: dropoff + + state space is represented by: + (taxi_row, taxi_col, passenger_location, destination) +""" +function TaxiEnv() + desc = Char.(vcat([Int.(collect(row))' for row in MAP]...)) + + locs = [[1, 1], [1, 5], [5, 1], [5, 4]] + + num_states = 500 + num_rows = 5 + num_columns = 5 + max_row = num_rows + max_col = num_columns + initial_state_distrib = zeros(Float32, num_states) + num_actions = 6 + P = Dict(state => Dict(action => [] for action=1:num_actions) for state=1:num_states) + for row=1:num_rows + for col=1:num_columns + for pass_idx=1:length(locs)+1 + for dest_idx=1:length(locs) + state = encode(row, col, pass_idx, dest_idx) + if pass_idx ≤ 4 && pass_idx != dest_idx + initial_state_distrib[state] += 1 + end + for action=1:num_actions + # defaults + new_row, new_col, new_pass_idx = row, col, pass_idx + reward = -1 + done = false + taxi_loc = [row, col] + + if action == 1 + new_row = min(row + 1, max_row) + elseif action == 2 + new_row = max(row - 1, 1) + end + + if action == 3 && desc[row+1, 2*(col-1) + 3] == ':' + new_col = min(col + 1, max_col) + elseif action == 4 && desc[row+1, 2 * (col-1) + 1] == ':' + new_col = max(col - 1, 1) + elseif action == 5 # pickup + if pass_idx ≤ 4 && taxi_loc == locs[pass_idx] + new_pass_idx = 5 + else # passenger not at location + reward = -10 + end + elseif action == 6 # dropoff + if (taxi_loc == locs[dest_idx]) && pass_idx == 5 + new_pass_idx = dest_idx + done = true + reward = 20 + elseif taxi_loc ∈ locs && pass_idx == 4 + new_pass_idx = findall(x -> x == taxi_loc, locs)[1] + else + reward = -10 + end + end + new_state = encode(new_row, new_col, new_pass_idx, dest_idx) + push!(P[state][action], (1.0, new_state, reward, done)) + end + end + end + end + end + initial_state_distrib ./= sum(initial_state_distrib) + discenv_obj = DiscreteEnvObj(num_states, num_actions, P, initial_state_distrib) + action_space = Discrete(num_actions) + observation_space = Discrete(num_states) + TaxiEnv(desc, locs, discenv_obj, action_space, observation_space) +end + +function encode(taxi_row, taxi_col, pass_loc, dest_idx) + i = taxi_row - 1 + i *= 5 + i += taxi_col - 1 + i *= 5 + i += pass_loc - 1 + i *= 4 + i += dest_idx + return i +end + +function decode(i) + i -= 1 + out = [] + push!(out, i%4+1) + i = i ÷ 4 + push!(out, i%5+1) + i = i ÷ 5 + push!(out, i%5+1) + i = i ÷ 5 + @assert 1 ≤ i+1 ≤ 5 + push!(out, i+1) + reverse!(out) + return out +end + +function drawcanvas!(env::TaxiEnv) + actionarr = ["South", "North", "East", "West", "Pickup", "Dropoff"] + outfile = "" + + out = ["$char" for char in env.desc] + taxi_row, taxi_col, pass_idx, dest_idx = decode(env.s) + + ul(x) = x == " " ? "_" : x + + if pass_idx ≤ 4 + out[1 + taxi_row, 2 * (taxi_col-1) + 2] = colorize( + out[1 + taxi_row, 2 * (taxi_col-1) + 2], :yellow, highlight=true) + pi, pj = env.locs[pass_idx] + out[1 + pi, 2 * (pj-1) + 2] = colorize(ul(out[1 + pi, 2 * (pj-1) + 2]), :blue, bold=true) + else + out[1 + taxi_row, 2 * (taxi_col-1) + 2] = colorize_bold( + out[1 + taxi_row, 2 * (taxi_col-1) + 2], :green, highlight=true) + end + di, dj = env.locs[dest_idx] + out[1 + di, 2 * (dj-1) + 2] = colorize(out[1 + di, 2 * (dj-1) + 2], :magenta) + outfile *= join([join(out[row, :], "") for row in 1:size(env.desc, 1)], "\n") + outfile *= isnothing(env.lastaction) ? "\n" : " ($(actionarr[env.lastaction]))\n" + + return outfile +end diff --git a/src/Envs/toytext/vis/blackjack.jl b/src/Envs/toytext/vis/blackjack.jl new file mode 100644 index 0000000..ea7cba8 --- /dev/null +++ b/src/Envs/toytext/vis/blackjack.jl @@ -0,0 +1,13 @@ +include("utils.jl") + +function Ctx(env::BlackjackEnv, mode::Symbol=:human) + if mode == :human + HumanCtx() + elseif mode == :ansi + ANSICtx() + elseif mode == :no_render + NoCtx() + else + error("Unrecognized mode in Ctx(): $(mode)") + end +end diff --git a/src/Envs/toytext/vis/cliffwalking.jl b/src/Envs/toytext/vis/cliffwalking.jl new file mode 100644 index 0000000..10ee417 --- /dev/null +++ b/src/Envs/toytext/vis/cliffwalking.jl @@ -0,0 +1,13 @@ +include("utils.jl") + +function Ctx(env::CliffWalkingEnv, mode::Symbol=:human) + if mode == :human + HumanCtx() + elseif mode == :ansi + ANSICtx() + elseif mode == :no_render + NoCtx() + else + error("Unrecognized mode in Ctx(): $(mode)") + end +end diff --git a/src/Envs/toytext/vis/frozen_lake.jl b/src/Envs/toytext/vis/frozen_lake.jl new file mode 100644 index 0000000..678d37f --- /dev/null +++ b/src/Envs/toytext/vis/frozen_lake.jl @@ -0,0 +1,13 @@ +include("utils.jl") + +function Ctx(env::FrozenLakeEnv, mode::Symbol=:human) + if mode == :human + HumanCtx() + elseif mode == :ansi + ANSICtx() + elseif mode == :no_render + NoCtx() + else + error("Unrecognized mode in Ctx(): $(mode)") + end +end diff --git a/src/Envs/toytext/vis/guessing_game.jl b/src/Envs/toytext/vis/guessing_game.jl new file mode 100644 index 0000000..9fb541c --- /dev/null +++ b/src/Envs/toytext/vis/guessing_game.jl @@ -0,0 +1,13 @@ +include("utils.jl") + +function Ctx(env::GuessingGameEnv, mode::Symbol=:human) + if mode == :human + HumanCtx() + elseif mode == :ansi + ANSICtx() + elseif mode == :no_render + NoCtx() + else + error("Unrecognized mode in Ctx(): $(mode)") + end +end diff --git a/src/Envs/toytext/vis/hotter_colder.jl b/src/Envs/toytext/vis/hotter_colder.jl new file mode 100644 index 0000000..4684c35 --- /dev/null +++ b/src/Envs/toytext/vis/hotter_colder.jl @@ -0,0 +1,13 @@ +include("utils.jl") + +function Ctx(env::HotterColderEnv, mode::Symbol=:human) + if mode == :human + HumanCtx() + elseif mode == :ansi + ANSICtx() + elseif mode == :no_render + NoCtx() + else + error("Unrecognized mode in Ctx(): $(mode)") + end +end diff --git a/src/Envs/toytext/vis/kellycoinflip.jl b/src/Envs/toytext/vis/kellycoinflip.jl new file mode 100644 index 0000000..e72f7b2 --- /dev/null +++ b/src/Envs/toytext/vis/kellycoinflip.jl @@ -0,0 +1,13 @@ +include("utils.jl") + +function Ctx(env::KellyCoinFlipEnv, mode::Symbol=:human) + if mode == :human + HumanCtx() + elseif mode == :ansi + ANSICtx() + elseif mode == :no_render + NoCtx() + else + error("Unrecognized mode in Ctx(): $(mode)") + end +end diff --git a/src/Envs/toytext/vis/taxi.jl b/src/Envs/toytext/vis/taxi.jl new file mode 100644 index 0000000..0ae9336 --- /dev/null +++ b/src/Envs/toytext/vis/taxi.jl @@ -0,0 +1,13 @@ +include("utils.jl") + +function Ctx(env::TaxiEnv, mode::Symbol=:human) + if mode == :human + HumanCtx() + elseif mode == :ansi + ANSICtx() + elseif mode == :no_render + NoCtx() + else + error("Unrecognized mode in Ctx(): $(mode)") + end +end diff --git a/src/Envs/toytext/vis/utils.jl b/src/Envs/toytext/vis/utils.jl new file mode 100644 index 0000000..88534ec --- /dev/null +++ b/src/Envs/toytext/vis/utils.jl @@ -0,0 +1,29 @@ +struct HumanCtx <: AbstractCtx end +struct ANSICtx <: AbstractCtx end +struct NoCtx <: AbstractCtx end + +render!(env::AbstractEnv, ctx::ANSICtx) = drawcanvas!(env) +render!(env::AbstractEnv, ctx::NoCtx) = nothing +function render!(env::AbstractEnv, ctx::HumanCtx) + print(drawcanvas!(env)) +end + +function colorize(string::Union{<:AbstractString, Char}, color::Symbol; bold::Bool=false, + highlight::Bool=false) + + colormap = Dict( + :gray => 30, # highlights to black + :red => 31, + :green => 32, + :yellow => 33, + :blue => 34, + :magenta => 35, + :cyan => 36, + :white => 37, + :crimson => 38) + + num = colormap[color] + num = highlight ? num + 10 : num # For highlighting + attr = bold ? "$num;1" : "$num" + return "\x1b[" * attr * "m" * string * "\x1b[0m" +end diff --git a/src/Gym.jl b/src/Gym.jl index ecb6a5c..dcda2b4 100644 --- a/src/Gym.jl +++ b/src/Gym.jl @@ -1,20 +1,18 @@ module Gym using Flux -using Flux.Tracker +#using Flux.Tracker -#Spaces -include("Spaces/Space.jl") -using .Space -export sample -using Requires +# GymSpaces exports +using GymSpaces +export sample, seed! -@init @require Gtk="4c0ca9eb-093a-5379-98c5-f87ac0bbbf44" using Gtk +using Requires include("Envs/registry.jl") -export make, register, # Registry functions - EnvWrapper, reset!, step!, state, - trainable, game_over, render!, testmode! # Environment interaction functions +export make, register, speclist, # Registry functions + EnvWrapper, reset!, step!, seed!, state, + trainable, is_over, render!, testmode! # Environment interaction functions end #module diff --git a/src/Spaces/Space.jl b/src/Spaces/Space.jl deleted file mode 100644 index f82af46..0000000 --- a/src/Spaces/Space.jl +++ /dev/null @@ -1,19 +0,0 @@ -module Space - -using Zygote: @nograd - -export sample - -abstract type AbstractSpace end - -include("box.jl") -include("discrete.jl") -include("tuple-space.jl") -include("dict-space.jl") -include("multi-binary.jl") -include("multi-discrete.jl") - -@nograd contains -Base.in(x, space_obj::AbstractSpace) = contains(x, space_obj) -Base.size(space_obj::AbstractSpace) = space_obj.shape -end #module diff --git a/src/Spaces/box.jl b/src/Spaces/box.jl deleted file mode 100644 index ed2b110..0000000 --- a/src/Spaces/box.jl +++ /dev/null @@ -1,85 +0,0 @@ -# TODO: seed, copy - -""" -A box in R^n, i.e., each coordinate is bounded. - -Two kinds of valid input: - Box(-1.0, 1.0, (3,4)) # low and high are scalars, and shape is provided - Box([-1.0,-2.0], [2.0,4.0]) # low and high are arrays of the same shape -""" -mutable struct Box <: AbstractSpace - low::Array - high::Array - shape::Tuple -end - -function Box(low::Number, high::Number, shape::Union{Tuple, Array{Int64, 1}}, dtype::Union{DataType, Nothing}=nothing) - if isnothing(dtype) - dtype = high == 255 ? UInt8 : Float32 - @warn "dtype was autodetected as $(dtype). Please provide explicit data type." - end - - if low > high - @warn "low > high. Swapping values to preserve sanity" - (low, high) = (high, low) # Preserves sanity if low > high - end - - if dtype <: Integer - if !isa(low, Integer) || !isa(high, Integer) - @warn "dtype is an Integer, but the values are floating points. Using ceiling of lower bound and floor of upper bound" - end - low = ceil(dtype, low) - high = floor(dtype, high) - end - - Low = dtype(low) .+ zeros(dtype, shape) - High = dtype(high) .+ zeros(dtype, shape) - return Box(Low, High, shape) -end - -function Box(low::Array, high::Array, dtype::Union{DataType, Nothing}=nothing) - @assert size(low) == size(high) "Dimension mismatch between low and high arrays." - shape = size(low) - @assert all(low .< high) "elements of low must be lesser than their respective counterparts in high" - - if isnothing(dtype) - dtype = all(high .== 255) ? UInt8 : Float32 - @warn "dtype was autodetected as $(dtype). Please provide explicit data type." - end - if dtype <: Integer - if !all(isa.(low, Integer)) || !all(isa(high, Integer)) - @warn "dtype is an Integer, but the values are floating points. Using ceiling of lower bound and floor of upper bound" - end - low = ceil.(dtype, low) - high = floor.(dtype, high) - else - low = dtype.(low) - high = dtype.(high) - end - return Box(low, high, shape) -end -#= -function seed!(box_obj::Box, seed::Int) - box_obj.seed = seed -end -=# - -Base.:(==)(box_obj::Box, other::Box) = checkvalidtypes(box_obj, other) && isapprox(box_obj.low, other.low) && isapprox(box_obj.high, other.high) - -function sample(box_obj::Box) - dtype = eltype(box_obj.low) - dtype <: AbstractFloat ? - rand(dtype, size(box_obj)) .* (box_obj.high .- box_obj.low) .+ box_obj.low : - rand.(UnitRange.(box_obj.low, box_obj.high)) -end - -function contains(x::Union{Real, AbstractArray, NTuple}, box_obj::Box) - isa(x, Number) && size(box_obj.low) == (1,) && (x = [x]) - size(x) == size(box_obj) && all(box_obj.low .<= x .<= box_obj.high) -end - -function checkvalidtypes(box_obj1::Box, box_obj2::Box) - dtype1, dtype2 = eltype(box_obj1.low), eltype(box_obj2.low) - dtype1 == dtype2 || # If the dtypes of both boxes are not the same... - (dtype1 <: Unsigned && dtype2 <: Unsigned) || (dtype1 <: Signed && dtype2 <: Signed) # then check if they're both signed or both unsigned. -end diff --git a/src/Spaces/dict-space.jl b/src/Spaces/dict-space.jl deleted file mode 100644 index 6d0a930..0000000 --- a/src/Spaces/dict-space.jl +++ /dev/null @@ -1,30 +0,0 @@ -using DataStructures: OrderedDict -#TODO: seed, copy - - -mutable struct DictSpace <: AbstractSpace - spaces::OrderedDict{Union{Symbol, AbstractString}, AbstractSpace} - shape::Tuple - DictSpace(spaces::OrderedDict{<:Union{Symbol, AbstractString}, <:AbstractSpace}) = new(spaces, ()) - DictSpace(;space_kwargs...) = new(OrderedDict{Symbol, AbstractSpace}(space_kwargs), ()) -end - -DictSpace(spaces::Dict{<:Union{Symbol, AbstractString}, <:AbstractSpace}) = - DictSpace(OrderedDict(sort([(sym, space) for (sym, space) in pairs(spaces)]))) - -sample(dict_obj::DictSpace) = OrderedDict([(k, sample(space)) for (k, space) in pairs(dict_obj.spaces)]) - -function contains(x, dict_obj::DictSpace) - # If x is not a dict or OrderedDict or if x doesn't have the same length as spaces - if !(isa(x, Dict) || isa(x, OrderedDict)) || length(x) != length(dict_obj.spaces) - return false - end - - for (k, space) in pairs(dict_obj.spaces) - # If k is not in x, or if x[k] ∉ space return false - (isnothing(get(x, k, nothing)) || !(x[k] ∈ space)) && return false - end - return true -end - -Base.:(==)(dict_obj::DictSpace, other::DictSpace) = dict_obj.spaces == other.spaces diff --git a/src/Spaces/discrete.jl b/src/Spaces/discrete.jl deleted file mode 100644 index ac1bb0e..0000000 --- a/src/Spaces/discrete.jl +++ /dev/null @@ -1,27 +0,0 @@ -#TODO : seed, copy -""" -{0,1,...,n-1} - -Example usage: -discrete_obj.observation_space = Discrete(2) -""" -mutable struct Discrete <: AbstractSpace - n::Int - shape::Tuple - Discrete(N::Int) = new(N, (N, )) -end - -sample(discrete_obj::Discrete) = rand(1:discrete_obj.n) - -function contains(x::Union{Number, AbstractArray}, discrete_obj::Discrete) - as_int = nothing - try - as_int = Int.(x) - catch InexactError - return false - end - return all(1 .<= as_int .<= discrete_obj.n) -end - -Base.:(==)(discrete_obj::Discrete, other::Discrete) = discrete_obj.n == other.n -Base.length(discrete_obj::Discrete) = discrete_obj.n diff --git a/src/Spaces/multi-binary.jl b/src/Spaces/multi-binary.jl deleted file mode 100644 index 6824349..0000000 --- a/src/Spaces/multi-binary.jl +++ /dev/null @@ -1,15 +0,0 @@ - -#TODO: seed, copy - -mutable struct MultiBinary <: AbstractSpace - n::Int - shape::Tuple - - MultiBinary(n::Int) = new(n, (n, )) -end - -sample(multibin_obj::MultiBinary) = (multibin_obj.dtype)(rand(0:1, multibin_obj.n)) - -contains(x, multibin_obj::MultiBinary) = all((x .== 0) .| (x .== 1)) - -Base.:(==)(multibin_obj::MultiBinary, other::MultiBinary) = multibin_obj.n == other.n diff --git a/src/Spaces/multi-discrete.jl b/src/Spaces/multi-discrete.jl deleted file mode 100644 index 30bc38b..0000000 --- a/src/Spaces/multi-discrete.jl +++ /dev/null @@ -1,39 +0,0 @@ -#TODO: seed - -""" -- The multi-discrete action space consists of a series of discrete action spaces with different number of actions in eachs -- It is useful to represent game controllers or keyboards where each key can be represented as a discrete action space -- It is parametrized by passing an array of positive integers specifying number of actions for each discrete action space - -Note: A value of 0 always need to represent the NOOP action. - -e.g. Nintendo Game Controller -- Can be conceptualized as 3 discrete action spaces: - - 1) Arrow Keys: Discrete 5 - NOOP[0], UP[1], RIGHT[2], DOWN[3], LEFT[4] - params: min: 0, max: 4 - 2) Button A: Discrete 2 - NOOP[0], Pressed[1] - params: min: 0, max: 1 - 3) Button B: Discrete 2 - NOOP[0], Pressed[1] - params: min: 0, max: 1 - -- Can be initialized as - - MultiDiscrete([ 5, 2, 2 ]) - or MultiDiscrete((5, 2, 3)) - -""" -mutable struct MultiDiscrete <: AbstractSpace - nvec::NTuple{N, UInt32} where N - shape::Tuple -end - -function MultiDiscrete(nvec::NTuple{N, Int} where N) # nvec: vector of counts of each categorical variable - @assert all(nvec .> 0) "nvec (counts) have to be positive" - MultiDiscrete(nvec, nvec) -end - -MultiDiscrete(nvec::Array{Int, 1}) = MultiDiscrete(Tuple(nvec)) - -sample(multidisc_obj::MultiDiscrete) = [UInt32(rand(1:counts)) for counts in multidisc_obj.nvec] - -contains(x, multidisc_obj::MultiDiscrete) = all(0 .< x .<= multidisc_obj.nvec) - -Base.:(==)(multidisc_obj::MultiDiscrete, other::MultiDiscrete) = multidisc_obj.nvec == other.nvec diff --git a/src/Spaces/tuple-space.jl b/src/Spaces/tuple-space.jl deleted file mode 100644 index 5d63c94..0000000 --- a/src/Spaces/tuple-space.jl +++ /dev/null @@ -1,29 +0,0 @@ -# TODO: seed, __getitem__, copy - -""" -A tuple (i.e., product) of simpler spaces - -Example usage: -tuple_obj.observation_space = spaces.Tuple((spaces.Discrete(2), spaces.Discrete(3))) -""" -mutable struct TupleSpace <: AbstractSpace - spaces::NTuple{N, AbstractSpace} where N - shape::Int - TupleSpace(space_array::NTuple{N, AbstractSpace}) where N = new(space_array, length(space_array)) - TupleSpace(space_array::Array{<:AbstractSpace, 1}) = new(Tuple(space_array), length(space_array)) -end - -sample(tuple_obj::TupleSpace) = Tuple(sample(space) for space in tuple_obj.spaces) - -function contains(x, tuple_obj::TupleSpace) - if isa(x, Array) - x = Tuple(x) - end - return isa(x, Tuple) && Base.length(x) == Base.length(tuple_obj.spaces) && - all(part ∈ space for (space, part) in zip(tuple_obj.spaces, x)) -end - -Base.length(tuple_obj::TupleSpace) = length(tuple_obj.spaces) - -Base.:(==)(tuple_obj::TupleSpace, other::TupleSpace) = tuple_obj.spaces == other.spaces -# Base.getindex(::Box, index...) diff --git a/test/runtests.jl b/test/runtests.jl new file mode 100644 index 0000000..05952cb --- /dev/null +++ b/test/runtests.jl @@ -0,0 +1,44 @@ +using Test +using Gym + +speclist_ = collect(speclist()) + +@testset "Basic tests" begin + @testset "$envname" for envname in speclist_ + + # Create an environment using no_render mode + env1 = make(envname, :no_render) + env2 = make(envname, :no_render) + @test true + + #Get its action space + actionspace = env1.action_space + + #Set its seed + seed!(env1, 42) + seed!(env2, 42) + @test true + + #Reset it + obs11 = reset!(env1) + obs21 = reset!(env2) + @test all(obs11 .== obs21) + + #Get an outcome + seed!(actionspace, 42) + action = sample(actionspace) + obs12 = step!(env1, action) + obs22 = step!(env2, action) + @test all(obs12 .== obs22) + + #Set another seed + seed!(env1, 42) + seed!(env2, 42) + + obs13 = reset!(env1) + obs23 = reset!(env2) + + @test all(obs11 .== obs13) + @test all(obs21 .== obs23) + end +end