diff --git a/docs/configuration.md b/docs/configuration.md index 6faa5e749bfa..13dae042ee8c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -2624,6 +2624,9 @@ Spark subsystems. ### Spark SQL +{% for static_file in site.static_files %} + {% if static_file.name == 'generated-runtime-sql-config-table.html' %} + #### Runtime SQL Configuration Runtime SQL configurations are per-session, mutable Spark SQL configurations. They can be set with initial values by the config file @@ -2631,13 +2634,13 @@ and command-line options with `--conf/-c` prefixed, or by setting `SparkConf` th Also, they can be set and queried by SET commands and rest to their initial values by RESET command, or by `SparkSession.conf`'s setter and getter methods in runtime. -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-runtime-sql-config-table.html' %} - {% include_relative generated-runtime-sql-config-table.html %} +{% include_relative generated-runtime-sql-config-table.html %} {% break %} {% endif %} {% endfor %} +{% for static_file in site.static_files %} + {% if static_file.name == 'generated-static-sql-config-table.html' %} #### Static SQL Configuration @@ -2645,9 +2648,7 @@ Static SQL configurations are cross-session, immutable Spark SQL configurations. and command-line options with `--conf/-c` prefixed, or by setting `SparkConf` that are used to create `SparkSession`. External users can query the static sql config values via `SparkSession.conf` or via set command, e.g. `SET spark.sql.extensions;`, but cannot set/unset them. -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-static-sql-config-table.html' %} - {% include_relative generated-static-sql-config-table.html %} +{% include_relative generated-static-sql-config-table.html %} {% break %} {% endif %} {% endfor %} diff --git a/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala index 2e5f59edcf1d..89bb4f362de7 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala @@ -40,17 +40,20 @@ private[sql] object PythonSQLUtils { FunctionRegistry.functionSet.flatMap(f => FunctionRegistry.builtin.lookupFunction(f)).toArray } - def listSQLConfigs(): Array[(String, String, String, String)] = { + private def listAllSQLConfigs(): Seq[(String, String, String, String)] = { val conf = new SQLConf() + // Force to build static SQL configurations + StaticSQLConf + conf.getAllDefinedConfs + } + + def listRuntimeSQLConfigs(): Array[(String, String, String, String)] = { // Py4J doesn't seem to translate Seq well, so we convert to an Array. - conf.getAllDefinedConfs.filterNot(p => SQLConf.staticConfKeys.contains(p._1)).toArray + listAllSQLConfigs().filterNot(p => SQLConf.staticConfKeys.contains(p._1)).toArray } def listStaticSQLConfigs(): Array[(String, String, String, String)] = { - val conf = new SQLConf() - // Force to build static SQL configurations - StaticSQLConf - conf.getAllDefinedConfs.filter(p => SQLConf.staticConfKeys.contains(p._1)).toArray + listAllSQLConfigs().filter(p => SQLConf.staticConfKeys.contains(p._1)).toArray } /** diff --git a/sql/core/src/test/scala/org/apache/spark/sql/api/python/PythonSQLUtilsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/api/python/PythonSQLUtilsSuite.scala index 0d18d123e328..524fc8a9b0de 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/api/python/PythonSQLUtilsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/api/python/PythonSQLUtilsSuite.scala @@ -23,7 +23,7 @@ import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf} class PythonSQLUtilsSuite extends SparkFunSuite { test("listing sql configurations contains runtime ones only") { - val configs = PythonSQLUtils.listSQLConfigs() + val configs = PythonSQLUtils.listRuntimeSQLConfigs() // static sql configurations assert(!configs.exists(entry => entry._1 == StaticSQLConf.SPARK_SESSION_EXTENSIONS.key), diff --git a/sql/create-docs.sh b/sql/create-docs.sh index 336afc4fcb9f..6614c714e90c 100755 --- a/sql/create-docs.sh +++ b/sql/create-docs.sh @@ -45,11 +45,8 @@ mkdir docs echo "Generating SQL API Markdown files." "$SPARK_HOME/bin/spark-submit" gen-sql-api-docs.py -echo "Generating runtime SQL runtime configuration table HTML file." -"$SPARK_HOME/bin/spark-submit" gen-sql-config-docs.py runtime - -echo "Generating static SQL configuration table HTML file." -"$SPARK_HOME/bin/spark-submit" gen-sql-config-docs.py static +echo "Generating SQL configuration table HTML file." +"$SPARK_HOME/bin/spark-submit" gen-sql-config-docs.py echo "Generating HTML files for SQL function table and examples." "$SPARK_HOME/bin/spark-submit" gen-sql-functions-docs.py diff --git a/sql/gen-sql-config-docs.py b/sql/gen-sql-config-docs.py index 848d2f21f114..f1980bcc0d80 100644 --- a/sql/gen-sql-config-docs.py +++ b/sql/gen-sql-config-docs.py @@ -17,7 +17,7 @@ import os import re -import sys + from collections import namedtuple from textwrap import dedent @@ -31,11 +31,11 @@ "SQLConfEntry", ["name", "default", "description", "version"]) -def get_public_sql_configs(jvm, group): +def get_sql_configs(jvm, group): if group == "static": config_set = jvm.org.apache.spark.sql.api.python.PythonSQLUtils.listStaticSQLConfigs() else: - config_set = jvm.org.apache.spark.sql.api.python.PythonSQLUtils.listSQLConfigs() + config_set = jvm.org.apache.spark.sql.api.python.PythonSQLUtils.listRuntimeSQLConfigs() sql_configs = [ SQLConfEntry( name=_sql_config._1(), @@ -81,7 +81,11 @@ def generate_sql_configs_table_html(sql_configs, path): """ )) for config in sorted(sql_configs, key=lambda x: x.name): - if config.default == "": + if config.name == "spark.sql.session.timeZone": + default = "(value of local timezone)" + elif config.name == "spark.sql.warehouse.dir": + default = "(value of $PWD/spark-warehouse)" + elif config.default == "": default = "(none)" elif config.default.startswith("") - sys.exit(-1) - else: - group = sys.argv[1] - jvm = launch_gateway().jvm - sql_configs = get_public_sql_configs(jvm, group) + docs_root_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "docs") - spark_root_dir = os.path.dirname(os.path.dirname(__file__)) - sql_configs_table_path = os.path\ - .join(spark_root_dir, "docs", "generated-" + group + "-sql-config-table.html") + sql_configs = get_sql_configs(jvm, "runtime") + sql_configs_table_path = os.path.join(docs_root_dir, "generated-runtime-sql-config-table.html") + generate_sql_configs_table_html(sql_configs, path=sql_configs_table_path) + sql_configs = get_sql_configs(jvm, "static") + sql_configs_table_path = os.path.join(docs_root_dir, "generated-static-sql-config-table.html") generate_sql_configs_table_html(sql_configs, path=sql_configs_table_path)