separate sql confs

Author: yaooqinn

docs/.gitignore

@@ -1 +1 @@
-sql-configs.html
+*sql-configs.html

docs/configuration.md

@@ -2622,16 +2622,31 @@ Please refer to the [Security](security.html) page for available options on how
 Spark subsystems.
 
 
-{% for static_file in site.static_files %}
-    {% if static_file.name == 'sql-configs.html' %}
 ### Spark SQL
 
-        {% include_relative sql-configs.html %}
+#### Runtime SQL Configuration
+
+Runtime SQL configurations inner-session, mutable Spark SQL configurations. They can be set and queried by SET commands and rest by RESET command, or by `SparkSession.conf`'s setter and getter methods.
+
+{% for static_file in site.static_files %}
+    {% if static_file.name == 'runtime-sql-configs.html' %}
+        {% include_relative runtime-sql-configs.html %}
         {% break %}
     {% endif %}
 {% endfor %}
 
 
+#### Static SQL Configuration
+
+Static SQL configurations are cross-session, immutable Spark SQL configurations. External users can query the static sql configs value via `SparkSession.conf` or via set command, e.g. `SET spark.sql.extensions;`, but cannot set/unset them.
+
+{% for static_file in site.static_files %}
+    {% if static_file.name == 'static-sql-configs.html' %}
+        {% include_relative static-sql-configs.html %}
+        {% break %}
+    {% endif %}
+{% endfor %}
+
 ### Spark Streaming
 
 <table class="table">

sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala

@@ -29,7 +29,7 @@ import org.apache.spark.sql.catalyst.expressions.ExpressionInfo
 import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
 import org.apache.spark.sql.execution.{ExplainMode, QueryExecution}
 import org.apache.spark.sql.execution.arrow.ArrowConverters
-import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
 import org.apache.spark.sql.types.DataType
 
 private[sql] object PythonSQLUtils {
@@ -42,12 +42,17 @@ private[sql] object PythonSQLUtils {
 
   def listSQLConfigs(): Array[(String, String, String, String)] = {
     val conf = new SQLConf()
-    // Force to build StaticSQLConf, which is a little bit hacky here
-    conf.warehousePath
     // Py4J doesn't seem to translate Seq well, so we convert to an Array.
     conf.getAllDefinedConfs.toArray
   }
 
+  def listStaticSQLConfigs(): Array[(String, String, String, String)] = {
+    val conf = new SQLConf()
+    // Force to build static SQL configurations
+    StaticSQLConf.WAREHOUSE_PATH.key -> ()
+    conf.getAllDefinedConfs.filter(p => SQLConf.staticConfKeys.contains(p._1)).toArray
+  }
+
   /**
    * Python callable function to read a file in Arrow stream format and create a [[RDD]]
    * using each serialized ArrowRecordBatch as a partition.

sql/core/src/test/scala/org/apache/spark/sql/api/python/PythonSQLUtilsSuite.scala

@@ -22,11 +22,11 @@ import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
 
 class PythonSQLUtilsSuite extends SparkFunSuite {
 
-  test("list sql configurations should include all public one") {
+  test("listing sql configurations contains runtime ones only") {
     val configs = PythonSQLUtils.listSQLConfigs()
 
     // static sql configurations
-    assert(configs.exists(entry => entry._1 == StaticSQLConf.SPARK_SESSION_EXTENSIONS.key),
+    assert(!configs.exists(entry => entry._1 == StaticSQLConf.SPARK_SESSION_EXTENSIONS.key),
       "listSQLConfigs should contain public static sql configuration")
     assert(!configs.exists(entry => entry._1 == StaticSQLConf.DEBUG_MODE.key),
       "listSQLConfigs should not contain internal static sql configuration")
@@ -39,6 +39,26 @@ class PythonSQLUtilsSuite extends SparkFunSuite {
 
     // spark core configurations
     assert(!configs.exists(entry => entry._1 == "spark.master"),
-      "listSQLConfigs should not contain internal dynamic sql configuration")
+      "listSQLConfigs should not contain core configuration")
+  }
+
+  test("listing static sql configurations contains public static ones only") {
+    val configs = PythonSQLUtils.listStaticSQLConfigs()
+
+    // static sql configurations
+    assert(configs.exists(entry => entry._1 == StaticSQLConf.SPARK_SESSION_EXTENSIONS.key),
+      "listStaticSQLConfigs should contain public static sql configuration")
+    assert(!configs.exists(entry => entry._1 == StaticSQLConf.DEBUG_MODE.key),
+      "listStaticSQLConfigs should not contain internal static sql configuration")
+
+    // dynamic sql configurations
+    assert(!configs.exists(entry => entry._1 == SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key),
+      "listStaticSQLConfigs should not contain dynamic sql configuration")
+    assert(!configs.exists(entry => entry._1 == SQLConf.ANALYZER_MAX_ITERATIONS.key),
+      "listStaticSQLConfigs should not contain internal dynamic sql configuration")
+
+    // spark core configurations
+    assert(!configs.exists(entry => entry._1 == "spark.master"),
+      "listStaticSQLConfigs should not contain core configuration")
   }
 }

sql/create-docs.sh

@@ -45,8 +45,11 @@ mkdir docs
 echo "Generating SQL API Markdown files."
 "$SPARK_HOME/bin/spark-submit" gen-sql-api-docs.py
 
-echo "Generating SQL configuration table HTML file."
-"$SPARK_HOME/bin/spark-submit" gen-sql-config-docs.py
+echo "Generating runtime SQL runtime configuration table HTML file."
+"$SPARK_HOME/bin/spark-submit" gen-sql-config-docs.py runtime
+
+echo "Generating static SQL configuration table HTML file."
+"$SPARK_HOME/bin/spark-submit" gen-sql-config-docs.py static
 
 echo "Generating HTML files for SQL API documentation."
 mkdocs build --clean

sql/gen-sql-config-docs.py

@@ -17,6 +17,7 @@
 
 import os
 import re
+import sys
 from collections import namedtuple
 from textwrap import dedent
 
@@ -28,15 +29,19 @@
     "SQLConfEntry", ["name", "default", "description", "version"])
 
 
-def get_public_sql_configs(jvm):
+def get_public_sql_configs(jvm, group):
+    if group == "static":
+        config_set = jvm.org.apache.spark.sql.api.python.PythonSQLUtils.listStaticSQLConfigs()
+    else:
+        config_set = jvm.org.apache.spark.sql.api.python.PythonSQLUtils.listSQLConfigs()
     sql_configs = [
         SQLConfEntry(
             name=_sql_config._1(),
             default=_sql_config._2(),
             description=_sql_config._3(),
             version=_sql_config._4()
         )
-        for _sql_config in jvm.org.apache.spark.sql.api.python.PythonSQLUtils.listSQLConfigs()
+        for _sql_config in config_set
     ]
     return sql_configs
 
@@ -112,10 +117,16 @@ def generate_sql_configs_table(sql_configs, path):
 
 
 if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print("Usage: ./bin/spark-submit sql/gen-sql-config-docs.py <static|runtime>")
+        sys.exit(-1)
+    else:
+        group = sys.argv[1]
+
     jvm = launch_gateway().jvm
-    sql_configs = get_public_sql_configs(jvm)
+    sql_configs = get_public_sql_configs(jvm, group)
 
     spark_root_dir = os.path.dirname(os.path.dirname(__file__))
-    sql_configs_table_path = os.path.join(spark_root_dir, "docs/sql-configs.html")
+    sql_configs_table_path = os.path.join(spark_root_dir, "docs", group + "-sql-configs.html")
 
     generate_sql_configs_table(sql_configs, path=sql_configs_table_path)