[SPARK-31498][SQL][DOCS] Dump public static sql configurations through doc generation

### What changes were proposed in this pull request?

Currently, only the non-static public SQL configurations are dump to public doc, we'd better also add those static public ones as the command `set -v`

This PR force call StaticSQLConf to buildStaticConf.

### Why are the changes needed?

Fix missing SQL configurations in doc

### Does this PR introduce any user-facing change?

NO

### How was this patch tested?

add unit test and verify locally to see if public static SQL conf is in `docs/sql-config.html`

Closes #28274 from yaooqinn/SPARK-31498.

Authored-by: Kent Yao <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>

Author: yaooqinn

docs/configuration.md

@@ -2622,11 +2622,32 @@ Please refer to the [Security](security.html) page for available options on how
 Spark subsystems.
 
 
-{% for static_file in site.static_files %}
-    {% if static_file.name == 'generated-sql-configuration-table.html' %}
 ### Spark SQL
 
-{% include_relative generated-sql-configuration-table.html %}
+#### Runtime SQL Configuration
+
+Runtime SQL configurations are per-session, mutable Spark SQL configurations. They can be set with initial values by the config file
+and command-line options with `--conf/-c` prefixed, or by setting `SparkConf` that are used to create `SparkSession`.
+Also, they can be set and queried by SET commands and rest to their initial values by RESET command,
+or by `SparkSession.conf`'s setter and getter methods in runtime.
+
+{% for static_file in site.static_files %}
+    {% if static_file.name == 'generated-runtime-sql-config-table.html' %}
+        {% include_relative generated-runtime-sql-config-table.html %}
+        {% break %}
+    {% endif %}
+{% endfor %}
+
+
+#### Static SQL Configuration
+
+Static SQL configurations are cross-session, immutable Spark SQL configurations. They can be set with final values by the config file
+and command-line options with `--conf/-c` prefixed, or by setting `SparkConf` that are used to create `SparkSession`.
+External users can query the static sql config values via `SparkSession.conf` or via set command, e.g. `SET spark.sql.extensions;`, but cannot set/unset them.
+
+{% for static_file in site.static_files %}
+    {% if static_file.name == 'generated-static-sql-config-table.html' %}
+        {% include_relative generated-static-sql-config-table.html %}
         {% break %}
     {% endif %}
 {% endfor %}

sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala

@@ -23,13 +23,13 @@ import java.nio.channels.Channels
 import org.apache.spark.api.java.JavaRDD
 import org.apache.spark.api.python.PythonRDDServer
 import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.{DataFrame, Dataset, SQLContext}
+import org.apache.spark.sql.{DataFrame, SQLContext}
 import org.apache.spark.sql.catalyst.analysis.FunctionRegistry
 import org.apache.spark.sql.catalyst.expressions.ExpressionInfo
 import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
 import org.apache.spark.sql.execution.{ExplainMode, QueryExecution}
 import org.apache.spark.sql.execution.arrow.ArrowConverters
-import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
 import org.apache.spark.sql.types.DataType
 
 private[sql] object PythonSQLUtils {
@@ -43,7 +43,14 @@ private[sql] object PythonSQLUtils {
   def listSQLConfigs(): Array[(String, String, String, String)] = {
     val conf = new SQLConf()
     // Py4J doesn't seem to translate Seq well, so we convert to an Array.
-    conf.getAllDefinedConfs.toArray
+    conf.getAllDefinedConfs.filterNot(p => SQLConf.staticConfKeys.contains(p._1)).toArray
+  }
+
+  def listStaticSQLConfigs(): Array[(String, String, String, String)] = {
+    val conf = new SQLConf()
+    // Force to build static SQL configurations
+    StaticSQLConf
+    conf.getAllDefinedConfs.filter(p => SQLConf.staticConfKeys.contains(p._1)).toArray
   }
 
   /**

sql/core/src/test/scala/org/apache/spark/sql/api/python/PythonSQLUtilsSuite.scala

@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.api.python
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
+
+class PythonSQLUtilsSuite extends SparkFunSuite {
+
+  test("listing sql configurations contains runtime ones only") {
+    val configs = PythonSQLUtils.listSQLConfigs()
+
+    // static sql configurations
+    assert(!configs.exists(entry => entry._1 == StaticSQLConf.SPARK_SESSION_EXTENSIONS.key),
+      "listSQLConfigs should contain public static sql configuration")
+    assert(!configs.exists(entry => entry._1 == StaticSQLConf.DEBUG_MODE.key),
+      "listSQLConfigs should not contain internal static sql configuration")
+
+    // dynamic sql configurations
+    assert(configs.exists(entry => entry._1 == SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key),
+      "listSQLConfigs should contain public dynamic sql configuration")
+    assert(!configs.exists(entry => entry._1 == SQLConf.ANALYZER_MAX_ITERATIONS.key),
+      "listSQLConfigs should not contain internal dynamic sql configuration")
+
+    // spark core configurations
+    assert(!configs.exists(entry => entry._1 == "spark.master"),
+      "listSQLConfigs should not contain core configuration")
+  }
+
+  test("listing static sql configurations contains public static ones only") {
+    val configs = PythonSQLUtils.listStaticSQLConfigs()
+
+    // static sql configurations
+    assert(configs.exists(entry => entry._1 == StaticSQLConf.SPARK_SESSION_EXTENSIONS.key),
+      "listStaticSQLConfigs should contain public static sql configuration")
+    assert(!configs.exists(entry => entry._1 == StaticSQLConf.DEBUG_MODE.key),
+      "listStaticSQLConfigs should not contain internal static sql configuration")
+
+    // dynamic sql configurations
+    assert(!configs.exists(entry => entry._1 == SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key),
+      "listStaticSQLConfigs should not contain dynamic sql configuration")
+    assert(!configs.exists(entry => entry._1 == SQLConf.ANALYZER_MAX_ITERATIONS.key),
+      "listStaticSQLConfigs should not contain internal dynamic sql configuration")
+
+    // spark core configurations
+    assert(!configs.exists(entry => entry._1 == "spark.master"),
+      "listStaticSQLConfigs should not contain core configuration")
+  }
+}

sql/create-docs.sh

@@ -45,8 +45,11 @@ mkdir docs
 echo "Generating SQL API Markdown files."
 "$SPARK_HOME/bin/spark-submit" gen-sql-api-docs.py
 
-echo "Generating SQL configuration table HTML file."
-"$SPARK_HOME/bin/spark-submit" gen-sql-config-docs.py
+echo "Generating runtime SQL runtime configuration table HTML file."
+"$SPARK_HOME/bin/spark-submit" gen-sql-config-docs.py runtime
+
+echo "Generating static SQL configuration table HTML file."
+"$SPARK_HOME/bin/spark-submit" gen-sql-config-docs.py static
 
 echo "Generating HTML files for SQL function table and examples."
 "$SPARK_HOME/bin/spark-submit" gen-sql-functions-docs.py

sql/gen-sql-config-docs.py

@@ -17,6 +17,7 @@
 
 import os
 import re
+import sys
 from collections import namedtuple
 from textwrap import dedent
 
@@ -30,15 +31,19 @@
     "SQLConfEntry", ["name", "default", "description", "version"])
 
 
-def get_public_sql_configs(jvm):
+def get_public_sql_configs(jvm, group):
+    if group == "static":
+        config_set = jvm.org.apache.spark.sql.api.python.PythonSQLUtils.listStaticSQLConfigs()
+    else:
+        config_set = jvm.org.apache.spark.sql.api.python.PythonSQLUtils.listSQLConfigs()
     sql_configs = [
         SQLConfEntry(
             name=_sql_config._1(),
             default=_sql_config._2(),
             description=_sql_config._3(),
             version=_sql_config._4()
         )
-        for _sql_config in jvm.org.apache.spark.sql.api.python.PythonSQLUtils.listSQLConfigs()
+        for _sql_config in config_set
     ]
     return sql_configs
 
@@ -114,11 +119,17 @@ def generate_sql_configs_table_html(sql_configs, path):
 
 
 if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print("Usage: ./bin/spark-submit sql/gen-sql-config-docs.py <static|runtime>")
+        sys.exit(-1)
+    else:
+        group = sys.argv[1]
+
     jvm = launch_gateway().jvm
-    sql_configs = get_public_sql_configs(jvm)
+    sql_configs = get_public_sql_configs(jvm, group)
 
     spark_root_dir = os.path.dirname(os.path.dirname(__file__))
-    sql_configs_table_path = os.path.join(
-        spark_root_dir, "docs/generated-sql-configuration-table.html")
+    sql_configs_table_path = os.path\
+        .join(spark_root_dir, "docs", "generated-" + group + "-sql-config-table.html")
 
     generate_sql_configs_table_html(sql_configs, path=sql_configs_table_path)