diff --git a/docs/deployment/settings.md b/docs/deployment/settings.md index 13c7e8ac0a6..a1d168ccc17 100644 --- a/docs/deployment/settings.md +++ b/docs/deployment/settings.md @@ -136,8 +136,13 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co Key | Default | Meaning | Type | Since --- | --- | --- | --- | --- -kyuubi.authentication|NONE|A comma separated list of client authentication types. Note that: For KERBEROS, it is SASL/GSSAPI mechanism, and for NONE, CUSTOM and LDAP, they are all SASL/PLAIN mechanism. If only NOSASL is specified, the authentication will be NOSASL. For SASL authentication, KERBEROS and PLAIN auth type are supported at the same time, and only the first specified PLAIN auth type is valid.|seq|1.0.0 +kyuubi.authentication|NONE|A comma separated list of client authentication types. Note that: For KERBEROS, it is SASL/GSSAPI mechanism, and for NONE, CUSTOM and LDAP, they are all SASL/PLAIN mechanism. If only NOSASL is specified, the authentication will be NOSASL. For SASL authentication, KERBEROS and PLAIN auth type are supported at the same time, and only the first specified PLAIN auth type is valid.|seq|1.0.0 kyuubi.authentication.custom.class|<undefined>|User-defined authentication implementation of org.apache.kyuubi.service.authentication.PasswdAuthenticationProvider|string|1.3.0 +kyuubi.authentication.jdbc.driver.class|<undefined>|Driver class name for JDBC Authentication Provider.|string|1.6.0 +kyuubi.authentication.jdbc.password|<undefined>|Database password for JDBC Authentication Provider.|string|1.6.0 +kyuubi.authentication.jdbc.query|<undefined>|Query SQL template with placeholders for JDBC Authentication Provider to execute. Authentication passes if at least one row fetched in the result set.Available placeholders are: eg.: query sql `SELECT 1 FROM auth_table WHERE user=${username} AND passwd=MD5(CONCAT(salt,${password}));` will be prepared as: `SELECT 1 FROM auth_table WHERE user=? AND passwd=MD5(CONCAT(salt,?));` with value replacement of `username` and `password` in string type.|string|1.6.0 +kyuubi.authentication.jdbc.url|<undefined>|JDBC URL for JDBC Authentication Provider.|string|1.6.0 +kyuubi.authentication.jdbc.username|<undefined>|Database username for JDBC Authentication Provider.|string|1.6.0 kyuubi.authentication.ldap.base.dn|<undefined>|LDAP base DN.|string|1.0.0 kyuubi.authentication.ldap.domain|<undefined>|LDAP domain.|string|1.0.0 kyuubi.authentication.ldap.guidKey|uid|LDAP attribute name whose values are unique in this LDAP server.For example:uid or cn.|string|1.2.0 diff --git a/kyuubi-common/pom.xml b/kyuubi-common/pom.xml index cc0cb9927d0..73cdf3b76b7 100644 --- a/kyuubi-common/pom.xml +++ b/kyuubi-common/pom.xml @@ -114,6 +114,11 @@ jackson-databind + + com.zaxxer + HikariCP + + org.apache.hadoop hadoop-minikdc @@ -137,6 +142,12 @@ failureaccess test + + + org.apache.derby + derby + test + diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala index da1d96cf363..6b7596e7ae5 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala @@ -593,7 +593,9 @@ object KyuubiConf { "
  • NONE: no authentication check.
    • " + "
  • KERBEROS: Kerberos/GSSAPI authentication.
    • " + "
  • CUSTOM: User-defined authentication.
    • " + - "
  • LDAP: Lightweight Directory Access Protocol authentication.
    • " + + "
  • JDBC: JDBC query authentication.
    • " + + "
  • LDAP: Lightweight Directory Access Protocol authentication.
    • " + + "" + " Note that: For KERBEROS, it is SASL/GSSAPI mechanism," + " and for NONE, CUSTOM and LDAP, they are all SASL/PLAIN mechanism." + " If only NOSASL is specified, the authentication will be NOSASL." + @@ -645,6 +647,51 @@ object KyuubiConf { .stringConf .createWithDefault("uid") + val AUTHENTICATION_JDBC_DRIVER: OptionalConfigEntry[String] = + buildConf("kyuubi.authentication.jdbc.driver.class") + .doc("Driver class name for JDBC Authentication Provider.") + .version("1.6.0") + .stringConf + .createOptional + + val AUTHENTICATION_JDBC_URL: OptionalConfigEntry[String] = + buildConf("kyuubi.authentication.jdbc.url") + .doc("JDBC URL for JDBC Authentication Provider.") + .version("1.6.0") + .stringConf + .createOptional + + val AUTHENTICATION_JDBC_USERNAME: OptionalConfigEntry[String] = + buildConf("kyuubi.authentication.jdbc.username") + .doc("Database username for JDBC Authentication Provider.") + .version("1.6.0") + .stringConf + .createOptional + + val AUTHENTICATION_JDBC_PASSWORD: OptionalConfigEntry[String] = + buildConf("kyuubi.authentication.jdbc.password") + .doc("Database password for JDBC Authentication Provider.") + .version("1.6.0") + .stringConf + .createOptional + + val AUTHENTICATION_JDBC_QUERY: OptionalConfigEntry[String] = + buildConf("kyuubi.authentication.jdbc.query") + .doc("Query SQL template with placeholders " + + "for JDBC Authentication Provider to execute. " + + "Authentication passes if at least one row fetched in the result set." + + "Available placeholders are: " + + "eg.: query sql `SELECT 1 FROM auth_table WHERE user=${username} AND " + + "passwd=MD5(CONCAT(salt,${password}));` " + + "will be prepared as: `SELECT 1 FROM auth_table " + + "WHERE user=? AND passwd=MD5(CONCAT(salt,?));`" + + " with value replacement of `username` and `password` in string type.") + .version("1.6.0") + .stringConf + .createOptional + val DELEGATION_KEY_UPDATE_INTERVAL: ConfigEntry[Long] = buildConf("kyuubi.delegation.key.update.interval") .doc("unused yet") diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/AuthMethods.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/AuthMethods.scala index fcbf39f699a..54c95acbbb3 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/AuthMethods.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/AuthMethods.scala @@ -19,5 +19,5 @@ package org.apache.kyuubi.service.authentication object AuthMethods extends Enumeration { type AuthMethod = Value - val NONE, LDAP, CUSTOM = Value + val NONE, LDAP, JDBC, CUSTOM = Value } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/AuthTypes.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/AuthTypes.scala index eacebad31ca..8bb1f6c4971 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/AuthTypes.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/AuthTypes.scala @@ -20,5 +20,5 @@ package org.apache.kyuubi.service.authentication object AuthTypes extends Enumeration { type AuthType = Value - val NOSASL, NONE, LDAP, KERBEROS, CUSTOM = Value + val NOSASL, NONE, LDAP, JDBC, KERBEROS, CUSTOM = Value } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/AuthenticationProviderFactory.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/AuthenticationProviderFactory.scala index 8210c83c60d..ffdd9b8bb90 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/AuthenticationProviderFactory.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/AuthenticationProviderFactory.scala @@ -44,6 +44,7 @@ object AuthenticationProviderFactory { conf: KyuubiConf): PasswdAuthenticationProvider = method match { case AuthMethods.NONE => new AnonymousAuthenticationProviderImpl case AuthMethods.LDAP => new LdapAuthenticationProviderImpl(conf) + case AuthMethods.JDBC => new JdbcAuthenticationProviderImpl(conf) case AuthMethods.CUSTOM => val className = conf.get(KyuubiConf.AUTHENTICATION_CUSTOM_CLASS) if (className.isEmpty) { diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/JdbcAuthenticationProviderImpl.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/JdbcAuthenticationProviderImpl.scala new file mode 100644 index 00000000000..c4b0abdc7ab --- /dev/null +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/JdbcAuthenticationProviderImpl.scala @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.service.authentication + +import java.sql.{Connection, PreparedStatement, Statement} +import java.util.Properties +import javax.security.sasl.AuthenticationException + +import com.zaxxer.hikari.{HikariConfig, HikariDataSource} +import org.apache.commons.lang3.StringUtils + +import org.apache.kyuubi.Logging +import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.config.KyuubiConf._ + +class JdbcAuthenticationProviderImpl(conf: KyuubiConf) extends PasswdAuthenticationProvider + with Logging { + + private val driverClass = conf.get(AUTHENTICATION_JDBC_DRIVER) + private val jdbcUrl = conf.get(AUTHENTICATION_JDBC_URL) + private val jdbcUsername = conf.get(AUTHENTICATION_JDBC_USERNAME) + private val jdbcUserPassword = conf.get(AUTHENTICATION_JDBC_PASSWORD) + private val authQuerySql = conf.get(AUTHENTICATION_JDBC_QUERY) + + private val SQL_PLACEHOLDER_REGEX = """\$\{.+?}""".r + private val USERNAME_SQL_PLACEHOLDER = "${username}" + private val PASSWORD_SQL_PLACEHOLDER = "${password}" + + checkJdbcConfigs() + + private[kyuubi] val hikariDataSource = getHikariDataSource + + /** + * The authenticate method is called by the Kyuubi Server authentication layer + * to authenticate users for their requests. + * If a user is to be granted, return nothing/throw nothing. + * When a user is to be disallowed, throw an appropriate [[AuthenticationException]]. + * + * @param user The username received over the connection request + * @param password The password received over the connection request + * @throws AuthenticationException When a user is found to be invalid by the implementation + */ + @throws[AuthenticationException] + override def authenticate(user: String, password: String): Unit = { + if (StringUtils.isBlank(user)) { + throw new AuthenticationException(s"Error validating, user is null" + + s" or contains blank space") + } + + if (StringUtils.isBlank(password)) { + throw new AuthenticationException(s"Error validating, password is null" + + s" or contains blank space") + } + + var connection: Connection = null + var queryStatement: PreparedStatement = null + + try { + connection = hikariDataSource.getConnection + + queryStatement = getAndPrepareQueryStatement(connection, user, password) + + val resultSet = queryStatement.executeQuery() + + if (resultSet == null || !resultSet.next()) { + // auth failed + throw new AuthenticationException(s"Password does not match or no such user. user:" + + s" $user , password length: ${password.length}") + } + + // auth passed + + } catch { + case e: AuthenticationException => + throw e + case e: Exception => + error("Cannot get user info", e); + throw e + } finally { + closeDbConnection(connection, queryStatement) + } + } + + private def checkJdbcConfigs(): Unit = { + def configLog(config: String, value: String): String = s"JDBCAuthConfig: $config = '$value'" + + debug(configLog("Driver Class", driverClass.orNull)) + debug(configLog("JDBC URL", jdbcUrl.orNull)) + debug(configLog("Database username", jdbcUsername.orNull)) + debug(configLog("Database password length", jdbcUserPassword.getOrElse("").length.toString)) + debug(configLog("Query SQL", authQuerySql.orNull)) + + // Check if JDBC parameters valid + if (driverClass.isEmpty) { + throw new IllegalArgumentException("JDBC driver class is not configured.") + } + + if (jdbcUrl.isEmpty) { + throw new IllegalArgumentException("JDBC url is not configured") + } + + if (jdbcUsername.isEmpty || jdbcUserPassword.isEmpty) { + throw new IllegalArgumentException("JDBC username or password is not configured") + } + + // Check Query SQL + if (authQuerySql.isEmpty) { + throw new IllegalArgumentException("Query SQL is not configured") + } + val querySqlInLowerCase = authQuerySql.get.trim.toLowerCase + if (!querySqlInLowerCase.startsWith("select")) { // allow select query sql only + throw new IllegalArgumentException("Query SQL must start with \"SELECT\""); + } + if (!querySqlInLowerCase.contains("where")) { + warn("Query SQL does not contains \"WHERE\" keyword"); + } + if (!querySqlInLowerCase.contains("${username}")) { + warn("Query SQL does not contains \"${username}\" placeholder"); + } + } + + private def getPlaceholderList(sql: String): List[String] = { + SQL_PLACEHOLDER_REGEX.findAllMatchIn(sql) + .map(m => m.matched) + .toList + } + + private def getAndPrepareQueryStatement( + connection: Connection, + user: String, + password: String): PreparedStatement = { + + val preparedSql: String = { + SQL_PLACEHOLDER_REGEX.replaceAllIn(authQuerySql.get, "?") + } + debug(s"prepared auth query sql: $preparedSql") + + val stmt = connection.prepareStatement(preparedSql) + stmt.setMaxRows(1) // minimum result size required for authentication + + // Extract placeholder list and fill parameters to placeholders + val placeholderList: List[String] = getPlaceholderList(authQuerySql.get) + for (i <- placeholderList.indices) { + val param = placeholderList(i) match { + case USERNAME_SQL_PLACEHOLDER => user + case PASSWORD_SQL_PLACEHOLDER => password + case otherPlaceholder => + throw new IllegalArgumentException( + s"Unrecognized Placeholder In Query SQL: $otherPlaceholder") + } + + stmt.setString(i + 1, param) + } + + stmt + } + + private def closeDbConnection(connection: Connection, statement: Statement): Unit = { + if (statement != null && !statement.isClosed) { + try { + statement.close() + } catch { + case e: Exception => + error("Cannot close PreparedStatement to auth database ", e) + } + } + + if (connection != null && !connection.isClosed) { + try { + connection.close() + } catch { + case e: Exception => + error("Cannot close connection to auth database ", e) + } + } + } + + private def getHikariDataSource: HikariDataSource = { + val datasourceProperties = new Properties() + val hikariConfig = new HikariConfig(datasourceProperties) + hikariConfig.setDriverClassName(driverClass.orNull) + hikariConfig.setJdbcUrl(jdbcUrl.orNull) + hikariConfig.setUsername(jdbcUsername.orNull) + hikariConfig.setPassword(jdbcUserPassword.orNull) + hikariConfig.setPoolName("jdbc-auth-pool") + + new HikariDataSource(hikariConfig) + } +} diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactory.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactory.scala index fa019ebde7c..5f429fa4ed7 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactory.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactory.scala @@ -140,6 +140,7 @@ class KyuubiAuthenticationFactory(conf: KyuubiConf, isServer: Boolean = true) ex debug(authTypes) if (none) AuthMethods.NONE else if (authTypes.contains(LDAP)) AuthMethods.LDAP + else if (authTypes.contains(JDBC)) AuthMethods.JDBC else if (authTypes.contains(CUSTOM)) AuthMethods.CUSTOM else throw new IllegalArgumentException("No valid Password Auth detected") } diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/JdbcAuthenticationProviderImplSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/JdbcAuthenticationProviderImplSuite.scala new file mode 100644 index 00000000000..617a64fbae9 --- /dev/null +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/JdbcAuthenticationProviderImplSuite.scala @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.service.authentication + +import java.sql.{Connection, DriverManager} +import java.util.Properties +import javax.security.sasl.AuthenticationException + +import org.apache.kyuubi.KyuubiFunSuite +import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.config.KyuubiConf._ + +class JdbcAuthenticationProviderImplSuite extends KyuubiFunSuite { + protected val dbUser: String = "liangbowen" + protected val dbPasswd: String = "liangbowen" + protected var jdbcUrl: String = _ + + protected val authUser: String = "liangtiancheng" + protected val authPasswd: String = "liangtiancheng" + + protected var conf = new KyuubiConf() + var conn: Connection = _ + var authDbName: String = "auth_db" + + override def beforeAll(): Unit = { + // init db + val datasourceProperties = new Properties() + datasourceProperties.put("user", dbUser) + datasourceProperties.put("password", dbPasswd) + + jdbcUrl = s"jdbc:derby:memory:$authDbName;create=true" + conn = DriverManager.getConnection( + s"$jdbcUrl;user=$dbUser;password=$dbPasswd", + datasourceProperties) + + conn.prepareStatement(s"CREATE SCHEMA $dbUser").execute + + conn.prepareStatement( + """CREATE TABLE user_auth ( + |username VARCHAR(64) NOT NULL PRIMARY KEY, + |passwd VARCHAR(64))""".stripMargin).execute(); + + val insertStmt = conn.prepareStatement("INSERT INTO user_auth " + + "(username, passwd) VALUES (?,?)") + insertStmt.setString(1, authUser) + insertStmt.setString(2, authPasswd) + insertStmt.execute(); + + conf = genJdbcAuthConfigs + + super.beforeAll() + } + + override def afterAll(): Unit = { + super.afterAll() + + // cleanup db + try { + DriverManager.getConnection(s"jdbc:derby:memory:$authDbName;shutdown=true") + } catch { + case e: Throwable => + } + } + + test("authenticate tests") { + val unchangedConf = genJdbcAuthConfigs + + val providerImpl = new JdbcAuthenticationProviderImpl(conf) + providerImpl.authenticate(authUser, authPasswd) + + val e1 = intercept[AuthenticationException] { + providerImpl.authenticate("", "") + } + assert(e1.getMessage.contains("user is null")) + + val e2 = intercept[AuthenticationException] { + providerImpl.authenticate(authUser, "") + } + assert(e2.getMessage.contains("password is null")) + + val e4 = intercept[AuthenticationException] { + providerImpl.authenticate(authUser, "wrong_password") + } + assert(e4.isInstanceOf[AuthenticationException]) + + conf = unchangedConf.clone() + conf.unset(AUTHENTICATION_JDBC_URL) + val e5 = intercept[IllegalArgumentException] { new JdbcAuthenticationProviderImpl(conf) } + assert(e5.getMessage.contains("JDBC url is not configured")) + + conf = unchangedConf.clone() + conf.unset(AUTHENTICATION_JDBC_USERNAME) + val e6 = intercept[IllegalArgumentException] { new JdbcAuthenticationProviderImpl(conf) } + assert(e6.getMessage.contains("JDBC username or password is not configured")) + + conf = unchangedConf.clone() + conf.unset(AUTHENTICATION_JDBC_PASSWORD) + val e7 = intercept[IllegalArgumentException] { new JdbcAuthenticationProviderImpl(conf) } + assert(e7.getMessage.contains("JDBC username or password is not configured")) + + conf = unchangedConf.clone() + conf.unset(AUTHENTICATION_JDBC_QUERY) + val e8 = intercept[IllegalArgumentException] { new JdbcAuthenticationProviderImpl(conf) } + assert(e8.getMessage.contains("Query SQL is not configured")) + + conf.set( + AUTHENTICATION_JDBC_QUERY, + "INSERT INTO user_auth (username, password) " + + " VALUES ('demouser','demopassword'); ") + val e9 = intercept[IllegalArgumentException] { new JdbcAuthenticationProviderImpl(conf) } + assert(e9.getMessage.contains("Query SQL must start with \"SELECT\"")) + + conf.unset(AUTHENTICATION_JDBC_URL) + val e10 = intercept[IllegalArgumentException] { new JdbcAuthenticationProviderImpl(conf) } + assert(e10.getMessage.contains("JDBC url is not configured")) + } + + private def genJdbcAuthConfigs: KyuubiConf = { + conf = new KyuubiConf() + conf.set(AUTHENTICATION_JDBC_DRIVER, "org.apache.derby.jdbc.AutoloadedDriver") + conf.set(AUTHENTICATION_JDBC_URL, jdbcUrl) + conf.set(AUTHENTICATION_JDBC_USERNAME, dbUser) + conf.set(AUTHENTICATION_JDBC_PASSWORD, dbPasswd) + conf.set( + AUTHENTICATION_JDBC_QUERY, + "SELECT 1 FROM user_auth " + + " WHERE username=${username} and passwd=${password}") + conf + } +} diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala index bfcd3011e9b..19b89b47e41 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala @@ -59,7 +59,7 @@ class KyuubiAuthenticationFactorySuite extends KyuubiFunSuite { val conf = KyuubiConf().set(KyuubiConf.AUTHENTICATION_METHOD, Seq("INVALID")) val e = intercept[IllegalArgumentException](new KyuubiAuthenticationFactory(conf)) assert(e.getMessage contains "the authentication type should be one or more of" + - " NOSASL,NONE,LDAP,KERBEROS,CUSTOM") + " NOSASL,NONE,LDAP,JDBC,KERBEROS,CUSTOM") } test("AuthType LDAP") {