Skip to content

Commit 58ed4fa

Browse files
aw-was-hereaw-was-here
committed
HADOOP-13341. Deprecate HADOOP_SERVERNAME_OPTS; replace with (command)_(subcommand)_OPTS
This commit includes the following changes: HADOOP-13356. Add a function to handle command_subcommand_OPTS HADOOP-13355. Handle HADOOP_CLIENT_OPTS in a function HADOOP-13554. Add an equivalent of hadoop_subcmd_opts for secure opts HADOOP-13562. Change hadoop_subcommand_opts to use only uppercase HADOOP-13358. Modify HDFS to use hadoop_subcommand_opts HADOOP-13357. Modify common to use hadoop_subcommand_opts HADOOP-13359. Modify YARN to use hadoop_subcommand_opts HADOOP-13361. Modify hadoop_verify_user to be consistent with hadoop_subcommand_opts (ie more granularity) HADOOP-13564. modify mapred to use hadoop_subcommand_opts HADOOP-13563. hadoop_subcommand_opts should print name not actual content during debug HADOOP-13360. Documentation for HADOOP_subcommand_OPTS This closes #126
1 parent 9faccd1 commit 58ed4fa

File tree

23 files changed

+477
-166
lines changed

23 files changed

+477
-166
lines changed

‎hadoop-common-project/hadoop-common/src/main/bin/hadoop‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -161,10 +161,6 @@ function hadoopcmd_case
161161
fi
162162
;;
163163
esac
164-
165-
# Always respect HADOOP_OPTS and HADOOP_CLIENT_OPTS
166-
hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
167-
HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
168164
}
169165

170166
# This script runs the hadoop core commands.
@@ -194,6 +190,8 @@ fi
194190
HADOOP_SUBCMD=$1
195191
shift
196192

193+
hadoop_verify_user "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}"
194+
197195
HADOOP_SUBCMD_ARGS=("$@")
198196

199197
if declare -f hadoop_subcommand_"${HADOOP_SUBCMD}" >/dev/null 2>&1; then
@@ -203,15 +201,20 @@ else
203201
hadoopcmd_case "${HADOOP_SUBCMD}" "${HADOOP_SUBCMD_ARGS[@]}"
204202
fi
205203

206-
hadoop_verify_user "${HADOOP_SUBCMD}"
204+
hadoop_add_client_opts
207205

208206
if [[ ${HADOOP_WORKER_MODE} = true ]]; then
209207
hadoop_common_worker_mode_execute "${HADOOP_COMMON_HOME}/bin/hadoop" "${HADOOP_USER_PARAMS[@]}"
210208
exit $?
211209
fi
212210

211+
hadoop_subcommand_opts "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}"
212+
213213
if [[ "${HADOOP_SUBCMD_SECURESERVICE}" = true ]]; then
214214
HADOOP_SECURE_USER="${HADOOP_SUBCMD_SECUREUSER}"
215+
216+
hadoop_subcommand_secure_opts "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}"
217+
215218
hadoop_verify_secure_prereq
216219
hadoop_setup_secure_service
217220
priv_outfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out"

‎hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh‎

Lines changed: 139 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -306,6 +306,13 @@ function hadoop_bootstrap
306306
HADOOP_TOOLS_DIR=${HADOOP_TOOLS_DIR:-"share/hadoop/tools"}
307307
HADOOP_TOOLS_LIB_JARS_DIR=${HADOOP_TOOLS_LIB_JARS_DIR:-"${HADOOP_TOOLS_DIR}/lib"}
308308

309+
# by default, whatever we are about to run doesn't support
310+
# daemonization
311+
HADOOP_SUBCMD_SUPPORTDAEMONIZATION=false
312+
313+
# shellcheck disable=SC2034
314+
HADOOP_SUBCMD_SECURESERVICE=false
315+
309316
# usage output set to zero
310317
hadoop_reset_usage
311318

@@ -1230,6 +1237,20 @@ function hadoop_translate_cygwin_path
12301237
fi
12311238
}
12321239

1240+
## @description Adds the HADOOP_CLIENT_OPTS variable to
1241+
## @description HADOOP_OPTS if HADOOP_SUBCMD_SUPPORTDAEMONIZATION is false
1242+
## @audience public
1243+
## @stability stable
1244+
## @replaceable yes
1245+
function hadoop_add_client_opts
1246+
{
1247+
if [[ "${HADOOP_SUBCMD_SUPPORTDAEMONIZATION}" = false
1248+
|| -z "${HADOOP_SUBCMD_SUPPORTDAEMONIZATION}" ]]; then
1249+
hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
1250+
HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
1251+
fi
1252+
}
1253+
12331254
## @description Finish configuring Hadoop specific system properties
12341255
## @description prior to executing Java
12351256
## @audience private
@@ -1963,17 +1984,130 @@ function hadoop_secure_daemon_handler
19631984
## @return will exit on failure conditions
19641985
function hadoop_verify_user
19651986
{
1966-
local command=$1
1967-
local uservar="HADOOP_${command}_USER"
1987+
declare program=$1
1988+
declare command=$2
1989+
declare uprogram
1990+
declare ucommand
1991+
declare uvar
1992+
1993+
if [[ -z "${BASH_VERSINFO[0]}" ]] \
1994+
|| [[ "${BASH_VERSINFO[0]}" -lt 4 ]]; then
1995+
uprogram=$(echo "${program}" | tr '[:lower:]' '[:upper:]')
1996+
ucommand=$(echo "${command}" | tr '[:lower:]' '[:upper:]')
1997+
else
1998+
uprogram=${program^^}
1999+
ucommand=${command^^}
2000+
fi
2001+
2002+
uvar="${uprogram}_${ucommand}_USER"
19682003

1969-
if [[ -n ${!uservar} ]]; then
1970-
if [[ ${!uservar} != "${USER}" ]]; then
1971-
hadoop_error "ERROR: ${command} can only be executed by ${!uservar}."
2004+
if [[ -n ${!uvar} ]]; then
2005+
if [[ ${!uvar} != "${USER}" ]]; then
2006+
hadoop_error "ERROR: ${command} can only be executed by ${!uvar}."
19722007
exit 1
19732008
fi
19742009
fi
19752010
}
19762011

2012+
## @description Add custom (program)_(command)_OPTS to HADOOP_OPTS.
2013+
## @description Also handles the deprecated cases from pre-3.x.
2014+
## @audience public
2015+
## @stability stable
2016+
## @replaceable yes
2017+
## @param program
2018+
## @param subcommand
2019+
## @return will exit on failure conditions
2020+
function hadoop_subcommand_opts
2021+
{
2022+
declare program=$1
2023+
declare command=$2
2024+
declare uvar
2025+
declare depvar
2026+
declare uprogram
2027+
declare ucommand
2028+
2029+
if [[ -z "${program}" || -z "${command}" ]]; then
2030+
return 1
2031+
fi
2032+
2033+
# bash 4 and up have built-in ways to upper and lower
2034+
# case the contents of vars. This is faster than
2035+
# calling tr.
2036+
2037+
if [[ -z "${BASH_VERSINFO[0]}" ]] \
2038+
|| [[ "${BASH_VERSINFO[0]}" -lt 4 ]]; then
2039+
uprogram=$(echo "${program}" | tr '[:lower:]' '[:upper:]')
2040+
ucommand=$(echo "${command}" | tr '[:lower:]' '[:upper:]')
2041+
else
2042+
uprogram=${program^^}
2043+
ucommand=${command^^}
2044+
fi
2045+
2046+
uvar="${uprogram}_${ucommand}_OPTS"
2047+
2048+
# Let's handle all of the deprecation cases early
2049+
# HADOOP_NAMENODE_OPTS -> HDFS_NAMENODE_OPTS
2050+
2051+
depvar="HADOOP_${ucommand}_OPTS"
2052+
2053+
if [[ "${depvar}" != "${uvar}" ]]; then
2054+
if [[ -n "${!depvar}" ]]; then
2055+
hadoop_deprecate_envvar "${depvar}" "${uvar}"
2056+
fi
2057+
fi
2058+
2059+
if [[ -n ${!uvar} ]]; then
2060+
hadoop_debug "Appending ${uvar} onto HADOOP_OPTS"
2061+
HADOOP_OPTS="${HADOOP_OPTS} ${!uvar}"
2062+
return 0
2063+
fi
2064+
}
2065+
2066+
## @description Add custom (program)_(command)_SECURE_EXTRA_OPTS to HADOOP_OPTS.
2067+
## @description This *does not* handle the pre-3.x deprecated cases
2068+
## @audience public
2069+
## @stability stable
2070+
## @replaceable yes
2071+
## @param program
2072+
## @param subcommand
2073+
## @return will exit on failure conditions
2074+
function hadoop_subcommand_secure_opts
2075+
{
2076+
declare program=$1
2077+
declare command=$2
2078+
declare uvar
2079+
declare uprogram
2080+
declare ucommand
2081+
2082+
if [[ -z "${program}" || -z "${command}" ]]; then
2083+
return 1
2084+
fi
2085+
2086+
# bash 4 and up have built-in ways to upper and lower
2087+
# case the contents of vars. This is faster than
2088+
# calling tr.
2089+
2090+
if [[ -z "${BASH_VERSINFO[0]}" ]] \
2091+
|| [[ "${BASH_VERSINFO[0]}" -lt 4 ]]; then
2092+
uprogram=$(echo "${program}" | tr '[:lower:]' '[:upper:]')
2093+
ucommand=$(echo "${command}" | tr '[:lower:]' '[:upper:]')
2094+
else
2095+
uprogram=${program^^}
2096+
ucommand=${command^^}
2097+
fi
2098+
2099+
# HDFS_DATANODE_SECURE_EXTRA_OPTS
2100+
# HDFS_NFS3_SECURE_EXTRA_OPTS
2101+
# ...
2102+
uvar="${uprogram}_${ucommand}_SECURE_EXTRA_OPTS"
2103+
2104+
if [[ -n ${!uvar} ]]; then
2105+
hadoop_debug "Appending ${uvar} onto HADOOP_OPTS"
2106+
HADOOP_OPTS="${HADOOP_OPTS} ${!uvar}"
2107+
return 0
2108+
fi
2109+
}
2110+
19772111
## @description Perform the 'hadoop classpath', etc subcommand with the given
19782112
## @description parameters
19792113
## @audience private

‎hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh‎

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -294,16 +294,16 @@ esac
294294
# and therefore may override any similar flags set in HADOOP_OPTS
295295
#
296296
# a) Set JMX options
297-
# export HADOOP_NAMENODE_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=1026"
297+
# export HDFS_NAMENODE_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=1026"
298298
#
299299
# b) Set garbage collection logs
300-
# export HADOOP_NAMENODE_OPTS="${HADOOP_GC_SETTINGS} -Xloggc:${HADOOP_LOG_DIR}/gc-rm.log-$(date +'%Y%m%d%H%M')"
300+
# export HDFS_NAMENODE_OPTS="${HADOOP_GC_SETTINGS} -Xloggc:${HADOOP_LOG_DIR}/gc-rm.log-$(date +'%Y%m%d%H%M')"
301301
#
302302
# c) ... or set them directly
303-
# export HADOOP_NAMENODE_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:${HADOOP_LOG_DIR}/gc-rm.log-$(date +'%Y%m%d%H%M')"
303+
# export HDFS_NAMENODE_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:${HADOOP_LOG_DIR}/gc-rm.log-$(date +'%Y%m%d%H%M')"
304304

305305
# this is the default:
306-
# export HADOOP_NAMENODE_OPTS="-Dhadoop.security.logger=INFO,RFAS"
306+
# export HDFS_NAMENODE_OPTS="-Dhadoop.security.logger=INFO,RFAS"
307307

308308
###
309309
# SecondaryNameNode specific parameters
@@ -313,7 +313,7 @@ esac
313313
# and therefore may override any similar flags set in HADOOP_OPTS
314314
#
315315
# This is the default:
316-
# export HADOOP_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=INFO,RFAS"
316+
# export HDFS_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=INFO,RFAS"
317317

318318
###
319319
# DataNode specific parameters
@@ -323,7 +323,7 @@ esac
323323
# and therefore may override any similar flags set in HADOOP_OPTS
324324
#
325325
# This is the default:
326-
# export HADOOP_DATANODE_OPTS="-Dhadoop.security.logger=ERROR,RFAS"
326+
# export HDFS_DATANODE_OPTS="-Dhadoop.security.logger=ERROR,RFAS"
327327

328328
# On secure datanodes, user to run the datanode as after dropping privileges.
329329
# This **MUST** be uncommented to enable secure HDFS if using privileged ports
@@ -336,7 +336,7 @@ esac
336336
# Supplemental options for secure datanodes
337337
# By default, Hadoop uses jsvc which needs to know to launch a
338338
# server jvm.
339-
# export HADOOP_DN_SECURE_EXTRA_OPTS="-jvm server"
339+
# export HDFS_DATANODE_SECURE_EXTRA_OPTS="-jvm server"
340340

341341
# Where datanode log files are stored in the secure data environment.
342342
# This will replace the hadoop.log.dir Java property in secure mode.
@@ -352,18 +352,18 @@ esac
352352
# These options will be appended to the options specified as HADOOP_OPTS
353353
# and therefore may override any similar flags set in HADOOP_OPTS
354354
#
355-
# export HADOOP_NFS3_OPTS=""
355+
# export HDFS_NFS3_OPTS=""
356356

357357
# Specify the JVM options to be used when starting the Hadoop portmapper.
358358
# These options will be appended to the options specified as HADOOP_OPTS
359359
# and therefore may override any similar flags set in HADOOP_OPTS
360360
#
361-
# export HADOOP_PORTMAP_OPTS="-Xmx512m"
361+
# export HDFS_PORTMAP_OPTS="-Xmx512m"
362362

363363
# Supplemental options for priviliged gateways
364364
# By default, Hadoop uses jsvc which needs to know to launch a
365365
# server jvm.
366-
# export HADOOP_NFS3_SECURE_EXTRA_OPTS="-jvm server"
366+
# export HDFS_NFS3_SECURE_EXTRA_OPTS="-jvm server"
367367

368368
# On privileged gateways, user to run the gateway as after dropping privileges
369369
# This will replace the hadoop.id.str Java property in secure mode.
@@ -376,7 +376,7 @@ esac
376376
# These options will be appended to the options specified as HADOOP_OPTS
377377
# and therefore may override any similar flags set in HADOOP_OPTS
378378
#
379-
# export HADOOP_ZKFC_OPTS=""
379+
# export HDFS_ZKFC_OPTS=""
380380

381381
###
382382
# QuorumJournalNode specific parameters
@@ -385,7 +385,7 @@ esac
385385
# These options will be appended to the options specified as HADOOP_OPTS
386386
# and therefore may override any similar flags set in HADOOP_OPTS
387387
#
388-
# export HADOOP_JOURNALNODE_OPTS=""
388+
# export HDFS_JOURNALNODE_OPTS=""
389389

390390
###
391391
# HDFS Balancer specific parameters
@@ -394,7 +394,7 @@ esac
394394
# These options will be appended to the options specified as HADOOP_OPTS
395395
# and therefore may override any similar flags set in HADOOP_OPTS
396396
#
397-
# export HADOOP_BALANCER_OPTS=""
397+
# export HDFS_BALANCER_OPTS=""
398398

399399
###
400400
# HDFS Mover specific parameters
@@ -403,7 +403,7 @@ esac
403403
# These options will be appended to the options specified as HADOOP_OPTS
404404
# and therefore may override any similar flags set in HADOOP_OPTS
405405
#
406-
# export HADOOP_MOVER_OPTS=""
406+
# export HDFS_MOVER_OPTS=""
407407

408408
###
409409
# Advanced Users Only!
@@ -417,6 +417,7 @@ esac
417417
#
418418
# To prevent accidents, shell commands be (superficially) locked
419419
# to only allow certain users to execute certain subcommands.
420+
# It uses the format of (command)_(subcommand)_USER.
420421
#
421422
# For example, to limit who can execute the namenode command,
422-
# export HADOOP_namenode_USER=hdfs
423+
# export HDFS_NAMENODE_USER=hdfs

‎hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md‎

Lines changed: 6 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -64,17 +64,17 @@ Administrators can configure individual daemons using the configuration options
6464

6565
| Daemon | Environment Variable |
6666
|:---- |:---- |
67-
| NameNode | HADOOP\_NAMENODE\_OPTS |
68-
| DataNode | HADOOP\_DATANODE\_OPTS |
69-
| Secondary NameNode | HADOOP\_SECONDARYNAMENODE\_OPTS |
67+
| NameNode | HDFS\_NAMENODE\_OPTS |
68+
| DataNode | HDFS\_DATANODE\_OPTS |
69+
| Secondary NameNode | HDFS\_SECONDARYNAMENODE\_OPTS |
7070
| ResourceManager | YARN\_RESOURCEMANAGER\_OPTS |
7171
| NodeManager | YARN\_NODEMANAGER\_OPTS |
7272
| WebAppProxy | YARN\_PROXYSERVER\_OPTS |
73-
| Map Reduce Job History Server | HADOOP\_JOB\_HISTORYSERVER\_OPTS |
73+
| Map Reduce Job History Server | MAPRED\_HISTORYSERVER\_OPTS |
7474

75-
For example, To configure Namenode to use parallelGC, the following statement should be added in hadoop-env.sh :
75+
For example, To configure Namenode to use parallelGC and a 4GB Java Heap, the following statement should be added in hadoop-env.sh :
7676

77-
export HADOOP_NAMENODE_OPTS="-XX:+UseParallelGC"
77+
export HDFS_NAMENODE_OPTS="-XX:+UseParallelGC -Xmx4g"
7878

7979
See `etc/hadoop/hadoop-env.sh` for other examples.
8080

@@ -91,13 +91,6 @@ It is also traditional to configure `HADOOP_HOME` in the system-wide shell envir
9191
HADOOP_HOME=/path/to/hadoop
9292
export HADOOP_HOME
9393

94-
| Daemon | Environment Variable |
95-
|:---- |:---- |
96-
| ResourceManager | YARN\_RESOURCEMANAGER\_HEAPSIZE |
97-
| NodeManager | YARN\_NODEMANAGER\_HEAPSIZE |
98-
| WebAppProxy | YARN\_PROXYSERVER\_HEAPSIZE |
99-
| Map Reduce Job History Server | HADOOP\_JOB\_HISTORYSERVER\_HEAPSIZE |
100-
10194
### Configuring the Hadoop Daemons
10295

10396
This section deals with important parameters to be specified in the given configuration files:

0 commit comments

Comments
 (0)