RANGER-5373: apache#3

mneethiraj · mneethiraj · commit 4573f138f082 · 2025-10-19T18:20:53.000-07:00
diff --git a/dev-support/ranger-docker/Dockerfile.ranger b/dev-support/ranger-docker/Dockerfile.ranger
@@ -29,7 +29,7 @@ COPY ./dist/ranger-${RANGER_VERSION}-admin.tar.gz /home/ranger/dist/
 COPY ./scripts/ranger.sh                                         ${RANGER_SCRIPTS}/
 COPY ./scripts/ranger-admin-install-${RANGER_DB_TYPE}.properties ${RANGER_SCRIPTS}/ranger-admin-install.properties
 COPY ./scripts/create-ranger-services.py                         ${RANGER_SCRIPTS}/
-COPY ./scripts/core-site-kerberos.xml                            ${RANGER_SCRIPTS}/
+COPY ./scripts/core-site.xml                                     ${RANGER_SCRIPTS}/
 
 RUN    tar xvfz /home/ranger/dist/ranger-${RANGER_VERSION}-admin.tar.gz --directory=${RANGER_HOME} \
     && ln -s ${RANGER_HOME}/ranger-${RANGER_VERSION}-admin ${RANGER_HOME}/admin \
diff --git a/dev-support/ranger-docker/Dockerfile.ranger-hadoop b/dev-support/ranger-docker/Dockerfile.ranger-hadoop
@@ -33,6 +33,9 @@ COPY ./scripts/ranger-hadoop.sh                         /home/ranger/scripts/
 COPY ./scripts/ranger-hadoop-mkdir.sh                   /home/ranger/scripts/
 COPY ./scripts/ranger-hdfs-plugin-install.properties    /home/ranger/scripts/
 COPY ./scripts/ranger-yarn-plugin-install.properties    /home/ranger/scripts/
+COPY ./scripts/core-site.xml                            /home/ranger/scripts/
+COPY ./scripts/hdfs-site.xml                            /home/ranger/scripts/
+COPY ./scripts/yarn-site.xml                            /home/ranger/scripts/
 
 RUN tar xvfz /home/ranger/dist/hadoop-${HADOOP_VERSION}.tar.gz --directory=/opt/ && \
     ln -s /opt/hadoop-${HADOOP_VERSION} /opt/hadoop && \
diff --git a/dev-support/ranger-docker/Dockerfile.ranger-kms b/dev-support/ranger-docker/Dockerfile.ranger-kms
@@ -26,7 +26,7 @@ COPY ./dist/ranger-${KMS_VERSION}-kms.tar.gz                   /home/ranger/dist
 
 COPY ./scripts/ranger-kms.sh                                   ${RANGER_SCRIPTS}/
 COPY ./scripts/ranger-kms-install-${RANGER_DB_TYPE}.properties ${RANGER_SCRIPTS}/ranger-kms-install.properties
-COPY ./scripts/core-site-kerberos.xml                          ${RANGER_SCRIPTS}/
+COPY ./scripts/core-site.xml                                   ${RANGER_SCRIPTS}/
 
 RUN tar xvfz /home/ranger/dist/ranger-${KMS_VERSION}-kms.tar.gz --directory=${RANGER_HOME} && \
     ln -s ${RANGER_HOME}/ranger-${KMS_VERSION}-kms ${RANGER_HOME}/kms && \
diff --git a/dev-support/ranger-docker/Dockerfile.ranger-tagsync b/dev-support/ranger-docker/Dockerfile.ranger-tagsync
@@ -26,7 +26,7 @@ COPY ./dist/ranger-${TAGSYNC_VERSION}-tagsync.tar.gz /home/ranger/dist/
 COPY ./scripts/ranger-tagsync.sh                     ${RANGER_SCRIPTS}/
 COPY ./scripts/ranger-tagsync-install.properties     ${RANGER_SCRIPTS}/
 COPY ./scripts/ranger-tagsync-tags.json              ${RANGER_SCRIPTS}/
-COPY ./scripts/core-site-kerberos.xml                ${RANGER_SCRIPTS}/
+COPY ./scripts/core-site.xml                         ${RANGER_SCRIPTS}/
 
 RUN tar xvfz /home/ranger/dist/ranger-${TAGSYNC_VERSION}-tagsync.tar.gz --directory=${RANGER_HOME} && \
     ln -s ${RANGER_HOME}/ranger-${TAGSYNC_VERSION}-tagsync ${RANGER_HOME}/tagsync && \
diff --git a/dev-support/ranger-docker/Dockerfile.ranger-usersync b/dev-support/ranger-docker/Dockerfile.ranger-usersync
@@ -26,7 +26,7 @@ COPY ./dist/ranger-${USERSYNC_VERSION}-usersync.tar.gz /home/ranger/dist/
 COPY ./scripts/ranger-usersync.sh                 ${RANGER_SCRIPTS}/
 COPY ./scripts/ranger-usersync-install.properties ${RANGER_SCRIPTS}/
 COPY ./scripts/ugsync-file-source.csv             ${RANGER_SCRIPTS}/
-COPY ./scripts/core-site-kerberos.xml             ${RANGER_SCRIPTS}/
+COPY ./scripts/core-site.xml                      ${RANGER_SCRIPTS}/
 
 RUN tar xvfz /home/ranger/dist/ranger-${USERSYNC_VERSION}-usersync.tar.gz --directory=${RANGER_HOME} && \
     ln -s ${RANGER_HOME}/ranger-${USERSYNC_VERSION}-usersync ${RANGER_HOME}/usersync && \
diff --git a/dev-support/ranger-docker/scripts/core-site-kerberos.xml b/dev-support/ranger-docker/scripts/core-site-kerberos.xml
diff --git a/dev-support/ranger-docker/scripts/core-site.xml b/dev-support/ranger-docker/scripts/core-site.xml
@@ -0,0 +1,20 @@
+<?xml version="1.0"?>
+<configuration>
+  <property>
+    <name>fs.defaultFS</name>
+    <value>hdfs://ranger-hadoop:9000</value>
+  </property>
+  <property>
+    <name>hadoop.security.authentication</name>
+    <value>kerberos</value>
+  </property>
+  <property>
+    <name>hadoop.security.auth_to_local</name>
+    <value>
+      RULE:[2:$1/$2@$0]([ndj]n/.*@EXAMPLE\.COM)s/.*/hdfs/
+      RULE:[2:$1/$2@$0]([rn]m/.*@EXAMPLE\.COM)s/.*/yarn/
+      RULE:[2:$1/$2@$0](jhs/.*@EXAMPLE\.COM)s/.*/mapred/
+      DEFAULT
+    </value>
+  </property>
+</configuration>
diff --git a/dev-support/ranger-docker/scripts/create-ranger-services.py b/dev-support/ranger-docker/scripts/create-ranger-services.py
@@ -18,27 +18,42 @@ def service_not_exists(service):
                                   'fs.default.name': 'hdfs://ranger-hadoop:9000',
                                   'hadoop.security.authentication': 'simple',
                                   'hadoop.security.authorization': 'true',
+                                  'policy.download.auth.users': 'hdfs',
+                                  'tag.download.auth.users': 'hdfs',
+                                  'userstore.download.auth.users': 'hdfs',
                                   'ranger.plugin.hdfs.policy.refresh.synchronous':'true'}})
 
 hive = RangerService({'name': 'dev_hive', 'type': 'hive',
                       'configs': {'username': 'hive', 'password': 'hive',
                                   'jdbc.driverClassName': 'org.apache.hive.jdbc.HiveDriver',
                                   'jdbc.url': 'jdbc:hive2://ranger-hive:10000',
                                   'hadoop.security.authorization': 'true',
+                                  'policy.download.auth.users': 'hive',
+                                  'tag.download.auth.users': 'hive',
+                                  'userstore.download.auth.users': 'hive',
                                   'ranger.plugin.hive.policy.refresh.synchronous':'true'}})
 
 kafka = RangerService({'name': 'dev_kafka', 'type': 'kafka',
                        'configs': {'username': 'kafka', 'password': 'kafka',
                                    'zookeeper.connect': 'ranger-zk.rangernw:2181',
+                                   'policy.download.auth.users': 'kafka',
+                                   'tag.download.auth.users': 'kafka',
+                                   'userstore.download.auth.users': 'kafka',
                                    'ranger.plugin.kafka.policy.refresh.synchronous':'true'}})
 
 knox = RangerService({'name': 'dev_knox', 'type': 'knox',
                       'configs': {'username': 'knox', 'password': 'knox', 'knox.url': 'https://ranger-knox:8443',
+                      'policy.download.auth.users': 'knox',
+                      'tag.download.auth.users': 'knox',
+                      'userstore.download.auth.users': 'knox',
                       'ranger.plugin.knox.policy.refresh.synchronous':'true'}})
 
 yarn = RangerService({'name': 'dev_yarn', 'type': 'yarn',
                       'configs': {'username': 'yarn', 'password': 'yarn',
                                   'yarn.url': 'http://ranger-hadoop:8088',
+                                  'policy.download.auth.users': 'yarn',
+                                  'tag.download.auth.users': 'yarn',
+                                  'userstore.download.auth.users': 'yarn',
                                   'ranger.plugin.yarn.policy.refresh.synchronous':'true'}})
 
 hbase = RangerService({'name': 'dev_hbase', 'type': 'hbase',
@@ -49,11 +64,17 @@ def service_not_exists(service):
                                    'hbase.zookeeper.property.clientPort': '2181',
                                    'hbase.zookeeper.quorum': 'ranger-zk',
                                    'zookeeper.znode.parent': '/hbase',
+                                   'policy.download.auth.users': 'hbase',
+                                   'tag.download.auth.users': 'hbase',
+                                   'userstore.download.auth.users': 'hbase',
                                    'ranger.plugin.hbase.policy.refresh.synchronous':'true'}})
 
 kms = RangerService({'name': 'dev_kms', 'type': 'kms',
                      'configs': {'username': 'keyadmin', 'password': 'rangerR0cks!',
                                  'provider': 'http://ranger-kms:9292',
+                                 'policy.download.auth.users': 'rangerkms',
+                                 'tag.download.auth.users': 'rangerkms',
+                                 'userstore.download.auth.users': 'rangerkms',
                                  'ranger.plugin.kms.policy.refresh.synchronous':'true'}})
 
 trino = RangerService({'name': 'dev_trino',
@@ -63,6 +84,9 @@ def service_not_exists(service):
                            'password': 'trino',
                            'jdbc.driverClassName': 'io.trino.jdbc.TrinoDriver',
                            'jdbc.url': 'jdbc:trino://ranger-trino:8080',
+                           'policy.download.auth.users': 'trino',
+                           'tag.download.auth.users': 'trino',
+                           'userstore.download.auth.users': 'trino',
                            'ranger.plugin.trino.policy.refresh.synchronous':'true'
                        }})
 
@@ -72,6 +96,9 @@ def service_not_exists(service):
                        'configs': {'username': 'hdfs', 'password': 'hdfs',
                                    'ozone.om.http-address': 'http://om:9874',
                                    'hadoop.security.authentication': 'simple',
+                                   'policy.download.auth.users': 'ozone',
+                                   'tag.download.auth.users': 'ozone',
+                                   'userstore.download.auth.users': 'ozone',
                                    'ranger.plugin.ozone.policy.refresh.synchronous':'true'}})
 
 services = [hdfs, yarn, hive, hbase, kafka, knox, kms, trino, ozone]
diff --git a/dev-support/ranger-docker/scripts/hdfs-site.xml b/dev-support/ranger-docker/scripts/hdfs-site.xml
@@ -0,0 +1,59 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<configuration>
+  <property>
+    <name>dfs.replication</name>
+    <value>1</value>
+  </property>
+  <property>
+    <name>dfs.webhdfs.enabled</name>
+    <value>true</value>
+  </property>
+  <property>
+    <name>dfs.permissions.enabled</name>
+    <value>true</value>
+  </property>
+  <property>
+    <name>dfs.permissions</name>
+    <value>true</value>
+  </property>
+  <property>
+    <name>dfs.namenode.inode.attributes.provider.class</name>
+    <value>org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer</value>
+  </property>
+  <property>
+    <name>dfs.permissions.ContentSummary.subAccess</name>
+    <value>true</value>
+  </property>
+  <property>
+    <name>dfs.block.access.token.enable</name>
+    <value>true</value>
+  </property>
+  <property>
+    <name>dfs.namenode.kerberos.principal</name>
+    <value>nn/ranger-hadoop.rangernw@EXAMPLE.COM</value>
+  </property>
+  <property>
+    <name>dfs.namenode.keytab.file</name>
+    <value>/opt/hadoop/keytabs/nn.keytab</value>
+  </property>
+  <property>
+    <name>dfs.namenode.kerberos.internal.spnego.principal</name>
+    <value>HTTP/ranger-hadoop.rangernw@EXAMPLE.COM</value>
+  </property>
+  <property>
+    <name>dfs.datanode.kerberos.principal</name>
+    <value>dn/ranger-hadoop.rangernw@EXAMPLE.COM</value>
+  </property>
+  <property>
+    <name>dfs.datanode.keytab.file</name>
+    <value>/opt/hadoop/keytabs/dn.keytab</value>
+  </property>
+  <property>
+    <name>dfs.web.authentication.kerberos.principal</name>
+    <value>HTTP/ranger-hadoop.rangernw@EXAMPLE.COM</value>
+  </property>
+  <property>
+    <name>dfs.web.authentication.kerberos.keytab</name>
+    <value>/opt/hadoop/keytabs/HTTP.keytab</value>
+  </property>
+</configuration>
diff --git a/dev-support/ranger-docker/scripts/ranger-hadoop-mkdir.sh b/dev-support/ranger-docker/scripts/ranger-hadoop-mkdir.sh
@@ -17,6 +17,9 @@
 # limitations under the License.
 
 # setup directories for Ranger audits
+
+kinit -kt /opt/hadoop/keytabs/nn.keytab nn/ranger-hadoop.rangernw@EXAMPLE.COM
+
 ${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /ranger/audit/hdfs
 ${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /ranger/audit/yarn
 ${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /ranger/audit/hbaseMaster
@@ -42,3 +45,5 @@ ${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /user/hive/warehouse
 ${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /tmp/hive
 ${HADOOP_HOME}/bin/hdfs dfs -chown -R hive:hadoop /tmp/hive /user/hive
 ${HADOOP_HOME}/bin/hdfs dfs -chmod 777 /tmp/hive
+
+kdestroy
diff --git a/dev-support/ranger-docker/scripts/ranger-hadoop-setup.sh b/dev-support/ranger-docker/scripts/ranger-hadoop-setup.sh
@@ -24,40 +24,9 @@ Host *
    UserKnownHostsFile=/dev/null
 EOF
 
-cat <<EOF > ${HADOOP_HOME}/etc/hadoop/core-site.xml
-<configuration>
-  <property>
-    <name>fs.defaultFS</name>
-    <value>hdfs://ranger-hadoop:9000</value>
-  </property>
-</configuration>
-EOF
-
-cat <<EOF > ${HADOOP_HOME}/etc/hadoop/hdfs-site.xml
-<configuration>
-  <property>
-    <name>dfs.replication</name>
-    <value>1</value>
-  </property>
-  <property>
-    <name>dfs.webhdfs.enabled</name>
-    <value>true</value>
-  </property>
-</configuration>
-EOF
-
-cat <<EOF > ${HADOOP_HOME}/etc/hadoop/yarn-site.xml
-<configuration>
-  <property>
-    <name>yarn.nodemanager.aux-services</name>
-    <value>mapreduce_shuffle</value>
-  </property>
-  <property>
-    <name>yarn.nodemanager.env-whitelist</name>
-    <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME</value>
-  </property>
-</configuration>
-EOF
+cp ${RANGER_SCRIPTS}/core-site.xml ${HADOOP_HOME}/etc/hadoop/core-site.xml
+cp ${RANGER_SCRIPTS}/hdfs-site.xml ${HADOOP_HOME}/etc/hadoop/hdfs-site.xml
+cp ${RANGER_SCRIPTS}/yarn-site.xml ${HADOOP_HOME}/etc/hadoop/yarn-site.xml
 
 mkdir -p /opt/hadoop/logs
 chown -R hdfs:hadoop /opt/hadoop/
diff --git a/dev-support/ranger-docker/scripts/ranger-hadoop.sh b/dev-support/ranger-docker/scripts/ranger-hadoop.sh
@@ -39,6 +39,11 @@ then
   then
     /etc/keytabs/create_keytab.sh hdfs ${KEYTABS_DIR} hdfs:hadoop
     /etc/keytabs/create_keytab.sh yarn ${KEYTABS_DIR} yarn:hadoop
+    /etc/keytabs/create_keytab.sh nn ${KEYTABS_DIR} hdfs:hadoop
+    /etc/keytabs/create_keytab.sh dn ${KEYTABS_DIR} hdfs:hadoop
+    /etc/keytabs/create_keytab.sh nm ${KEYTABS_DIR} yarn:hadoop
+    /etc/keytabs/create_keytab.sh rm ${KEYTABS_DIR} yarn:hadoop
+    /etc/keytabs/create_keytab.sh HTTP ${KEYTABS_DIR} hdfs:hadoop
   fi
 
   if "${RANGER_SCRIPTS}"/ranger-hadoop-setup.sh;
diff --git a/dev-support/ranger-docker/scripts/ranger-kms.sh b/dev-support/ranger-docker/scripts/ranger-kms.sh
@@ -37,7 +37,7 @@ then
   then
     if [ "${KERBEROS_ENABLED}" == "true" ]
     then
-      cp ${RANGER_SCRIPTS}/core-site-kerberos.xml ${RANGER_HOME}/kms/ews/webapp/WEB-INF/classes/conf/core-site.xml
+      cp ${RANGER_SCRIPTS}/core-site.xml ${RANGER_HOME}/kms/ews/webapp/WEB-INF/classes/conf/core-site.xml
     fi
 
     touch "${RANGER_HOME}"/.setupDone
diff --git a/dev-support/ranger-docker/scripts/ranger-tagsync.sh b/dev-support/ranger-docker/scripts/ranger-tagsync.sh
@@ -37,7 +37,7 @@ then
   then
     if [ "${KERBEROS_ENABLED}" == "true" ]
     then
-      cp ${RANGER_SCRIPTS}/core-site-kerberos.xml ${RANGER_HOME}/tagsync/conf/core-site.xml
+      cp ${RANGER_SCRIPTS}/core-site.xml ${RANGER_HOME}/tagsync/conf/core-site.xml
     fi
 
     touch "${RANGER_HOME}"/.setupDone
diff --git a/dev-support/ranger-docker/scripts/ranger-usersync.sh b/dev-support/ranger-docker/scripts/ranger-usersync.sh
@@ -37,7 +37,7 @@ then
   then
     if [ "${KERBEROS_ENABLED}" == "true" ]
     then
-      cp ${RANGER_SCRIPTS}/core-site-kerberos.xml ${RANGER_HOME}/usersync/conf/core-site.xml
+      cp ${RANGER_SCRIPTS}/core-site.xml ${RANGER_HOME}/usersync/conf/core-site.xml
     fi
 
     touch "${RANGER_HOME}"/.setupDone
diff --git a/dev-support/ranger-docker/scripts/ranger.sh b/dev-support/ranger-docker/scripts/ranger.sh
@@ -39,7 +39,7 @@ then
   then
     if [ "${KERBEROS_ENABLED}" == "true" ]
     then
-      cp ${RANGER_SCRIPTS}/core-site-kerberos.xml ${RANGER_HOME}/admin/conf/core-site.xml
+      cp ${RANGER_SCRIPTS}/core-site.xml ${RANGER_HOME}/admin/conf/core-site.xml
     fi
 
     touch "${RANGER_HOME}"/.setupDone
diff --git a/dev-support/ranger-docker/scripts/yarn-site.xml b/dev-support/ranger-docker/scripts/yarn-site.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<configuration>
+  <property>
+    <name>yarn.nodemanager.aux-services</name>
+    <value>mapreduce_shuffle</value>
+  </property>
+  <property>
+    <name>yarn.nodemanager.env-whitelist</name>
+    <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME</value>
+  </property>
+  <property>
+    <name>yarn.acl.enable</name>
+    <value>true</value>
+  </property>
+  <property>
+    <name>yarn.authorization-provider</name>
+    <value>org.apache.ranger.authorization.yarn.authorizer.RangerYarnAuthorizer</value>
+  </property>
+  <property>
+    <name>yarn.resourcemanager.principal</name>
+    <value>rm/ranger-hadoop.rangernw@EXAMPLE.COM</value>
+  </property>
+  <property>
+    <name>yarn.resourcemanager.keytab</name>
+    <value>/opt/hadoop/keytabs/rm.keytab</value>
+  </property>
+  <property>
+    <name>yarn.nodemanager.principal</name>
+    <value>nm/ranger-hadoop.rangernw@EXAMPLE.COM</value>
+  </property>
+  <property>
+    <name>yarn.nodemanager.keytab</name>
+    <value>/opt/hadoop/keytabs/nm.keytab</value>
+  </property>
+</configuration>
