Skip to content

Commit 1eaf645

Browse files
mneethirajmneethiraj
committed
RANGER-5373: update Kafka container for Kerberos authentication - apache#6 
1 parent ae9f814 commit 1eaf645

File tree

8 files changed

+61
-21
lines changed

8 files changed

+61
-21
lines changed

‎dev-support/ranger-docker/Dockerfile.ranger-kafka‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@ COPY ./downloads/kafka_2.12-${KAFKA_VERSION}.tgz /home/ranger/dist
2929
COPY ./scripts/ranger-kafka-setup.sh /home/ranger/scripts/
3030
COPY ./scripts/ranger-kafka.sh /home/ranger/scripts/
3131
COPY ./scripts/ranger-kafka-plugin-install.properties /home/ranger/scripts/
32+
COPY ./scripts/kafka-server-jaas.conf /home/ranger/scripts/
33+
COPY ./scripts/core-site.xml /home/ranger/scripts/
3234

3335
RUN tar xvfz /home/ranger/dist/kafka_2.12-${KAFKA_VERSION}.tgz --directory=/opt/ && \
3436
ln -s /opt/kafka_2.12-${KAFKA_VERSION} /opt/kafka && \

‎dev-support/ranger-docker/scripts/create-ranger-services.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,16 @@ def service_not_exists(service):
3838
'zookeeper.connect': 'ranger-zk.rangernw:2181',
3939
'policy.download.auth.users': 'kafka',
4040
'tag.download.auth.users': 'kafka',
41+
'setup.additional.default.policies': 'true',
42+
'default-policy.1.name': 'topic: ATLAS_ENTITIES',
43+
'default-policy.1.resource.topic': 'ATLAS_ENTITIES',
44+
'default-policy.1.policyItem.1.users': 'rangertagsync',
45+
'default-policy.1.policyItem.1.accessTypes': 'create,consume',
46+
'default-policy.2.name': 'consumergroup: ranger_entities_consumer',
47+
'default-policy.2.resource.consumergroup': 'ranger_entities_consumer',
48+
'default-policy.2.policyItem.1.users': 'rangertagsync',
49+
'default-policy.2.policyItem.1.accessTypes': 'consume,describe',
50+
'ranger.plugin.audit.filters': "[{'accessResult': 'DENIED', 'isAudited': true},{'resources':{'topic':{'values':['ATLAS_ENTITIES']}},'users':['rangertagsync'],'actions':['create','consume','describe'],'isAudited':false},{'resources':{'consumergroup':{'values':['ranger_entities_consumer']}},'users':['rangertagsync'],'actions':['consume'],'isAudited':false}]",
4151
'userstore.download.auth.users': 'kafka',
4252
'ranger.plugin.kafka.policy.refresh.synchronous':'true'}})
4353

‎dev-support/ranger-docker/scripts/kafka-server-jaas.conf‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
KafkaServer {
2+
com.sun.security.auth.module.Krb5LoginModule required
3+
useKeyTab=true
4+
keyTab="/opt/kafka/keytabs/kafka.keytab"
5+
principal="kafka/[email protected]";
6+
};

‎dev-support/ranger-docker/scripts/ranger-kafka-plugin-install.properties‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
# See the License for the specific language governing permissions and
1414
# limitations under the License.
1515

16-
POLICY_MGR_URL=http://ranger:6080
16+
POLICY_MGR_URL=http://ranger.rangernw:6080
1717
REPOSITORY_NAME=dev_kafka
1818
COMPONENT_INSTALL_DIR_NAME=/opt/kafka
1919

@@ -26,7 +26,7 @@ UPDATE_XAPOLICIES_ON_GRANT_REVOKE=true
2626
XAAUDIT.SOLR.IS_ENABLED=true
2727
XAAUDIT.SOLR.MAX_QUEUE_SIZE=1
2828
XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000
29-
XAAUDIT.SOLR.SOLR_URL=http://ranger-solr:8983/solr/ranger_audits
29+
XAAUDIT.SOLR.SOLR_URL=http://ranger-solr.rangernw:8983/solr/ranger_audits
3030

3131
# Following properties are needed to get past installation script! Please don't remove
3232
XAAUDIT.HDFS.IS_ENABLED=false
@@ -43,7 +43,7 @@ XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
4343
XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
4444

4545
XAAUDIT.SOLR.ENABLE=true
46-
XAAUDIT.SOLR.URL=http://ranger-solr:8983/solr/ranger_audits
46+
XAAUDIT.SOLR.URL=http://ranger-solr.rangernw:8983/solr/ranger_audits
4747
XAAUDIT.SOLR.USER=NONE
4848
XAAUDIT.SOLR.PASSWORD=NONE
4949
XAAUDIT.SOLR.ZOOKEEPER=NONE
@@ -58,7 +58,7 @@ XAAUDIT.ELASTICSEARCH.PORT=NONE
5858
XAAUDIT.ELASTICSEARCH.PROTOCOL=NONE
5959

6060
XAAUDIT.HDFS.ENABLE=true
61-
XAAUDIT.HDFS.HDFS_DIR=hdfs://ranger-hadoop:9000/ranger/audit
61+
XAAUDIT.HDFS.HDFS_DIR=hdfs://ranger-hadoop.rangernw:9000/ranger/audit
6262
XAAUDIT.HDFS.FILE_SPOOL_DIR=/var/log/kafka/audit/hdfs/spool
6363

6464
XAAUDIT.HDFS.AZURE_ACCOUNTNAME=__REPLACE_AZURE_ACCOUNT_NAME

‎dev-support/ranger-docker/scripts/ranger-kafka-setup.sh‎

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,18 +16,48 @@
1616
# See the License for the specific language governing permissions and
1717
# limitations under the License.
1818

19+
KEYTABS_DIR=/opt/kafka/keytabs
20+
21+
if [ "${KERBEROS_ENABLED}" == "true" ]
22+
then
23+
/etc/keytabs/create_keytab.sh kafka ${KEYTABS_DIR} kafka:hadoop
24+
fi
25+
1926
cat <<EOF > /etc/ssh/ssh_config
2027
Host *
2128
StrictHostKeyChecking no
2229
UserKnownHostsFile=/dev/null
2330
EOF
2431

32+
cp ${RANGER_SCRIPTS}/core-site.xml ${KAFKA_HOME}/config/
33+
cp ${RANGER_SCRIPTS}/kafka-server-jaas.conf ${KAFKA_HOME}/config/
34+
2535
chown -R kafka:hadoop /opt/kafka/
2636

2737
cd ${RANGER_HOME}/ranger-kafka-plugin
2838
./enable-kafka-plugin.sh
2939

3040
sed -i 's/localhost:2181/ranger-zk.rangernw:2181/' ${KAFKA_HOME}/config/server.properties
3141

32-
echo >> ${KAFKA_HOME}/config/server.properties
33-
echo "authorizer.class.name=org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer" >> ${KAFKA_HOME}/config/server.properties
42+
cat <<EOF >> ${KAFKA_HOME}/config/server.properties
43+
# Enable SASL/GSSAPI mechanism
44+
sasl.enabled.mechanisms=GSSAPI
45+
sasl.mechanism.inter.broker.protocol=GSSAPI
46+
security.inter.broker.protocol=SASL_PLAINTEXT
47+
48+
# Listener configuration
49+
listeners=SASL_PLAINTEXT://:9092
50+
advertised.listeners=SASL_PLAINTEXT://ranger-kafka.rangernw:9092
51+
52+
# JAAS configuration for Kerberos
53+
listener.name.sasl_plaintext.gssapi.sasl.jaas.config=com.sun.security.auth.module.Krb5LoginModule required \
54+
useKeyTab=true \
55+
storeKey=true \
56+
keyTab="/opt/kafka/keytabs/kafka.keytab" \
57+
principal="kafka/[email protected]";
58+
59+
# Kerberos service name
60+
sasl.kerberos.service.name=kafka
61+
62+
authorizer.class.name=org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer
63+
EOF

‎dev-support/ranger-docker/scripts/ranger-kafka.sh‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,6 @@
1616
# See the License for the specific language governing permissions and
1717
# limitations under the License.
1818

19-
KEYTABS_DIR=/opt/kafka/keytabs
20-
2119
if [ "${OS_NAME}" = "UBUNTU" ]; then
2220
service ssh start
2321
fi
@@ -36,11 +34,6 @@ then
3634
# pdsh is unavailable with microdnf in rhel based image.
3735
echo "ssh" > /etc/pdsh/rcmd_default
3836

39-
if [ "${KERBEROS_ENABLED}" == "true" ]
40-
then
41-
/etc/keytabs/create_keytab.sh kafka ${KEYTABS_DIR} kafka:hadoop
42-
fi
43-
4437
if "${RANGER_SCRIPTS}"/ranger-kafka-setup.sh;
4538
then
4639
touch "${KAFKA_HOME}"/.setupDone
@@ -49,4 +42,4 @@ then
4942
fi
5043
fi
5144

52-
su -c "cd ${KAFKA_HOME} && CLASSPATH=${KAFKA_HOME}/config ./bin/kafka-server-start.sh config/server.properties" kafka
45+
su -c "cd ${KAFKA_HOME} && CLASSPATH=${KAFKA_HOME}/config KAFKA_OPTS='-Djava.security.krb5.conf=/etc/krb5.conf -Djava.security.auth.login.config=/opt/kafka/config/kafka-server-jaas.conf' ./bin/kafka-server-start.sh config/server.properties" kafka

‎dev-support/ranger-docker/scripts/ranger-tagsync-install.properties‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -25,18 +25,16 @@ TAG_DEST_RANGER_ENDPOINT = http://ranger:6080
2525
# SSL config file name for HTTPS messages to tag destination - Ranger
2626
TAG_DEST_RANGER_SSL_CONFIG_FILENAME =
2727

28-
TAG_SOURCE_ATLAS_ENABLED = false
28+
TAG_SOURCE_ATLAS_ENABLED = true
2929

3030
# Endpoint specifications needed by Atlas
31-
TAG_SOURCE_ATLAS_KAFKA_BOOTSTRAP_SERVERS = ranger-kafka.rangernw:6667
31+
TAG_SOURCE_ATLAS_KAFKA_BOOTSTRAP_SERVERS = ranger-kafka.rangernw:9092
3232
TAG_SOURCE_ATLAS_KAFKA_ZOOKEEPER_CONNECT = ranger-zk.rangernw:2181
3333
TAG_SOURCE_ATLAS_KAFKA_ENTITIES_GROUP_ID = ranger_entities_consumer
34-
3534
TAG_SOURCE_ATLAS_KAFKA_SERVICE_NAME = kafka
36-
TAG_SOURCE_ATLAS_KAFKA_SECURITY_PROTOCOL = PLAINTEXTSASL
37-
38-
TAG_SOURCE_ATLAS_KERBEROS_PRINCIPAL =
39-
TAG_SOURCE_ATLAS_KERBEROS_KEYTAB =
35+
TAG_SOURCE_ATLAS_KAFKA_SECURITY_PROTOCOL = SASL_PLAINTEXT
36+
TAG_SOURCE_ATLAS_KERBEROS_PRINCIPAL = rangertagsync/[email protected]
37+
TAG_SOURCE_ATLAS_KERBEROS_KEYTAB = /opt/ranger/tagsync/keytabs/rangertagsync.keytab
4038

4139
TAG_SOURCE_ATLASREST_ENABLED = false
4240

‎distro/src/main/assembly/tagsync.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@
9898
<include>org.apache.curator:curator-client:jar:${curator.version}</include>
9999
<include>org.apache.zookeeper:zookeeper:jar:${zookeeper.version}</include>
100100
<include>org.apache.zookeeper:zookeeper-jute:jar:${zookeeper.version}</include>
101+
<include>com.google.guava:guava</include>
101102
<include>org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version}</include>
102103
</includes>
103104
</dependencySet>

0 commit comments

Comments
 (0)