Wednesday, November 9, 2016

Flume Elastic Search config

flume1.sources = source1
flume1.channels = channel1
flume1.sinks = k1

flume1.channels.channel1.type = memory
flume1.channels.channel1.capacity = 10000000
flume1.channels.channel1.transactionCapacity = 1000

# For each source, channel, and sink, set standard properties
flume1.sources.source1.type = org.apache.flume.source.kafka.KafkaSource
flume1.sources.source1.zookeeperConnect = 135.250.193.206:2181
flume1.sources.source1.topic = airtel_ims_p1360com_cpqhocpuutiltable_kpi
flume1.sources.source1.batchSize = 5
flume1.sources.source1.batchDurationMillis = 200
flume1.sources.source1.channels = channel1


flume1.sinks.k1.channel = channel1
flume1.sinks.k1.type = elasticsearch
flume1.sinks.k1.batchSize = 100
flume1.sinks.k1.hostNames = 135.250.193.206:9300
flume1.sinks.k1.indexName = p1360com_cpqhocpuutiltable_index
flume1.sinks.k1.indexType = p1360com_cpqhocpuutiltable_type
flume1.sinks.k1.clusterName = spmdevteam
flume1.sinks.k1.serializer = org.apache.flume.sink.elasticsearch.ElasticSearchDynamicSerializer