当前位置: 代码迷 >> 综合 >> flume Consolidation
  详细解决方案

flume Consolidation

热度:45   发布时间:2024-02-11 01:04:48.0

Consolidation

flume1配置

# Name the components on this agent
a1.sources = r1
a1.sinks = k1 
a1.channels = c1 c2# Describe/configure the source
a1.sources.r1.type = netcat
a1.sources.r1.bind = hadoop001
a1.sources.r1.port = 44441# Use a channel which buffers events in memory
a1.channels.c1.type = memorya1.sinks.k1.type = avro
a1.sinks.k1.hostname = hadoop001
a1.sinks.k1.port = 44444# Bind the source and sink to the channel
a1.sources.r1.channels = c1 
a1.sinks.k1.channel = c1
a1.sinks.k2.channel = c2

flume2配置

# Name the components on this agent
a2.sources = r1
a2.sinks = k1 
a2.channels = c1 c2# Describe/configure the source
a2.sources.r1.type = netcat
a2.sources.r1.bind = hadoop001
a2.sources.r1.port = 44442# Use a channel which buffers events in memory
a2.channels.c1.type = memorya2.sinks.k1.type = avro
a2.sinks.k1.hostname = hadoop001
a2.sinks.k1.port = 44444# Bind the source and sink to the channel
a2.sources.r1.channels = c1 
a2.sinks.k1.channel = c1
a2.sinks.k2.channel = c2

flume3配置

# Name the components on this agent
a3.sources = r1
a3.sinks = k1 
a3.channels = c1 c2# Describe/configure the source
a3.sources.r1.type = netcat
a3.sources.r1.bind = hadoop001
a3.sources.r1.port = 44443# Use a channel which buffers events in memory
a3.channels.c1.type = memorya3.sinks.k1.type = avro
a3.sinks.k1.hostname = hadoop001
a3.sinks.k1.port = 44444# Bind the source and sink to the channel
a3.sources.r1.channels = c1 
a3.sinks.k1.channel = c1
a3.sinks.k2.channel = c2

flume4配置

# Name the components on this agent
a4.sources = r1
a4.sinks = k1 
a4.channels = c1 a4.sources.r1.type = avro
a4.sources.r1.bind = hadoop001
a4.sources.r1.port = 44444a4.channels.c1.type = memorya4.sinks.k1.type = logger# Bind the source and sink to the channel
a4.sources.r1.channels = c1
a4.sinks.k1.channel = c1

启动项

flume-ng agent \
--name a1 \
--conf  $FLUME_HOME/conf \
--conf-file /home/hadoop/script/flume/Consolidation/flume1.conf \
-Dflume.root.logger=INFO,consoleflume-ng agent \
--name a2 \
--conf  $FLUME_HOME/conf \
--conf-file /home/hadoop/script/flume/Consolidation/flume2.conf \
-Dflume.root.logger=INFO,consoleflume-ng agent \
--name a3 \
--conf  $FLUME_HOME/conf \
--conf-file /home/hadoop/script/flume/Consolidation/flume3.conf \
-Dflume.root.logger=INFO,consoleflume-ng agent \
--name a4 \
--conf  $FLUME_HOME/conf \
--conf-file /home/hadoop/script/flume/Consolidation/flume4.conf \
-Dflume.root.logger=INFO,console
  相关解决方案