flume常面对的两种场景

场景一

某平台服务器不定期产生新日志文件，flume实时传输新文件到分布式存储服务器HDFS
flume配置文件：

#定义三大组件的名称

agent1.sources = s1

agent1.sinks = k1

agent1.channels = c1

# 配置source组件，spooldir用于监控文件目录

agent1.sources.s1.type = spooldir

agent1.sources.s1.spoolDir =/home/hadoop/logs

agent1.sources.s1.fileHeader = false

#配置拦截器

agent1.sources.s1.interceptors = i1

agent1.sources.s1.interceptors.i1.type = host

agent1.sources.s1.interceptors.i1.hostHeader = hostname

# 配置sink组件

agent1.sinks.k1.type = hdfs

agent1.sinks.k1.hdfs.path =hdfs://master:9000/weblog/%y-%m-%d/%H-%M

agent1.sinks.k1.hdfs.filePrefix = access_log

agent1.sinks.k1.hdfs.maxOpenFiles = 5000

agent1.sinks.k1.hdfs.batchSize= 100

agent1.sinks.k1.hdfs.fileType = DataStream

agent1.sinks.sink1.hdfs.writeFormat =Text

agent1.sinks.sink1.hdfs.useLocalTimeStamp = true

#配置channels组件，缓存到内存

agent1.channels.c1.type = memory

agent1.channels.c1.keep-alive = 120

agent1.channels.c1.capacity = 4000

agent1.channels.c1.transactionCapacity = 600

#连接三大组件

agent1.sources.s1.channels = c1

agent1.sinks.k1.channel = c1

场景二

某平台通过log4j产生日志信息，flume监控日志文件，把文件中追加的日志实时传输到HDFS。
flume配置文件：

#定义三大组件的名称

agent1.sources = s1

agent1.sinks = k1

agent1.channels = c1

# 配置source组件，exec用于监控文件内容更新

agent1.sources.s1.type = exec

agent1.sources.s1.command = tail -F/home/hadoop/weblogs/access_log

#配置拦截器

agent1.sources.s1.interceptors = i1

agent1.sources.s1.interceptors.i1.type = host

agent1.sources.s1.interceptors.i1.hostHeader = hostname

# 配置sink组件

agent1.sinks.k1.type = hdfs

#a1.sinks.k1.channel = c1

agent1.sinks.k1.hdfs.path =hdfs://master:9000/weblog/%y-%m-%d/%H-%M

agent1.sinks.k1.hdfs.filePrefix = access_log

agent1.sinks.k1.hdfs.maxOpenFiles = 5000

agent1.sinks.k1.hdfs.batchSize= 100

agent1.sinks.k1.hdfs.fileType = DataStream

agent1.sinks.k1.hdfs.writeFormat =Text

agent1.sinks.k1.hdfs.useLocalTimeStamp = true

#配置channels组件，缓存到内存

agent1.channels.c1.type = memory

agent1.channels.c1.keep-alive = 120

agent1.channels.c1.capacity = 4000

agent1.channels.c1.transactionCapacity = 600

# 连接三大组件

agent1.sources.s1.channels = c1

agent1.sinks.k1.channel = c1

flume常用组件