kafka生产者数据压缩前后吞吐量对比

1.服务器信息:
Cpu: 16 Intel(R) Xeon(R) CPU E5-26xx v3
内存: 32G
硬盘: 500G

2.测试结果:
1000万条数据压力测试,默认平均吞吐量是55MB/S,压缩后平均吞吐量是64MB/s,整体提升16.4%。

3.默认配置

vim config/producer.properties
compression.codec=null


[root@hjl01kafka]# bin/kafka-producer-perf-test.sh --messages 10000000 --topic test --threads 15 --broker-list
10.47.97.119:9092,10.27.225.114:9092,10.26.21.73:9092 --batch-size 1000
start.time, end.time, compression, message.size, batch.size, total.data.sent.in.MB, MB.sec, total.data.sent.in.nMsg, nMsg.sec
2017-05-02 18:03:29:968, 2017-05-02 18:03:49:511, 0, 100, 1000, 953.67, 48.7987, 9999990, 511691.6543
[root@hjl02 kafka]# bin/kafka-producer-perf-test.sh --messages 10000000 --topic test --threads 15 --broker-list
10.47.97.119:9092,10.27.225.114:9092,10.26.21.73:9092 --batch-size 1000
start.time, end.time, compression, message.size, batch.size, total.data.sent.in.MB, MB.sec, total.data.sent.in.nMsg, nMsg.sec
2017-05-02 18:05:04:153, 2017-05-02 18:05:20:025, 0, 100, 1000, 953.67, 60.0853, 9999990, 630039.6925
[root@hjl03 kafka]# bin/kafka-producer-perf-test.sh --messages 10000000 --topic test --threads 15 --broker-list
10.47.97.119:9092,10.27.225.114:9092,10.26.21.73:9092 --batch-size 1000
start.time, end.time, compression, message.size, batch.size, total.data.sent.in.MB, MB.sec, total.data.sent.in.nMsg, nMsg.sec
2017-05-02 18:05:10:790, 2017-05-02 18:05:27:683, 0, 100, 1000, 953.67, 56.4538, 9999990, 591960.5754

4.压缩配置

vim config/producer.properties
compression.codec=gzip
compressed.topics=test


[root@hjl01kafka]# bin/kafka-producer-perf-test.sh --messages 10000000 --topic test --threads 15 --broker-list 10.47.97.119:9092,10.27.225.114:9092,10.26.21.73:9092 --batch-size 1000
start.time, end.time, compression, message.size, batch.size, total.data.sent.in.MB, MB.sec, total.data.sent.in.nMsg, nMsg.sec
2017-07-28 10:33:16:769, 2017-07-28 10:33:31:749, 0, 100, 1000, 953.67, 63.6631, 9999990, 667556.0748
[root@hjl02 kafka]# bin/kafka-producer-perf-test.sh --messages 10000000 --topic test --threads 15 --broker-list 10.47.97.119:9092,10.27.225.114:9092,10.26.21.73:9092 --batch-size 1000
start.time, end.time, compression, message.size, batch.size, total.data.sent.in.MB, MB.sec, total.data.sent.in.nMsg, nMsg.sec
2017-07-28 10:37:07:753, 2017-07-28 10:37:20:850, 0, 100, 1000, 953.67, 72.8162, 9999990, 763532.8701
[root@hjl03 kafka]# bin/kafka-producer-perf-test.sh --messages 10000000 --topic test --threads 15 --broker-list 10.47.97.119:9092,10.27.225.114:9092,10.26.21.73:9092 --batch-size 1000
start.time, end.time, compression, message.size, batch.size, total.data.sent.in.MB, MB.sec, total.data.sent.in.nMsg, nMsg.sec
2017-07-28 10:38:19:129, 2017-07-28 10:38:36:671, 0, 100, 1000, 953.67, 54.3651, 9999990, 570059.8563