# 查看topic下的每个partition的offset进度
./bin/kafka-run-class.sh kafka.tools.GetOffsetShell --topic logmonitor --time -1 --broker-list kafka01.prod.data.phd2.jianlc.jlc:9092 --partitions 0
# 如出现Leader 未 -1的情况:
# broker shutdown的时候,partition的leader在此broker上,controller选主没有成功,移除此broker后,对应的partition的leader就被赋值成-1了
# 重新启动broker即可。
# 查看当前的消费组
./bin/kafka-consumer-groups.sh --bootstrap-server 10.10.31.5:9091 --list
./bin/kafka-consumer-groups.sh --describe --bootstrap-server 10.11.31.5:9091 --group cloudera_mirrormaker
topic-generate.json
文件,写入需要处理的topic{
"topics": [
{
"topic": "radarlog"
}
],
"version": 1
}
# 使用命令
./bin/kafka-reassign-partitions.sh --zookeeper 127.0.0.1:2181 --topics-to-move-json-file topic-generate.json --broker-list "0,1,3" --generate
# 返回
Current partition replica assignment
{"version":1,"partitions":[{"topic":"radarlog","partition":2,"replicas":[3,0]},{"topic":"radarlog","partition":7,"replicas":[1,3]},{"topic":"radarlog","partition":0,"replicas":[0,1]},{"topic":"radarlog","partition":4,"replicas":[1,0]},{"topic":"radarlog","partition":1,"replicas":[1,3]},{"topic":"radarlog","partition":6,"replicas":[0,1]},{"topic":"radarlog","partition":5,"replicas":[3,1]},{"topic":"radarlog","partition":3,"replicas":[0,3]}]}
Proposed partition reassignment configuration
{"version":1,"partitions":[{"topic":"radarlog","partition":2,"replicas":[1,0]},{"topic":"radarlog","partition":7,"replicas":[0,3]},{"topic":"radarlog","partition":4,"replicas":[0,1]},{"topic":"radarlog","partition":0,"replicas":[3,1]},{"topic":"radarlog","partition":6,"replicas":[3,1]},{"topic":"radarlog","partition":1,"replicas":[0,3]},{"topic":"radarlog","partition":5,"replicas":[1,3]},{"topic":"radarlog","partition":3,"replicas":[3,0]}]}
# cat partition-replica-reassignment.json
{"version":1,"partitions":[{"topic":"radarlog","partition":2,"replicas":[1,0]},{"topic":"radarlog","partition":7,"replicas":[0,3]},{"topic":"radarlog","partition":4,"replicas":[0,1]},{"topic":"radarlog","partition":0,"replicas":[3,1]},{"topic":"radarlog","partition":6,"replicas":[3,1]},{"topic":"radarlog","partition":1,"replicas":[0,3]},{"topic":"radarlog","partition":5,"replicas":[1,3]},{"topic":"radarlog","partition":3,"replicas":[3,0]}]}
执行
./bin/kafka-reassign-partitions.sh --zookeeper 127.0.0.1:2181 --reassignment-json-file partition-replica-reassignment.json --execute
状态
./bin/kafka-reassign-partitions.sh --zookeeper 127.0.0.1:2181 --reassignment-json-file partition-replica-reassignment.json --verify