tech| 华为云: kafka 实战训练营

2019-08-16  本文已影响0人  daydaygo

date: 2019-08-12 13:07:29
title: tech| 华为云: kafka 实战训练营

实战课的好处就在这, 一直有「实操」来不断的激励, 而且贴近工作中的使用场景, 学了就可以用上.

day 1: basic

# producer
bin/kafka-console-producer.sh --broker-list 192.168.0.180:9092 --topic test
# consumer
bin/kafka-console-consumer.sh --bootstrap-server 192.168.0.180:9092 --topic test --group testgroup --consumer-property enable.auto.commit=true --from-beginning

day 2: producer demo

# day 2
# producer
java -cp .:./libs/* dms.kafka.demo.KafkaProducerDemo 192.168.0.180:9092 test01
# consumer 同 day 1

package dms.kafka.demo;

import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.RecordMetadata;

import java.util.Properties;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;

public class KafkaProducerDemo {
    public static void main(String[] args) throws InterruptedException, ExecutionException {
        if (args.length != 2) {
            throw new IllegalArgumentException("usage: dms.kafka.demo.KafkaProducerDemo bootstrap-servers topic-name.");
        }
        Properties props = new Properties();
        props.put("bootstrap.servers", args[0]);
        props.put("acks", "all");
        props.put("retries", 0);
        props.put("batch.size", 16384);
        props.put("linger.ms", 1);
        props.put("buffer.memory", 33554432);
        props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        Producer<String, String> producer = new KafkaProducer<>(props);
        for (int i = 0; i < 100; i++) { 
            Future<RecordMetadata> result = producer.send(new ProducerRecord<String, String>(args[1], Integer.toString(i), Integer.toString(i)));
            RecordMetadata rm = result.get();
            System.out.println("topic: " + rm.topic() + ", partition: " + rm.partition() + ", offset: " + rm.offset());
        }
        producer.close();
    }
};

day3: consumer demo

# day3 consumer
java -cp .:./libs/* dms.kafka.demo.KafkaConsumerDemo 192.168.0.18:9092,192.168.0.121:9092,192.168.0.206:9092 topic-1642673577 test-group

package dms.kafka.demo;

import java.util.Arrays;
import java.util.Properties;

import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;

public class KafkaConsumerDemo {
    public static void main(String[] args) {
        if (args.length != 3) {
            throw new IllegalArgumentException("usage: dms.kafka.demo.KafkaProducerDemo bootstrap-servers topic-name group-name.");
        }
        Properties props = new Properties();
        props.put("bootstrap.servers", args[0]);
        props.put("group.id", args[2]);
        props.put("enable.auto.commit", "true");
        props.put("auto.offset.reset", "earliest");
        props.put("auto.commit.interval.ms", "1000");
        props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props);
        consumer.subscribe(Arrays.asList(args[1]));
        while (true) {
            ConsumerRecords<String, String> records = consumer.poll(200);
            for (ConsumerRecord<String, String> record : records)
                System.out.printf("offset = %d, key = %s, value = %s%n", record.offset(), record.key(), record.value());
        }
    }
};

day 4: kafka 架构与机制

day 5: kafka 工具

root@ecs-s3-small-1-linux-20190812201034 /d/k/bin# pwd
/data/kafka_2.12-2.3.0/bin

# topic 管理
./kafka-topics.sh --help

# producer 测试
./kafka-console-producer.sh --help

# consumer 测试
./kafka-console-consumer.sh --help

# day 5
bin/kafka-console-producer.sh --broker-list 192.168.0.180:9092 --topic day5
bin/kafka-console-consumer.sh --bootstrap-server 192.168.0.180:9092 --topic day5 --group testgroup --consumer-property enable.auto.commit=true --from-beginning
上一篇 下一篇

猜你喜欢

热点阅读