使用物化视图将kafka数据导入到clickhouse

--------------------kafka通过物化视图导入clickhouse--------------
step1: 启动zk和kafka
step2:创建一个topic
./kafka-topics.sh --zookeeper localhost:2181 --create --topic test --replication-factor 3 --partitions 2

step3:创建一个生产者用于生产数据
./kafka-console-producer.sh --broker-list localhost:9092 --topic test

step4:创建一个消费之用于查看数据
./kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic test         #不加--from-beginning 从最新的一条消息开始消费

step5:在clickhouse中创建一个物理表
create table kafka_table (
    id Int16,
    code String,
    name Nullable(String),
    insert_time DateTime DEFAULT now(),
    update_time Date  DEFAULT toDate(now())
    ) engine = MergeTree () order by id;

step6:在clickhouse中创建一个kafka引擎表
create table kafka_queue
(
 id   Int16,
 code String,
 name String
  ) engine = Kafka ()
  settings
  kafka_broker_list = ‘192.168.235.140:9092‘,
  kafka_topic_list = ‘test‘,
  kafka_group_name = ‘test_group‘,
  kafka_format = ‘JSONEachRow‘,
  kafka_skip_broken_messages = 10;

step7:在clickhouse中创建一个物化视图
 create materialized view consumer TO kafka_table 
 as
 select id,code,name from kafka_queue;

step8:在生产中生产数据
{"id":12,"code":"12","name":null}
{"id":2,"code":"2","name":"2"}
{"id":3,"code":"3","name":""}
{"id":4,"code":null,"name":""}

--结论：在生产者发送数据后，当所有字段都非null时会写入ch,当某个字段为null时，该条数据不能写入ch,即使在创建物理表时设定了Nullable,但不会导致程序异常，只是不能写入这条记录。但是当kafka引擎表在创建是给定某个字段Nullable()时，此时这个字段为null值时，该条记录会被成功写入。


ref:https://blog.csdn.net/ffb920724/article/details/113114629