flink读取kafka数据并回写kafka


package Consumer;

import com.alibaba.fastjson.JSONObject;
import org.apache.flink.api.common.functions.FilterFunction;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer;

import java.util.Properties;

public class KafkaConsumer {
    public static void main(String[] args) throws Exception {
        // flink环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        Properties properties = new Properties();
        properties.setProperty("bootstrap.servers", "192.168.186.174:9092");
        properties.setProperty("group.id", "test");
        // source kafka消费者
        FlinkKafkaConsumer consumer = new FlinkKafkaConsumer<>("example", new SimpleStringSchema(), properties);
        consumer.setStartFromEarliest();//从初始值开始
        DataStream stream = env.addSource(consumer);
        stream.print();
        // 数据处理
        /*DataStream filterStream = stream.filter((FilterFunction) s -> {
            JSONObject object = JSONObject.parseObject(s);
            System.out.println(object.getString("type"));
            if("ALTER".equalsIgnoreCase(object.getString("type"))){
                return false;
            }
            return true;
        });
            SingleOutputStreamOperator mapStream = filterStream.map((MapFunction) s -> {
            JSONObject object = JSONObject.parseObject(s);
            return object.getJSONArray("data");
        });*/
        SingleOutputStreamOperator mapStream = stream.filter((FilterFunction) s -> {
            JSONObject object = JSONObject.parseObject(s);
            System.out.println(object.getString("type"));
            if("ALTER".equalsIgnoreCase(object.getString("type"))){
                return false;
            }
            return true;
        }).map((MapFunction) s -> {
            JSONObject object = JSONObject.parseObject(s);
            return object.getJSONArray("data").toJSONString();
        });
        // sink kafka生产者
        FlinkKafkaProducer producer = new FlinkKafkaProducer<>("sink",new SimpleStringSchema(), properties);
        mapStream.addSink(producer);
        mapStream.print();
        env.execute();
    }
}

        
        
            org.apache.flink
            flink-connector-kafka_2.11
            1.14.3
        
        
        
            org.apache.flink
            flink-java
            1.14.3
        
        
            org.apache.flink
            flink-streaming-java_2.11
            1.14.3
        
        
            org.apache.flink
            flink-clients_2.11
            1.14.3
        
        
        
            org.apache.flink
            flink-avro
            1.14.3
        
        
        
            com.alibaba
            fastjson
            1.2.59