Allowed Lateness案例演示

需求

有订单数据,格式为: (订单ID，用户ID，时间戳/事件时间，订单金额)

要求每隔5s,计算5秒内，每个用户的订单总金额

并添加Watermaker来解决一定程度上的数据延迟和数据乱序问题。

并使用OutputTag+allowedLateness解决数据丢失问题

​​​​​​​API

  

   

    

     

    
    

     

      package cn.itcast.watermaker;
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

      import lombok.AllArgsConstructor;
     
    
   

    

     

    
    

     

      import lombok.Data;
     
    
   

    

     

    
    

     

      import lombok.NoArgsConstructor;
     
    
   

    

     

    
    

     

      import org.apache.flink.api.common.eventtime.WatermarkStrategy;
     
    
   

    

     

    
    

     

      import org.apache.flink.api.common.typeinfo.TypeInformation;
     
    
   

    

     

    
    

     

      import org.apache.flink.streaming.api.datastream.DataStream;
     
    
   

    

     

    
    

     

      import org.apache.flink.streaming.api.datastream.DataStreamSource;
     
    
   

    

     

    
    

     

      import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
     
    
   

    

     

    
    

     

      import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
     
    
   

    

     

    
    

     

      import org.apache.flink.streaming.api.functions.source.SourceFunction;
     
    
   

    

     

    
    

     

      import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows;
     
    
   

    

     

    
    

     

      import org.apache.flink.streaming.api.windowing.time.Time;
     
    
   

    

     

    
    

     

      import org.apache.flink.util.OutputTag;
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

      import java.time.Duration;
     
    
   

    

     

    
    

     

      import java.util.Random;
     
    
   

    

     

    
    

     

      import java.util.UUID;
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

      /**
     
    
   

    

     

    
    

     

       * Author itcast
     
    
   

    

     

    
    

     

       * Desc
     
    
   

    

     

    
    

     

       * 模拟实时订单数据,格式为: (订单ID，用户ID，订单金额，时间戳/事件时间)
     
    
   

    

     

    
    

     

       * 要求每隔5s,计算5秒内(基于时间的滚动窗口)，每个用户的订单总金额
     
    
   

    

     

    
    

     

       * 并添加Watermaker来解决一定程度上的数据延迟和数据乱序问题。
     
    
   

    

     

    
    

     

       */
     
    
   

    

     

    
    

     

      public class WatermakerDemo03_AllowedLateness {
     
    
   

    

     

    
    

     

          public static void main(String[] args) throws Exception {
     
    
   

    

     

    
    

     

              //1.env
     
    
   

    

     

    
    

     

              StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
     
    
   

    

     

    
    

     

              //2.Source
     
    
   

    

     

    
    

     

              //模拟实时订单数据(数据有延迟和乱序)
     
    
   

    

     

    
    

     

              DataStreamSource<Order> orderDS = env.addSource(new SourceFunction<Order>() {
     
    
   

    

     

    
    

     

                  private boolean flag = true;
     
    
   

    

     

    
    

     

                  @Override
     
    
   

    

     

    
    

     

                  public void run(SourceContext<Order> ctx) throws Exception {
     
    
   

    

     

    
    

     

                      Random random = new Random();
     
    
   

    

     

    
    

     

                      while (flag) {
     
    
   

    

     

    
    

     

                          String orderId = UUID.randomUUID().toString();
     
    
   

    

     

    
    

     

                          int userId = random.nextInt(3);
     
    
   

    

     

    
    

     

                          int money = random.nextInt(100);
     
    
   

    

     

    
    

     

                          //模拟数据延迟和乱序!
     
    
   

    

     

    
    

     

                          long eventTime = System.currentTimeMillis() - random.nextInt(10) * 1000;
     
    
   

    

     

    
    

     

                          ctx.collect(new Order(orderId, userId, money, eventTime));
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

                          //TimeUnit.SECONDS.sleep(1);
     
    
   

    

     

    
    

     

                      }
     
    
   

    

     

    
    

     

                  }
     
    
   

    

     

    
    

     

                  @Override
     
    
   

    

     

    
    

     

                  public void cancel() {
     
    
   

    

     

    
    

     

                      flag = false;
     
    
   

    

     

    
    

     

                  }
     
    
   

    

     

    
    

     

              });
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

              //3.Transformation
     
    
   

    

     

    
    

     

              DataStream<Order> watermakerDS = orderDS
     
    
   

    

     

    
    

     

                      .assignTimestampsAndWatermarks(
     
    
   

    

     

    
    

     

                              WatermarkStrategy.<Order>forBoundedOutOfOrderness(Duration.ofSeconds(3))
     
    
   

    

     

    
    

     

                                      .withTimestampAssigner((event, timestamp) -> event.getEventTime())
     
    
   

    

     

    
    

     

                      );
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

              //代码走到这里,就已经被添加上Watermaker了!接下来就可以进行窗口计算了
     
    
   

    

     

    
    

     

              //要求每隔5s,计算5秒内(基于时间的滚动窗口)，每个用户的订单总金额
     
    
   

    

     

    
    

     

              OutputTag<Order> outputTag = new OutputTag<>("Seriouslylate", TypeInformation.of(Order.class));
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

              SingleOutputStreamOperator<Order> result = watermakerDS
     
    
   

    

     

    
    

     

                      .keyBy(Order::getUserId)
     
    
   

    

     

    
    

     

                      //.timeWindow(Time.seconds(5), Time.seconds(5))
     
    
   

    

     

    
    

     

                      .window(TumblingEventTimeWindows.of(Time.seconds(5)))
     
    
   

    

     

    
    

     

                      .allowedLateness(Time.seconds(5))
     
    
   

    

     

    
    

     

                      .sideOutputLateData(outputTag)
     
    
   

    

     

    
    

     

                      .sum("money");
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

              DataStream<Order> result2 = result.getSideOutput(outputTag);
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

              //4.Sink
     
    
   

    

     

    
    

     

              result.print("正常的数据和迟到不严重的数据");
     
    
   

    

     

    
    

     

              result2.print("迟到严重的数据");
     
    
   

    

     

    
    

     
 
     
    
   

    

     

    
    

     

              //5.execute
     
    
   

    

     

    
    

     

              env.execute();
     
    
   

    

     

    
    

     

          }
     
    
   

    

     

    
    

     

          @Data
     
    
   

    

     

    
    

     

          @AllArgsConstructor
     
    
   

    

     

    
    

     

          @NoArgsConstructor
     
    
   

    

     

    
    

     

          public static class Order {
     
    
   

    

     

    
    

     

              private String orderId;
     
    
   

    

     

    
    

     

              private Integer userId;
     
    
   

    

     

    
    

     

              private Integer money;
     
    
   

    

     

    
    

     

              private Long eventTime;
     
    
   

    

     

    
    

     

          }
     
    
   

    

     

    
    

     

      }
     
    
  
 

文章来源: lansonli.blog.csdn.net，作者：Lansonli，版权归原作者所有，如需转载，请联系作者。

原文链接：lansonli.blog.csdn.net/article/details/116279739