Stream Processing

1Date,Country,City,Specie,count,min,max,median,variance
228/06/2021,GR,Athens,no2,116,0.5,58.9,5.5,2824.39
328/06/2021,GR,Athens,wind-speed,133,0.1,11.2,3,96.69
428/06/2021,GR,Athens,dew,24,14,20,18,35.92
528/06/2021,GR,Athens,temperature,141,24.4,38.4,30.5,123.18
628/06/2021,GR,Athens,pm25,116,34,85,68,702.29
7

1Date,Country,City,Specie,count,min,max,median,variance
228/06/2021,GR,Athens,no2,116,0.5,58.9,5.5,2824.39
328/06/2021,GR,Athens,wind-speed,133,0.1,11.2,3,96.69
428/06/2021,GR,Athens,dew,24,14,20,18,35.92
528/06/2021,GR,Athens,temperature,141,24.4,38.4,30.5,123.18
628/06/2021,GR,Athens,pm25,116,34,85,68,702.29
7public class CityMetric {
8
9    private static final DateTimeFormatter dateFormatter = DateTimeFormatter.ofPattern("dd/MM/yyyy");
10
11    private final LocalDate localDate;
12    private final String country;
13    private final String city;
14    private final String reading;
15    private final int count;
16    private final double min;
17    private final double max;
18    private final double median;
19    private final double variance;
20
21    private CityMetric(LocalDate localDate, String country, String city, String reading, int count, double min, double max, double median, double variance) {
22        this.localDate = localDate;
23        this.country = country;
24        this.city = city;
25        this.reading = reading;
26        this.count = count;
27        this.min = min;
28        this.max = max;
29        this.median = median;
30        this.variance = variance;
31    }
32
33    public static CityMetric fromArray(String[] arr) {
34        LocalDate date = LocalDate.parse(arr[0], dateFormatter);
35        int count = Integer.parseInt(arr[4]);
36        double min = Double.parseDouble(arr[5]);
37        double max = Double.parseDouble(arr[6]);
38        double median = Double.parseDouble(arr[7]);
39        double variance = Double.parseDouble(arr[8]);
40
41        return new CityMetric(date, arr[1], arr[2], arr[3], count, min, max, median, variance);
42    }
43
44    public long getTimestamp() {
45        return getLocalDate()
46                .atStartOfDay()
47                .toInstant(ZoneOffset.UTC)
48                .toEpochMilli();
49    }
50
51//getters follow
52

1Date,Country,City,Specie,count,min,max,median,variance
228/06/2021,GR,Athens,no2,116,0.5,58.9,5.5,2824.39
328/06/2021,GR,Athens,wind-speed,133,0.1,11.2,3,96.69
428/06/2021,GR,Athens,dew,24,14,20,18,35.92
528/06/2021,GR,Athens,temperature,141,24.4,38.4,30.5,123.18
628/06/2021,GR,Athens,pm25,116,34,85,68,702.29
7public class CityMetric {
8
9    private static final DateTimeFormatter dateFormatter = DateTimeFormatter.ofPattern("dd/MM/yyyy");
10
11    private final LocalDate localDate;
12    private final String country;
13    private final String city;
14    private final String reading;
15    private final int count;
16    private final double min;
17    private final double max;
18    private final double median;
19    private final double variance;
20
21    private CityMetric(LocalDate localDate, String country, String city, String reading, int count, double min, double max, double median, double variance) {
22        this.localDate = localDate;
23        this.country = country;
24        this.city = city;
25        this.reading = reading;
26        this.count = count;
27        this.min = min;
28        this.max = max;
29        this.median = median;
30        this.variance = variance;
31    }
32
33    public static CityMetric fromArray(String[] arr) {
34        LocalDate date = LocalDate.parse(arr[0], dateFormatter);
35        int count = Integer.parseInt(arr[4]);
36        double min = Double.parseDouble(arr[5]);
37        double max = Double.parseDouble(arr[6]);
38        double median = Double.parseDouble(arr[7]);
39        double variance = Double.parseDouble(arr[8]);
40
41        return new CityMetric(date, arr[1], arr[2], arr[3], count, min, max, median, variance);
42    }
43
44    public long getTimestamp() {
45        return getLocalDate()
46                .atStartOfDay()
47                .toInstant(ZoneOffset.UTC)
48                .toEpochMilli();
49    }
50
51//getters follow
52   final WatermarkStrategy<CityMetric> cityMetricWatermarkStrategy =
53            WatermarkStrategy.<CityMetric>forMonotonousTimestamps()  //we know they are sorted by time
54                    .withTimestampAssigner((cityMetric, l) -> cityMetric.getTimestamp());
55

1Date,Country,City,Specie,count,min,max,median,variance
228/06/2021,GR,Athens,no2,116,0.5,58.9,5.5,2824.39
328/06/2021,GR,Athens,wind-speed,133,0.1,11.2,3,96.69
428/06/2021,GR,Athens,dew,24,14,20,18,35.92
528/06/2021,GR,Athens,temperature,141,24.4,38.4,30.5,123.18
628/06/2021,GR,Athens,pm25,116,34,85,68,702.29
7public class CityMetric {
8
9    private static final DateTimeFormatter dateFormatter = DateTimeFormatter.ofPattern("dd/MM/yyyy");
10
11    private final LocalDate localDate;
12    private final String country;
13    private final String city;
14    private final String reading;
15    private final int count;
16    private final double min;
17    private final double max;
18    private final double median;
19    private final double variance;
20
21    private CityMetric(LocalDate localDate, String country, String city, String reading, int count, double min, double max, double median, double variance) {
22        this.localDate = localDate;
23        this.country = country;
24        this.city = city;
25        this.reading = reading;
26        this.count = count;
27        this.min = min;
28        this.max = max;
29        this.median = median;
30        this.variance = variance;
31    }
32
33    public static CityMetric fromArray(String[] arr) {
34        LocalDate date = LocalDate.parse(arr[0], dateFormatter);
35        int count = Integer.parseInt(arr[4]);
36        double min = Double.parseDouble(arr[5]);
37        double max = Double.parseDouble(arr[6]);
38        double median = Double.parseDouble(arr[7]);
39        double variance = Double.parseDouble(arr[8]);
40
41        return new CityMetric(date, arr[1], arr[2], arr[3], count, min, max, median, variance);
42    }
43
44    public long getTimestamp() {
45        return getLocalDate()
46                .atStartOfDay()
47                .toInstant(ZoneOffset.UTC)
48                .toEpochMilli();
49    }
50
51//getters follow
52   final WatermarkStrategy<CityMetric> cityMetricWatermarkStrategy =
53            WatermarkStrategy.<CityMetric>forMonotonousTimestamps()  //we know they are sorted by time
54                    .withTimestampAssigner((cityMetric, l) -> cityMetric.getTimestamp());
55  final StreamingFileSink<Tuple4<LocalDate, LocalDate, String, Double>> fileSink =
56        StreamingFileSink.forRowFormat(
57                new Path("airquality"),
58                new SimpleStringEncoder<Tuple4<LocalDate, LocalDate, String, Double>>("UTF-8"))
59            .build();
60

1Date,Country,City,Specie,count,min,max,median,variance
228/06/2021,GR,Athens,no2,116,0.5,58.9,5.5,2824.39
328/06/2021,GR,Athens,wind-speed,133,0.1,11.2,3,96.69
428/06/2021,GR,Athens,dew,24,14,20,18,35.92
528/06/2021,GR,Athens,temperature,141,24.4,38.4,30.5,123.18
628/06/2021,GR,Athens,pm25,116,34,85,68,702.29
7public class CityMetric {
8
9    private static final DateTimeFormatter dateFormatter = DateTimeFormatter.ofPattern("dd/MM/yyyy");
10
11    private final LocalDate localDate;
12    private final String country;
13    private final String city;
14    private final String reading;
15    private final int count;
16    private final double min;
17    private final double max;
18    private final double median;
19    private final double variance;
20
21    private CityMetric(LocalDate localDate, String country, String city, String reading, int count, double min, double max, double median, double variance) {
22        this.localDate = localDate;
23        this.country = country;
24        this.city = city;
25        this.reading = reading;
26        this.count = count;
27        this.min = min;
28        this.max = max;
29        this.median = median;
30        this.variance = variance;
31    }
32
33    public static CityMetric fromArray(String[] arr) {
34        LocalDate date = LocalDate.parse(arr[0], dateFormatter);
35        int count = Integer.parseInt(arr[4]);
36        double min = Double.parseDouble(arr[5]);
37        double max = Double.parseDouble(arr[6]);
38        double median = Double.parseDouble(arr[7]);
39        double variance = Double.parseDouble(arr[8]);
40
41        return new CityMetric(date, arr[1], arr[2], arr[3], count, min, max, median, variance);
42    }
43
44    public long getTimestamp() {
45        return getLocalDate()
46                .atStartOfDay()
47                .toInstant(ZoneOffset.UTC)
48                .toEpochMilli();
49    }
50
51//getters follow
52   final WatermarkStrategy<CityMetric> cityMetricWatermarkStrategy =
53            WatermarkStrategy.<CityMetric>forMonotonousTimestamps()  //we know they are sorted by time
54                    .withTimestampAssigner((cityMetric, l) -> cityMetric.getTimestamp());
55  final StreamingFileSink<Tuple4<LocalDate, LocalDate, String, Double>> fileSink =
56        StreamingFileSink.forRowFormat(
57                new Path("airquality"),
58                new SimpleStringEncoder<Tuple4<LocalDate, LocalDate, String, Double>>("UTF-8"))
59            .build();
60 source
61        .map(s -> s.split(",")) //split the CSV row into its fields
62        .filter(arr -> !arr[0].startsWith("Date")) // if it starts with Date it means it is the top header
63        .map(CityMetric::fromArray)  //create the object from the fields
64        .assignTimestampsAndWatermarks(cityMetricWatermarkStrategy) // we use the date as the event time
65        .filter(cm -> cm.getReading().equals("pm25")) // we want air quality of fine particulate matter pm2.5
66        .keyBy(CityMetric::getCity) // partition by city name
67        .window(TumblingEventTimeWindows.of(Time.days(7))) //windows of 7 days
68        .aggregate(new CityAverageAggregate()) // average the values
69        .name("cityair")
70        .addSink(fileSink); //output each partition to a file
71

1Date,Country,City,Specie,count,min,max,median,variance
228/06/2021,GR,Athens,no2,116,0.5,58.9,5.5,2824.39
328/06/2021,GR,Athens,wind-speed,133,0.1,11.2,3,96.69
428/06/2021,GR,Athens,dew,24,14,20,18,35.92
528/06/2021,GR,Athens,temperature,141,24.4,38.4,30.5,123.18
628/06/2021,GR,Athens,pm25,116,34,85,68,702.29
7public class CityMetric {
8
9    private static final DateTimeFormatter dateFormatter = DateTimeFormatter.ofPattern("dd/MM/yyyy");
10
11    private final LocalDate localDate;
12    private final String country;
13    private final String city;
14    private final String reading;
15    private final int count;
16    private final double min;
17    private final double max;
18    private final double median;
19    private final double variance;
20
21    private CityMetric(LocalDate localDate, String country, String city, String reading, int count, double min, double max, double median, double variance) {
22        this.localDate = localDate;
23        this.country = country;
24        this.city = city;
25        this.reading = reading;
26        this.count = count;
27        this.min = min;
28        this.max = max;
29        this.median = median;
30        this.variance = variance;
31    }
32
33    public static CityMetric fromArray(String[] arr) {
34        LocalDate date = LocalDate.parse(arr[0], dateFormatter);
35        int count = Integer.parseInt(arr[4]);
36        double min = Double.parseDouble(arr[5]);
37        double max = Double.parseDouble(arr[6]);
38        double median = Double.parseDouble(arr[7]);
39        double variance = Double.parseDouble(arr[8]);
40
41        return new CityMetric(date, arr[1], arr[2], arr[3], count, min, max, median, variance);
42    }
43
44    public long getTimestamp() {
45        return getLocalDate()
46                .atStartOfDay()
47                .toInstant(ZoneOffset.UTC)
48                .toEpochMilli();
49    }
50
51//getters follow
52   final WatermarkStrategy<CityMetric> cityMetricWatermarkStrategy =
53            WatermarkStrategy.<CityMetric>forMonotonousTimestamps()  //we know they are sorted by time
54                    .withTimestampAssigner((cityMetric, l) -> cityMetric.getTimestamp());
55  final StreamingFileSink<Tuple4<LocalDate, LocalDate, String, Double>> fileSink =
56        StreamingFileSink.forRowFormat(
57                new Path("airquality"),
58                new SimpleStringEncoder<Tuple4<LocalDate, LocalDate, String, Double>>("UTF-8"))
59            .build();
60 source
61        .map(s -> s.split(",")) //split the CSV row into its fields
62        .filter(arr -> !arr[0].startsWith("Date")) // if it starts with Date it means it is the top header
63        .map(CityMetric::fromArray)  //create the object from the fields
64        .assignTimestampsAndWatermarks(cityMetricWatermarkStrategy) // we use the date as the event time
65        .filter(cm -> cm.getReading().equals("pm25")) // we want air quality of fine particulate matter pm2.5
66        .keyBy(CityMetric::getCity) // partition by city name
67        .window(TumblingEventTimeWindows.of(Time.days(7))) //windows of 7 days
68        .aggregate(new CityAverageAggregate()) // average the values
69        .name("cityair")
70        .addSink(fileSink); //output each partition to a file
71public class CityAverageAggregate
72    implements AggregateFunction<
73        CityMetric, CityAverageAggregate.AverageAccumulator, Tuple4<LocalDate, LocalDate, String, Double>> {
74
75  @Override
76  public AverageAccumulator createAccumulator() {
77    return new AverageAccumulator();
78  }
79
80  @Override
81  public AverageAccumulator add(CityMetric cityMetric, AverageAccumulator averageAccumulator) {
82    return averageAccumulator.add(
83        cityMetric.getCity(), cityMetric.getLocalDate(), cityMetric.getMedian());
84  }
85
86  @Override
87  public Tuple4<LocalDate, LocalDate, String, Double> getResult(
88      AverageAccumulator averageAccumulator) {
89    return Tuple4.of(
90        averageAccumulator.getStart(),
91        averageAccumulator.getEnd(),
92        averageAccumulator.getCity(),
93        averageAccumulator.average());
94  }
95
96  @Override
97  public AverageAccumulator merge(AverageAccumulator acc1, AverageAccumulator acc2) {
98    return acc1.merge(acc2);
99  }
100
101  public static class AverageAccumulator {
102    private final String city;
103    private final LocalDate start;
104    private final LocalDate end;
105    private final long count;
106    private final double sum;
107
108    public AverageAccumulator() {
109      city = "";
110      count = 0;
111      sum = 0;
112      start = null;
113      end = null;
114    }
115
116    AverageAccumulator(String city, LocalDate start, LocalDate end, long count, double sum) {
117      this.city = city;
118      this.count = count;
119      this.sum = sum;
120      this.start = start;
121      this.end = end;
122    }
123
124    public AverageAccumulator add(String city, LocalDate eventDate, double value) {
125      //make sure our dataflow is correct and we are summing data from the same city
126      if (!this.city.equals("") && !this.city.equals(city)) {
127        throw new IllegalArgumentException(city + " does not match " + this.city);
128      }
129
130      return new AverageAccumulator(
131          city,
132          earliest(this.start, eventDate),
133          latest(this.end, eventDate),
134          this.count + 1,
135          this.sum + value);
136    }
137
138    public AverageAccumulator merge(AverageAccumulator that) {
139      LocalDate mergedStart = earliest(this.start, that.start);
140      LocalDate mergedEnd = latest(this.end, that.end);
141      return new AverageAccumulator(
142          this.city, mergedStart, mergedEnd, this.count + that.count, this.sum + that.sum);
143    }
144
145    private LocalDate earliest(LocalDate d1, LocalDate d2) {
146      if (d1 == null) {
147        return d2;
148      } else if (d2 == null) {
149        return d1;
150      } else {
151        return d1.isBefore(d2) ? d1 : d2;
152      }
153    }
154
155    private LocalDate latest(LocalDate d1, LocalDate d2) {
156      if (d1 == null) {
157        return d2;
158      } else if (d2 == null) {
159        return d1;
160      } else {
161        return d1.isAfter(d2) ? d1 : d2;
162      }
163    }
164
165    public double average() {
166      return sum / count;
167    }
168
169    public String getCity() {
170      return city;
171    }
172
173    public LocalDate getStart() {
174      return start;
175    }
176
177    public LocalDate getEnd() {
178      return end;
179    }
180  }
181}
182

1Date,Country,City,Specie,count,min,max,median,variance
228/06/2021,GR,Athens,no2,116,0.5,58.9,5.5,2824.39
328/06/2021,GR,Athens,wind-speed,133,0.1,11.2,3,96.69
428/06/2021,GR,Athens,dew,24,14,20,18,35.92
528/06/2021,GR,Athens,temperature,141,24.4,38.4,30.5,123.18
628/06/2021,GR,Athens,pm25,116,34,85,68,702.29
7public class CityMetric {
8
9    private static final DateTimeFormatter dateFormatter = DateTimeFormatter.ofPattern("dd/MM/yyyy");
10
11    private final LocalDate localDate;
12    private final String country;
13    private final String city;
14    private final String reading;
15    private final int count;
16    private final double min;
17    private final double max;
18    private final double median;
19    private final double variance;
20
21    private CityMetric(LocalDate localDate, String country, String city, String reading, int count, double min, double max, double median, double variance) {
22        this.localDate = localDate;
23        this.country = country;
24        this.city = city;
25        this.reading = reading;
26        this.count = count;
27        this.min = min;
28        this.max = max;
29        this.median = median;
30        this.variance = variance;
31    }
32
33    public static CityMetric fromArray(String[] arr) {
34        LocalDate date = LocalDate.parse(arr[0], dateFormatter);
35        int count = Integer.parseInt(arr[4]);
36        double min = Double.parseDouble(arr[5]);
37        double max = Double.parseDouble(arr[6]);
38        double median = Double.parseDouble(arr[7]);
39        double variance = Double.parseDouble(arr[8]);
40
41        return new CityMetric(date, arr[1], arr[2], arr[3], count, min, max, median, variance);
42    }
43
44    public long getTimestamp() {
45        return getLocalDate()
46                .atStartOfDay()
47                .toInstant(ZoneOffset.UTC)
48                .toEpochMilli();
49    }
50
51//getters follow
52   final WatermarkStrategy<CityMetric> cityMetricWatermarkStrategy =
53            WatermarkStrategy.<CityMetric>forMonotonousTimestamps()  //we know they are sorted by time
54                    .withTimestampAssigner((cityMetric, l) -> cityMetric.getTimestamp());
55  final StreamingFileSink<Tuple4<LocalDate, LocalDate, String, Double>> fileSink =
56        StreamingFileSink.forRowFormat(
57                new Path("airquality"),
58                new SimpleStringEncoder<Tuple4<LocalDate, LocalDate, String, Double>>("UTF-8"))
59            .build();
60 source
61        .map(s -> s.split(",")) //split the CSV row into its fields
62        .filter(arr -> !arr[0].startsWith("Date")) // if it starts with Date it means it is the top header
63        .map(CityMetric::fromArray)  //create the object from the fields
64        .assignTimestampsAndWatermarks(cityMetricWatermarkStrategy) // we use the date as the event time
65        .filter(cm -> cm.getReading().equals("pm25")) // we want air quality of fine particulate matter pm2.5
66        .keyBy(CityMetric::getCity) // partition by city name
67        .window(TumblingEventTimeWindows.of(Time.days(7))) //windows of 7 days
68        .aggregate(new CityAverageAggregate()) // average the values
69        .name("cityair")
70        .addSink(fileSink); //output each partition to a file
71public class CityAverageAggregate
72    implements AggregateFunction<
73        CityMetric, CityAverageAggregate.AverageAccumulator, Tuple4<LocalDate, LocalDate, String, Double>> {
74
75  @Override
76  public AverageAccumulator createAccumulator() {
77    return new AverageAccumulator();
78  }
79
80  @Override
81  public AverageAccumulator add(CityMetric cityMetric, AverageAccumulator averageAccumulator) {
82    return averageAccumulator.add(
83        cityMetric.getCity(), cityMetric.getLocalDate(), cityMetric.getMedian());
84  }
85
86  @Override
87  public Tuple4<LocalDate, LocalDate, String, Double> getResult(
88      AverageAccumulator averageAccumulator) {
89    return Tuple4.of(
90        averageAccumulator.getStart(),
91        averageAccumulator.getEnd(),
92        averageAccumulator.getCity(),
93        averageAccumulator.average());
94  }
95
96  @Override
97  public AverageAccumulator merge(AverageAccumulator acc1, AverageAccumulator acc2) {
98    return acc1.merge(acc2);
99  }
100
101  public static class AverageAccumulator {
102    private final String city;
103    private final LocalDate start;
104    private final LocalDate end;
105    private final long count;
106    private final double sum;
107
108    public AverageAccumulator() {
109      city = "";
110      count = 0;
111      sum = 0;
112      start = null;
113      end = null;
114    }
115
116    AverageAccumulator(String city, LocalDate start, LocalDate end, long count, double sum) {
117      this.city = city;
118      this.count = count;
119      this.sum = sum;
120      this.start = start;
121      this.end = end;
122    }
123
124    public AverageAccumulator add(String city, LocalDate eventDate, double value) {
125      //make sure our dataflow is correct and we are summing data from the same city
126      if (!this.city.equals("") && !this.city.equals(city)) {
127        throw new IllegalArgumentException(city + " does not match " + this.city);
128      }
129
130      return new AverageAccumulator(
131          city,
132          earliest(this.start, eventDate),
133          latest(this.end, eventDate),
134          this.count + 1,
135          this.sum + value);
136    }
137
138    public AverageAccumulator merge(AverageAccumulator that) {
139      LocalDate mergedStart = earliest(this.start, that.start);
140      LocalDate mergedEnd = latest(this.end, that.end);
141      return new AverageAccumulator(
142          this.city, mergedStart, mergedEnd, this.count + that.count, this.sum + that.sum);
143    }
144
145    private LocalDate earliest(LocalDate d1, LocalDate d2) {
146      if (d1 == null) {
147        return d2;
148      } else if (d2 == null) {
149        return d1;
150      } else {
151        return d1.isBefore(d2) ? d1 : d2;
152      }
153    }
154
155    private LocalDate latest(LocalDate d1, LocalDate d2) {
156      if (d1 == null) {
157        return d2;
158      } else if (d2 == null) {
159        return d1;
160      } else {
161        return d1.isAfter(d2) ? d1 : d2;
162      }
163    }
164
165    public double average() {
166      return sum / count;
167    }
168
169    public String getCity() {
170      return city;
171    }
172
173    public LocalDate getStart() {
174      return start;
175    }
176
177    public LocalDate getEnd() {
178      return end;
179    }
180  }
181}
182(2021-07-12,2021-07-14,Belgrade,56.666666666666664)
183(2021-07-15,2021-07-21,Belgrade,56.0)
184(2021-07-22,2021-07-28,Belgrade,57.285714285714285)
185(2021-07-29,2021-08-04,Belgrade,43.57142857142857)
186(2021-08-05,2021-08-11,Belgrade,35.42857142857143)
187(2021-08-12,2021-08-18,Belgrade,43.42857142857143)
188(2021-08-19,2021-08-25,Belgrade,36.857142857142854)
189(2021-08-26,2021-09-01,Belgrade,50.285714285714285)
190(2021-09-02,2021-09-08,Belgrade,46.285714285714285)
191(2021-09-09,2021-09-15,Belgrade,54.857142857142854)
192(2021-09-16,2021-09-22,Belgrade,56.714285714285715)
193(2021-09-23,2021-09-29,Belgrade,59.285714285714285)
194(2021-09-30,2021-10-03,Belgrade,61.5)
195

1Date,Country,City,Specie,count,min,max,median,variance
228/06/2021,GR,Athens,no2,116,0.5,58.9,5.5,2824.39
328/06/2021,GR,Athens,wind-speed,133,0.1,11.2,3,96.69
428/06/2021,GR,Athens,dew,24,14,20,18,35.92
528/06/2021,GR,Athens,temperature,141,24.4,38.4,30.5,123.18
628/06/2021,GR,Athens,pm25,116,34,85,68,702.29
7public class CityMetric {
8
9    private static final DateTimeFormatter dateFormatter = DateTimeFormatter.ofPattern("dd/MM/yyyy");
10
11    private final LocalDate localDate;
12    private final String country;
13    private final String city;
14    private final String reading;
15    private final int count;
16    private final double min;
17    private final double max;
18    private final double median;
19    private final double variance;
20
21    private CityMetric(LocalDate localDate, String country, String city, String reading, int count, double min, double max, double median, double variance) {
22        this.localDate = localDate;
23        this.country = country;
24        this.city = city;
25        this.reading = reading;
26        this.count = count;
27        this.min = min;
28        this.max = max;
29        this.median = median;
30        this.variance = variance;
31    }
32
33    public static CityMetric fromArray(String[] arr) {
34        LocalDate date = LocalDate.parse(arr[0], dateFormatter);
35        int count = Integer.parseInt(arr[4]);
36        double min = Double.parseDouble(arr[5]);
37        double max = Double.parseDouble(arr[6]);
38        double median = Double.parseDouble(arr[7]);
39        double variance = Double.parseDouble(arr[8]);
40
41        return new CityMetric(date, arr[1], arr[2], arr[3], count, min, max, median, variance);
42    }
43
44    public long getTimestamp() {
45        return getLocalDate()
46                .atStartOfDay()
47                .toInstant(ZoneOffset.UTC)
48                .toEpochMilli();
49    }
50
51//getters follow
52   final WatermarkStrategy<CityMetric> cityMetricWatermarkStrategy =
53            WatermarkStrategy.<CityMetric>forMonotonousTimestamps()  //we know they are sorted by time
54                    .withTimestampAssigner((cityMetric, l) -> cityMetric.getTimestamp());
55  final StreamingFileSink<Tuple4<LocalDate, LocalDate, String, Double>> fileSink =
56        StreamingFileSink.forRowFormat(
57                new Path("airquality"),
58                new SimpleStringEncoder<Tuple4<LocalDate, LocalDate, String, Double>>("UTF-8"))
59            .build();
60 source
61        .map(s -> s.split(",")) //split the CSV row into its fields
62        .filter(arr -> !arr[0].startsWith("Date")) // if it starts with Date it means it is the top header
63        .map(CityMetric::fromArray)  //create the object from the fields
64        .assignTimestampsAndWatermarks(cityMetricWatermarkStrategy) // we use the date as the event time
65        .filter(cm -> cm.getReading().equals("pm25")) // we want air quality of fine particulate matter pm2.5
66        .keyBy(CityMetric::getCity) // partition by city name
67        .window(TumblingEventTimeWindows.of(Time.days(7))) //windows of 7 days
68        .aggregate(new CityAverageAggregate()) // average the values
69        .name("cityair")
70        .addSink(fileSink); //output each partition to a file
71public class CityAverageAggregate
72    implements AggregateFunction<
73        CityMetric, CityAverageAggregate.AverageAccumulator, Tuple4<LocalDate, LocalDate, String, Double>> {
74
75  @Override
76  public AverageAccumulator createAccumulator() {
77    return new AverageAccumulator();
78  }
79
80  @Override
81  public AverageAccumulator add(CityMetric cityMetric, AverageAccumulator averageAccumulator) {
82    return averageAccumulator.add(
83        cityMetric.getCity(), cityMetric.getLocalDate(), cityMetric.getMedian());
84  }
85
86  @Override
87  public Tuple4<LocalDate, LocalDate, String, Double> getResult(
88      AverageAccumulator averageAccumulator) {
89    return Tuple4.of(
90        averageAccumulator.getStart(),
91        averageAccumulator.getEnd(),
92        averageAccumulator.getCity(),
93        averageAccumulator.average());
94  }
95
96  @Override
97  public AverageAccumulator merge(AverageAccumulator acc1, AverageAccumulator acc2) {
98    return acc1.merge(acc2);
99  }
100
101  public static class AverageAccumulator {
102    private final String city;
103    private final LocalDate start;
104    private final LocalDate end;
105    private final long count;
106    private final double sum;
107
108    public AverageAccumulator() {
109      city = "";
110      count = 0;
111      sum = 0;
112      start = null;
113      end = null;
114    }
115
116    AverageAccumulator(String city, LocalDate start, LocalDate end, long count, double sum) {
117      this.city = city;
118      this.count = count;
119      this.sum = sum;
120      this.start = start;
121      this.end = end;
122    }
123
124    public AverageAccumulator add(String city, LocalDate eventDate, double value) {
125      //make sure our dataflow is correct and we are summing data from the same city
126      if (!this.city.equals("") && !this.city.equals(city)) {
127        throw new IllegalArgumentException(city + " does not match " + this.city);
128      }
129
130      return new AverageAccumulator(
131          city,
132          earliest(this.start, eventDate),
133          latest(this.end, eventDate),
134          this.count + 1,
135          this.sum + value);
136    }
137
138    public AverageAccumulator merge(AverageAccumulator that) {
139      LocalDate mergedStart = earliest(this.start, that.start);
140      LocalDate mergedEnd = latest(this.end, that.end);
141      return new AverageAccumulator(
142          this.city, mergedStart, mergedEnd, this.count + that.count, this.sum + that.sum);
143    }
144
145    private LocalDate earliest(LocalDate d1, LocalDate d2) {
146      if (d1 == null) {
147        return d2;
148      } else if (d2 == null) {
149        return d1;
150      } else {
151        return d1.isBefore(d2) ? d1 : d2;
152      }
153    }
154
155    private LocalDate latest(LocalDate d1, LocalDate d2) {
156      if (d1 == null) {
157        return d2;
158      } else if (d2 == null) {
159        return d1;
160      } else {
161        return d1.isAfter(d2) ? d1 : d2;
162      }
163    }
164
165    public double average() {
166      return sum / count;
167    }
168
169    public String getCity() {
170      return city;
171    }
172
173    public LocalDate getStart() {
174      return start;
175    }
176
177    public LocalDate getEnd() {
178      return end;
179    }
180  }
181}
182(2021-07-12,2021-07-14,Belgrade,56.666666666666664)
183(2021-07-15,2021-07-21,Belgrade,56.0)
184(2021-07-22,2021-07-28,Belgrade,57.285714285714285)
185(2021-07-29,2021-08-04,Belgrade,43.57142857142857)
186(2021-08-05,2021-08-11,Belgrade,35.42857142857143)
187(2021-08-12,2021-08-18,Belgrade,43.42857142857143)
188(2021-08-19,2021-08-25,Belgrade,36.857142857142854)
189(2021-08-26,2021-09-01,Belgrade,50.285714285714285)
190(2021-09-02,2021-09-08,Belgrade,46.285714285714285)
191(2021-09-09,2021-09-15,Belgrade,54.857142857142854)
192(2021-09-16,2021-09-22,Belgrade,56.714285714285715)
193(2021-09-23,2021-09-29,Belgrade,59.285714285714285)
194(2021-09-30,2021-10-03,Belgrade,61.5)
195(2021-06-28,2021-06-30,Belgrade,48.666666666666664)
196(2021-07-01,2021-07-07,Belgrade,41.142857142857146)
197(2021-07-08,2021-07-14,Belgrade,52.857142857142854)
198(2021-07-15,2021-07-21,Belgrade,56.0)
199(2021-07-22,2021-07-28,Belgrade,57.285714285714285)
200(2021-07-29,2021-08-04,Belgrade,43.57142857142857)
201(2021-08-05,2021-08-11,Belgrade,35.42857142857143)
202(2021-08-12,2021-08-18,Belgrade,43.42857142857143)
203(2021-08-19,2021-08-25,Belgrade,36.857142857142854)
204(2021-08-26,2021-09-01,Belgrade,50.285714285714285)
205(2021-09-02,2021-09-08,Belgrade,46.285714285714285)
206(2021-09-09,2021-09-15,Belgrade,54.857142857142854)
207(2021-09-16,2021-09-22,Belgrade,56.714285714285715)
208(2021-09-23,2021-09-29,Belgrade,59.285714285714285)
209(2021-09-30,2021-10-03,Belgrade,61.5)
210

1Date,Country,City,Specie,count,min,max,median,variance
228/06/2021,GR,Athens,no2,116,0.5,58.9,5.5,2824.39
328/06/2021,GR,Athens,wind-speed,133,0.1,11.2,3,96.69
428/06/2021,GR,Athens,dew,24,14,20,18,35.92
528/06/2021,GR,Athens,temperature,141,24.4,38.4,30.5,123.18
628/06/2021,GR,Athens,pm25,116,34,85,68,702.29
7public class CityMetric {
8
9    private static final DateTimeFormatter dateFormatter = DateTimeFormatter.ofPattern("dd/MM/yyyy");
10
11    private final LocalDate localDate;
12    private final String country;
13    private final String city;
14    private final String reading;
15    private final int count;
16    private final double min;
17    private final double max;
18    private final double median;
19    private final double variance;
20
21    private CityMetric(LocalDate localDate, String country, String city, String reading, int count, double min, double max, double median, double variance) {
22        this.localDate = localDate;
23        this.country = country;
24        this.city = city;
25        this.reading = reading;
26        this.count = count;
27        this.min = min;
28        this.max = max;
29        this.median = median;
30        this.variance = variance;
31    }
32
33    public static CityMetric fromArray(String[] arr) {
34        LocalDate date = LocalDate.parse(arr[0], dateFormatter);
35        int count = Integer.parseInt(arr[4]);
36        double min = Double.parseDouble(arr[5]);
37        double max = Double.parseDouble(arr[6]);
38        double median = Double.parseDouble(arr[7]);
39        double variance = Double.parseDouble(arr[8]);
40
41        return new CityMetric(date, arr[1], arr[2], arr[3], count, min, max, median, variance);
42    }
43
44    public long getTimestamp() {
45        return getLocalDate()
46                .atStartOfDay()
47                .toInstant(ZoneOffset.UTC)
48                .toEpochMilli();
49    }
50
51//getters follow
52   final WatermarkStrategy<CityMetric> cityMetricWatermarkStrategy =
53            WatermarkStrategy.<CityMetric>forMonotonousTimestamps()  //we know they are sorted by time
54                    .withTimestampAssigner((cityMetric, l) -> cityMetric.getTimestamp());
55  final StreamingFileSink<Tuple4<LocalDate, LocalDate, String, Double>> fileSink =
56        StreamingFileSink.forRowFormat(
57                new Path("airquality"),
58                new SimpleStringEncoder<Tuple4<LocalDate, LocalDate, String, Double>>("UTF-8"))
59            .build();
60 source
61        .map(s -> s.split(",")) //split the CSV row into its fields
62        .filter(arr -> !arr[0].startsWith("Date")) // if it starts with Date it means it is the top header
63        .map(CityMetric::fromArray)  //create the object from the fields
64        .assignTimestampsAndWatermarks(cityMetricWatermarkStrategy) // we use the date as the event time
65        .filter(cm -> cm.getReading().equals("pm25")) // we want air quality of fine particulate matter pm2.5
66        .keyBy(CityMetric::getCity) // partition by city name
67        .window(TumblingEventTimeWindows.of(Time.days(7))) //windows of 7 days
68        .aggregate(new CityAverageAggregate()) // average the values
69        .name("cityair")
70        .addSink(fileSink); //output each partition to a file
71public class CityAverageAggregate
72    implements AggregateFunction<
73        CityMetric, CityAverageAggregate.AverageAccumulator, Tuple4<LocalDate, LocalDate, String, Double>> {
74
75  @Override
76  public AverageAccumulator createAccumulator() {
77    return new AverageAccumulator();
78  }
79
80  @Override
81  public AverageAccumulator add(CityMetric cityMetric, AverageAccumulator averageAccumulator) {
82    return averageAccumulator.add(
83        cityMetric.getCity(), cityMetric.getLocalDate(), cityMetric.getMedian());
84  }
85
86  @Override
87  public Tuple4<LocalDate, LocalDate, String, Double> getResult(
88      AverageAccumulator averageAccumulator) {
89    return Tuple4.of(
90        averageAccumulator.getStart(),
91        averageAccumulator.getEnd(),
92        averageAccumulator.getCity(),
93        averageAccumulator.average());
94  }
95
96  @Override
97  public AverageAccumulator merge(AverageAccumulator acc1, AverageAccumulator acc2) {
98    return acc1.merge(acc2);
99  }
100
101  public static class AverageAccumulator {
102    private final String city;
103    private final LocalDate start;
104    private final LocalDate end;
105    private final long count;
106    private final double sum;
107
108    public AverageAccumulator() {
109      city = "";
110      count = 0;
111      sum = 0;
112      start = null;
113      end = null;
114    }
115
116    AverageAccumulator(String city, LocalDate start, LocalDate end, long count, double sum) {
117      this.city = city;
118      this.count = count;
119      this.sum = sum;
120      this.start = start;
121      this.end = end;
122    }
123
124    public AverageAccumulator add(String city, LocalDate eventDate, double value) {
125      //make sure our dataflow is correct and we are summing data from the same city
126      if (!this.city.equals("") && !this.city.equals(city)) {
127        throw new IllegalArgumentException(city + " does not match " + this.city);
128      }
129
130      return new AverageAccumulator(
131          city,
132          earliest(this.start, eventDate),
133          latest(this.end, eventDate),
134          this.count + 1,
135          this.sum + value);
136    }
137
138    public AverageAccumulator merge(AverageAccumulator that) {
139      LocalDate mergedStart = earliest(this.start, that.start);
140      LocalDate mergedEnd = latest(this.end, that.end);
141      return new AverageAccumulator(
142          this.city, mergedStart, mergedEnd, this.count + that.count, this.sum + that.sum);
143    }
144
145    private LocalDate earliest(LocalDate d1, LocalDate d2) {
146      if (d1 == null) {
147        return d2;
148      } else if (d2 == null) {
149        return d1;
150      } else {
151        return d1.isBefore(d2) ? d1 : d2;
152      }
153    }
154
155    private LocalDate latest(LocalDate d1, LocalDate d2) {
156      if (d1 == null) {
157        return d2;
158      } else if (d2 == null) {
159        return d1;
160      } else {
161        return d1.isAfter(d2) ? d1 : d2;
162      }
163    }
164
165    public double average() {
166      return sum / count;
167    }
168
169    public String getCity() {
170      return city;
171    }
172
173    public LocalDate getStart() {
174      return start;
175    }
176
177    public LocalDate getEnd() {
178      return end;
179    }
180  }
181}
182(2021-07-12,2021-07-14,Belgrade,56.666666666666664)
183(2021-07-15,2021-07-21,Belgrade,56.0)
184(2021-07-22,2021-07-28,Belgrade,57.285714285714285)
185(2021-07-29,2021-08-04,Belgrade,43.57142857142857)
186(2021-08-05,2021-08-11,Belgrade,35.42857142857143)
187(2021-08-12,2021-08-18,Belgrade,43.42857142857143)
188(2021-08-19,2021-08-25,Belgrade,36.857142857142854)
189(2021-08-26,2021-09-01,Belgrade,50.285714285714285)
190(2021-09-02,2021-09-08,Belgrade,46.285714285714285)
191(2021-09-09,2021-09-15,Belgrade,54.857142857142854)
192(2021-09-16,2021-09-22,Belgrade,56.714285714285715)
193(2021-09-23,2021-09-29,Belgrade,59.285714285714285)
194(2021-09-30,2021-10-03,Belgrade,61.5)
195(2021-06-28,2021-06-30,Belgrade,48.666666666666664)
196(2021-07-01,2021-07-07,Belgrade,41.142857142857146)
197(2021-07-08,2021-07-14,Belgrade,52.857142857142854)
198(2021-07-15,2021-07-21,Belgrade,56.0)
199(2021-07-22,2021-07-28,Belgrade,57.285714285714285)
200(2021-07-29,2021-08-04,Belgrade,43.57142857142857)
201(2021-08-05,2021-08-11,Belgrade,35.42857142857143)
202(2021-08-12,2021-08-18,Belgrade,43.42857142857143)
203(2021-08-19,2021-08-25,Belgrade,36.857142857142854)
204(2021-08-26,2021-09-01,Belgrade,50.285714285714285)
205(2021-09-02,2021-09-08,Belgrade,46.285714285714285)
206(2021-09-09,2021-09-15,Belgrade,54.857142857142854)
207(2021-09-16,2021-09-22,Belgrade,56.714285714285715)
208(2021-09-23,2021-09-29,Belgrade,59.285714285714285)
209(2021-09-30,2021-10-03,Belgrade,61.5)
210    final WatermarkStrategy<CityMetric> cityMetricWatermarkStrategy =
211            WatermarkStrategy.<CityMetric>noWatermarks()  
212                             .withTimestampAssigner((cityMetric, l) -> cityMetric.getTimestamp());
213

1import threading
2import <lots of other stuff>
3
4class ThreadClass(threading.Thread):
5    def __init__(self, outputs_dict_split):
6        super(ThreadClass, self).__init__()
7        self.outputs_dict_split = outputs_dict_split
8    def run(self):
9        outputs_dict_split = get_output(self.outputs_dict_split)
10        return outputs_dict_split
11
12def get_output(outputs_dict):
13    ### PEXPECT STUFF TO LOGIN AND RUN COMMANDS ####
14    ### WRITE DEVICE'S OUTPUTS TO DEVICE'S OUTPUTS_DICT RESULTS SUB-KEY ###
15
16def backbone(outputs_dict):
17    filterbykey = lambda keys: {x: outputs_dict[x] for x in keys}
18    num_threads = 2
19    device_split = np.array_split(list(outputs_dict.keys()), num_threads)
20
21    outputs_dict_split_list = []
22    split_list1 = list(device_split[0])
23    split_list2 = list(device_split[1])
24    outputs_dict_split1 = filterbykey(split_list1)
25    outputs_dict_split2 = filterbykey(split_list2)
26    t1 = ThreadClass(outputs_dict_split1)
27    t2 = ThreadClass(outputs_dict_split2)
28    t1.start()
29    t2.start()
30    t1.join()
31    t2.join()
32    outputs_dict_split1 = t1.outputs_dict_split
33    outputs_dict_split2 = t2.outputs_dict_split
34    outputs_dict_split_list.append(outputs_dict_split1)
35    outputs_dict_split_list.append(outputs_dict_split2)
36    outputs_dict = ChainMap(*outputs_dict_split_list)
37
38    ### Downstream Processing ###
39

1public class Test {
2
3    HazelcastInstance hz = Hazelcast.newHazelcastInstance();
4    JetInstance jet = Jet.newJetInstance();
5
6    public static void main(String[] args) {
7        Test t = new Test();
8        t.loadIntoIMap();
9        t.readFromIMap();
10    }
11
12    public void loadIntoIMap() {
13        IMap<String, String> map = hz.getMap("my-distributed-map");
14        // Standard Put and Get
15        map.put("1", "John");
16        map.put("2", "Mary");
17        map.put("3", "Jane");
18    }
19
20    public void readFromIMap() {
21        System.err.println("--manu---");
22        jet.getMap("s").put("1", "2");
23        System.err.println(jet.getMap("s").size());
24        System.err.println(jet.getMap("my-distributed-map").size());
25    }
26
27}
28
29