Kafka

1Exception in thread "main" joptsimple.UnrecognizedOptionException: zookeeper is not a recognized option
2        at joptsimple.OptionException.unrecognizedOption(OptionException.java:108)
3        at joptsimple.OptionParser.handleLongOptionToken(OptionParser.java:510)
4        at joptsimple.OptionParserState$2.handleArgument(OptionParserState.java:56)
5        at joptsimple.OptionParser.parse(OptionParser.java:396)
6        at kafka.admin.TopicCommand$TopicCommandOptions.<init>(TopicCommand.scala:517)
7        at kafka.admin.TopicCommand$.main(TopicCommand.scala:47)
8        at kafka.admin.TopicCommand.main(TopicCommand.scala)
9

1Exception in thread "main" joptsimple.UnrecognizedOptionException: zookeeper is not a recognized option
2        at joptsimple.OptionException.unrecognizedOption(OptionException.java:108)
3        at joptsimple.OptionParser.handleLongOptionToken(OptionParser.java:510)
4        at joptsimple.OptionParserState$2.handleArgument(OptionParserState.java:56)
5        at joptsimple.OptionParser.parse(OptionParser.java:396)
6        at kafka.admin.TopicCommand$TopicCommandOptions.<init>(TopicCommand.scala:517)
7        at kafka.admin.TopicCommand$.main(TopicCommand.scala:47)
8        at kafka.admin.TopicCommand.main(TopicCommand.scala)
9.\bin\windows\kafka-topics.bat --create --zookeeper localhost:2181 --replication-factor 1 --partions 1 --topic TestTopic
10

1curl -X POST -H  "Content-Type: application/vnd.schemaregistry.v1+json" \
2--data '{"schemaType":"PROTOBUF","references": [{"name": "other.proto","subject": "other.proto","version": 1}],"schema":"syntax = \"proto3\"; package com.acme; import \"other.proto\";\n\nmessage MyRecord {\n  string f1 = 1;\n  OtherRecord f2 = 2;\n }\n"}' \
3http://localhost:8081/subjects/test-schema/versions
4

1curl -X POST -H  "Content-Type: application/vnd.schemaregistry.v1+json" \
2--data '{"schemaType":"PROTOBUF","references": [{"name": "other.proto","subject": "other.proto","version": 1}],"schema":"syntax = \"proto3\"; package com.acme; import \"other.proto\";\n\nmessage MyRecord {\n  string f1 = 1;\n  OtherRecord f2 = 2;\n }\n"}' \
3http://localhost:8081/subjects/test-schema/versions
4syntax = "proto3";
5package some.other.package;
6
7message OtherRecord {
8  int32 other_id = 1;
9}
10

1curl -X POST -H  "Content-Type: application/vnd.schemaregistry.v1+json" \
2--data '{"schemaType":"PROTOBUF","references": [{"name": "other.proto","subject": "other.proto","version": 1}],"schema":"syntax = \"proto3\"; package com.acme; import \"other.proto\";\n\nmessage MyRecord {\n  string f1 = 1;\n  OtherRecord f2 = 2;\n }\n"}' \
3http://localhost:8081/subjects/test-schema/versions
4syntax = "proto3";
5package some.other.package;
6
7message OtherRecord {
8  int32 other_id = 1;
9}
10curl -X POST -H  "Content-Type: application/vnd.schemaregistry.v1+json" \
11--data '{"schemaType":"PROTOBUF","references": [{"name": "other.proto","subject": "other.proto","version": 1}],"schema":"syntax = \"proto3\"; package com.acme; import \"some.package.other.proto\";\n\nmessage MyRecord {\n  string f1 = 1;\n  OtherRecord f2 = 2;\n }\n"}' \
12http://localhost:8081/subjects/test-shcema/versions
13

1FROM mcr.microsoft.com/dotnet/runtime:3.1 AS base
2WORKDIR /app
3
4FROM mcr.microsoft.com/dotnet/sdk:3.1 AS build
5WORKDIR /src
6COPY ["Kafka-dotnet-consumer/Kafka-dotnet-consumer.csproj", "Kafka-dotnet-consumer/"]
7RUN dotnet restore "Kafka-dotnet-consumer/Kafka-dotnet-consumer.csproj"
8COPY . .
9WORKDIR "/src/Kafka-dotnet-consumer"
10RUN dotnet build "Kafka-dotnet-consumer.csproj" -c Release -o /app/build
11
12FROM build AS publish
13RUN dotnet publish "Kafka-dotnet-consumer.csproj" -c Release -o /app/publish
14
15FROM base AS final
16WORKDIR /app
17COPY --from=publish /app/publish .
18ENTRYPOINT ["dotnet", "Kafka-dotnet-consumer.dll"]
19

1FROM mcr.microsoft.com/dotnet/runtime:3.1 AS base
2WORKDIR /app
3
4FROM mcr.microsoft.com/dotnet/sdk:3.1 AS build
5WORKDIR /src
6COPY ["Kafka-dotnet-consumer/Kafka-dotnet-consumer.csproj", "Kafka-dotnet-consumer/"]
7RUN dotnet restore "Kafka-dotnet-consumer/Kafka-dotnet-consumer.csproj"
8COPY . .
9WORKDIR "/src/Kafka-dotnet-consumer"
10RUN dotnet build "Kafka-dotnet-consumer.csproj" -c Release -o /app/build
11
12FROM build AS publish
13RUN dotnet publish "Kafka-dotnet-consumer.csproj" -c Release -o /app/publish
14
15FROM base AS final
16WORKDIR /app
17COPY --from=publish /app/publish .
18ENTRYPOINT ["dotnet", "Kafka-dotnet-consumer.dll"]
19Severity    Code    Description Project File    Line    Suppression State
20Error   CTC1014 Docker command failed with exit code 1.
21#1 [internal] load build definition from Dockerfile
22#1 sha256:356bc2781f52b021d6dc2eefeef3212c983066d4fe3637fe7928c8165f181c52
23#1 DONE 0.0s
24
25#1 [internal] load build definition from Dockerfile
26#1 sha256:356bc2781f52b021d6dc2eefeef3212c983066d4fe3637fe7928c8165f181c52
27#1 transferring dockerfile: 826B done
28#1 DONE 0.0s
29
30#2 [internal] load .dockerignore
31#2 sha256:158b62c61546176b0f8a68c34f4aed7fe8e5f979cc578672dacd3c07aff01eb1
32#2 transferring context: 35B done
33#2 DONE 0.0s
34
35#3 [internal] load metadata for mcr.microsoft.com/dotnet/sdk:3.1-alpine
36#3 sha256:9c4e2456483bc5a0fda4bf0466bb996bef09b180bf33a44ede0bd988c1be9178
37#3 ...
38
39#4 [internal] load metadata for mcr.microsoft.com/dotnet/runtime:3.1-alpine
40#4 sha256:46a3f71ff7a02c9ad1111e95e06323b127c541ea25dc1bca0750bc3a2ea917ca
41#4 ERROR: failed to copy: httpReadSeeker: failed open: failed to do request: Get https://westeurope.data.mcr.microsoft.com/42012bb2682a4d76ba7fa17a9d9a9162-qb2vm9uiex//docker/registry/v2/blobs/sha256/87/87413803399bebbe093cfb4ef6c89d426c13a62811d7501d462f2f0e018321bb/data?P1=1627480321&P2=1&P3=1&P4=uDGSoX8YSljKnDQVR6fqniuqK8fjkRvyngwKxM7ljlM%3D&se=2021-07-28T13%3A52%3A01Z&sig=wJVu%2BBQo2sldEPr5ea6KHdflARqlzPZ9Ap7uBKcEYYw%3D&sp=r&spr=https&sr=b&sv=2016-05-31&regid=42012bb2682a4d76ba7fa17a9d9a9162: x509: certificate has expired or is not yet valid
42
43#3 [internal] load metadata for mcr.microsoft.com/dotnet/sdk:3.1-alpine
44#3 sha256:9c4e2456483bc5a0fda4bf0466bb996bef09b180bf33a44ede0bd988c1be9178
45#3 CANCELED
46------
47 > [internal] load metadata for mcr.microsoft.com/dotnet/runtime:3.1-alpine:
48------
49failed to solve with frontend dockerfile.v0: failed to create LLB definition: failed to copy: httpReadSeeker: failed open: failed to do request: Get https://westeurope.data.mcr.microsoft.com/42012bb2682a4d76ba7fa17a9d9a9162-qb2vm9uiex//docker/registry/v2/blobs/sha256/87/87413803399bebbe093cfb4ef6c89d426c13a62811d7501d462f2f0e018321bb/data?P1=1627480321&P2=1&P3=1&P4=uDGSoX8YSljKnDQVR6fqniuqK8fjkRvyngwKxM7ljlM%3D&se=2021-07-28T13%3A52%3A01Z&sig=wJVu%2BBQo2sldEPr5ea6KHdflARqlzPZ9Ap7uBKcEYYw%3D&sp=r&spr=https&sr=b&sv=2016-05-31&regid=42012bb2682a4d76ba7fa17a9d9a9162: x509: certificate has expired or is not yet valid Kafka-dotnet-consumer   C:\Ziad\repos\vdpm-datagrid\Kafka-dotnet-consumer\Kafka-dotnet-consumer\Dockerfile  1   
50

1Stopping service [Tomcat]
2Error starting ApplicationContext. To display the conditions report re-run your application with 'debug' enabled.
3***************************
4APPLICATION FAILED TO START
5***************************
6
7Description:
8
9Web server failed to start. Port 8080 was already in use.
10

1Stopping service [Tomcat]
2Error starting ApplicationContext. To display the conditions report re-run your application with 'debug' enabled.
3***************************
4APPLICATION FAILED TO START
5***************************
6
7Description:
8
9Web server failed to start. Port 8080 was already in use.
10<parent>
11  <groupId>org.springframework.boot</groupId>
12  <artifactId>spring-boot-starter-parent</artifactId>
13  <version>2.5.4</version>
14  <relativePath/> <!-- lookup parent from repository -->
15</parent>
16...
17<dependency>
18  <groupId>org.springframework.boot</groupId>
19  <artifactId>spring-boot-starter-web</artifactId>
20</dependency>
21<dependency>
22  <groupId>org.springframework.kafka</groupId>
23  <artifactId>spring-kafka</artifactId>
24</dependency>
25

1Stopping service [Tomcat]
2Error starting ApplicationContext. To display the conditions report re-run your application with 'debug' enabled.
3***************************
4APPLICATION FAILED TO START
5***************************
6
7Description:
8
9Web server failed to start. Port 8080 was already in use.
10<parent>
11  <groupId>org.springframework.boot</groupId>
12  <artifactId>spring-boot-starter-parent</artifactId>
13  <version>2.5.4</version>
14  <relativePath/> <!-- lookup parent from repository -->
15</parent>
16...
17<dependency>
18  <groupId>org.springframework.boot</groupId>
19  <artifactId>spring-boot-starter-web</artifactId>
20</dependency>
21<dependency>
22  <groupId>org.springframework.kafka</groupId>
23  <artifactId>spring-kafka</artifactId>
24</dependency>
25@Component
26public class KafkaConsumer {
27
28  @KafkaListener(topics = "test", groupId = "test")
29  public void process(String message) {
30
31  }
32}
33

1Stopping service [Tomcat]
2Error starting ApplicationContext. To display the conditions report re-run your application with 'debug' enabled.
3***************************
4APPLICATION FAILED TO START
5***************************
6
7Description:
8
9Web server failed to start. Port 8080 was already in use.
10<parent>
11  <groupId>org.springframework.boot</groupId>
12  <artifactId>spring-boot-starter-parent</artifactId>
13  <version>2.5.4</version>
14  <relativePath/> <!-- lookup parent from repository -->
15</parent>
16...
17<dependency>
18  <groupId>org.springframework.boot</groupId>
19  <artifactId>spring-boot-starter-web</artifactId>
20</dependency>
21<dependency>
22  <groupId>org.springframework.kafka</groupId>
23  <artifactId>spring-kafka</artifactId>
24</dependency>
25@Component
26public class KafkaConsumer {
27
28  @KafkaListener(topics = "test", groupId = "test")
29  public void process(String message) {
30
31  }
32}
33spring:
34  kafka:
35    bootstrap-servers: kafka:9092
36

1Stopping service [Tomcat]
2Error starting ApplicationContext. To display the conditions report re-run your application with 'debug' enabled.
3***************************
4APPLICATION FAILED TO START
5***************************
6
7Description:
8
9Web server failed to start. Port 8080 was already in use.
10<parent>
11  <groupId>org.springframework.boot</groupId>
12  <artifactId>spring-boot-starter-parent</artifactId>
13  <version>2.5.4</version>
14  <relativePath/> <!-- lookup parent from repository -->
15</parent>
16...
17<dependency>
18  <groupId>org.springframework.boot</groupId>
19  <artifactId>spring-boot-starter-web</artifactId>
20</dependency>
21<dependency>
22  <groupId>org.springframework.kafka</groupId>
23  <artifactId>spring-kafka</artifactId>
24</dependency>
25@Component
26public class KafkaConsumer {
27
28  @KafkaListener(topics = "test", groupId = "test")
29  public void process(String message) {
30
31  }
32}
33spring:
34  kafka:
35    bootstrap-servers: kafka:9092
362021-12-17 11:12:24.955  WARN 29067 --- [           main] ConfigServletWebServerApplicationContext : Exception encountered during context initialization - cancelling refresh attempt: org.springframework.context.ApplicationContextException: Failed to start bean 'webServerStartStop'; nested exception is org.springframework.boot.web.server.PortInUseException: Port 8080 is already in use
372021-12-17 11:12:24.959  INFO 29067 --- [           main] o.apache.catalina.core.StandardService   : Stopping service [Tomcat]
382021-12-17 11:12:24.969  INFO 29067 --- [           main] ConditionEvaluationReportLoggingListener : 
39
40Error starting ApplicationContext. To display the conditions report re-run your application with 'debug' enabled.
412021-12-17 11:12:24.978 ERROR 29067 --- [           main] o.s.b.d.LoggingFailureAnalysisReporter   : 
42
43***************************
44APPLICATION FAILED TO START
45***************************
46
47Description:
48
49Web server failed to start. Port 8080 was already in use.
50
51Action:
52
53Identify and stop the process that's listening on port 8080 or configure this application to listen on another port.
54
552021-12-17 11:12:25.151  WARN 29067 --- [ntainer#0-0-C-1] org.apache.kafka.clients.NetworkClient   : [Consumer clientId=consumer-test-1, groupId=test] Error while fetching metadata with correlation id 2 : {test=LEADER_NOT_AVAILABLE}
562021-12-17 11:12:25.154  INFO 29067 --- [ntainer#0-0-C-1] org.apache.kafka.clients.Metadata        : [Consumer clientId=consumer-test-1, groupId=test] Cluster ID: NwbnlV2vSdiYtDzgZ81TDQ
572021-12-17 11:12:25.156  INFO 29067 --- [ntainer#0-0-C-1] o.a.k.c.c.internals.AbstractCoordinator  : [Consumer clientId=consumer-test-1, groupId=test] Discovered group coordinator kafka:9092 (id: 2147483636 rack: null)
582021-12-17 11:12:25.159  INFO 29067 --- [ntainer#0-0-C-1] o.a.k.c.c.internals.AbstractCoordinator  : [Consumer clientId=consumer-test-1, groupId=test] (Re-)joining group
592021-12-17 11:12:25.179  INFO 29067 --- [ntainer#0-0-C-1] o.a.k.c.c.internals.AbstractCoordinator  : [Consumer clientId=consumer-test-1, groupId=test] (Re-)joining group
602021-12-17 11:12:27.004  INFO 29067 --- [ntainer#0-0-C-1] o.a.k.c.c.internals.AbstractCoordinator  : [Consumer clientId=consumer-test-1, groupId=test] Successfully joined group with generation Generation{generationId=2, memberId='consumer-test-1-c5924ab5-afc8-4720-a5d7-f8107ace3aad', protocol='range'}
612021-12-17 11:12:27.009  INFO 29067 --- [ntainer#0-0-C-1] o.a.k.c.c.internals.ConsumerCoordinator  : [Consumer clientId=consumer-test-1, groupId=test] Finished assignment for group at generation 2: {consumer-test-1-c5924ab5-afc8-4720-a5d7-f8107ace3aad=Assignment(partitions=[test-0])}
622021-12-17 11:12:27.021  INFO 29067 --- [ntainer#0-0-C-1] o.a.k.c.c.internals.AbstractCoordinator  : [Consumer clientId=consumer-test-1, groupId=test] Successfully synced group in generation Generation{generationId=2, memberId='consumer-test-1-c5924ab5-afc8-4720-a5d7-f8107ace3aad', protocol='range'}
632021-12-17 11:12:27.022  INFO 29067 --- [ntainer#0-0-C-1] o.a.k.c.c.internals.ConsumerCoordinator  : [Consumer clientId=consumer-test-1, groupId=test] Notifying assignor about the new Assignment(partitions=[test-0])
642021-12-17 11:12:27.025  INFO 29067 --- [ntainer#0-0-C-1] o.a.k.c.c.internals.ConsumerCoordinator  : [Consumer clientId=consumer-test-1, groupId=test] Adding newly assigned partitions: test-0
652021-12-17 11:12:27.029  INFO 29067 --- [ntainer#0-0-C-1] o.a.k.c.c.internals.ConsumerCoordinator  : [Consumer clientId=consumer-test-1, groupId=test] Found no committed offset for partition test-0
662021-12-17 11:12:27.034  INFO 29067 --- [ntainer#0-0-C-1] o.a.k.c.c.internals.ConsumerCoordinator  : [Consumer clientId=consumer-test-1, groupId=test] Found no committed offset for partition test-0
672021-12-17 11:12:27.040  INFO 29067 --- [ntainer#0-0-C-1] o.a.k.c.c.internals.SubscriptionState    : [Consumer clientId=consumer-test-1, groupId=test] Resetting offset for partition test-0 to position FetchPosition{offset=0, offsetEpoch=Optional.empty, currentLeader=LeaderAndEpoch{leader=Optional[kafka:9092 (id: 11 rack: null)], epoch=0}}.
682021-12-17 11:12:27.045  INFO 29067 --- [ntainer#0-0-C-1] o.s.k.l.KafkaMessageListenerContainer    : test: partitions assigned: [test-0]
69

1export JAVA_HOME=mnt/c/Program\ Files/Java/jdk1.8.0_231
2export PATH=$JAVA_HOME/bin:$PATH
3

1class Consumer(val consumer: KafkaConsumer<String, ConsumerRecord<String>>) {
2
3    fun run() {
4        consumer.seekToEnd(emptyList())
5        val pollDuration = 30 // seconds
6
7        while (true) {
8            val records = consumer.poll(Duration.ofSeconds(pollDuration))
9            // perform record analysis and commitSync()
10            }
11        }
12    }
13}
14

1class Consumer(val consumer: KafkaConsumer<String, ConsumerRecord<String>>) {
2
3    fun run() {
4        consumer.seekToEnd(emptyList())
5        val pollDuration = 30 // seconds
6
7        while (true) {
8            val records = consumer.poll(Duration.ofSeconds(pollDuration))
9            // perform record analysis and commitSync()
10            }
11        }
12    }
13}
14fun <T> buildConsumer(valueDeserializer: String): KafkaConsumer<String, T> {
15    val props = setupConfig(valueDeserializer)
16    Common.setupConsumerSecurityProtocol(props)
17    return createConsumer(props)
18}
19
20fun setupConfig(valueDeserializer: String): Properties {
21    // Configuration setup
22    val props = Properties()
23
24    props[ConsumerConfig.GROUP_ID_CONFIG] = config.applicationId
25    props[ConsumerConfig.CLIENT_ID_CONFIG] = config.kafka.clientId
26    props[ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG] = config.kafka.bootstrapServers
27    props[AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG] = config.kafka.schemaRegistryUrl
28
29    props[ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG] = config.kafka.stringDeserializer
30    props[ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG] = valueDeserializer
31    props[KafkaAvroDeserializerConfig.SPECIFIC_AVRO_READER_CONFIG] = "true"
32
33    props[ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG] = config.kafka.maxPollIntervalMs
34    props[ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG] = config.kafka.sessionTimeoutMs
35
36    props[ConsumerConfig.ALLOW_AUTO_CREATE_TOPICS_CONFIG] = "false"
37    props[ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG] = "false"
38    props[ConsumerConfig.AUTO_OFFSET_RESET_CONFIG] = "latest"
39
40    return props
41}
42
43fun <T> createConsumer(props: Properties): KafkaConsumer<String, T> {
44    val consumer = KafkaConsumer<String, T>(props)
45    consumer.subscribe(listOf(config.kafka.inputTopic))
46    return consumer
47}
48

1version: '2.1'
2
3services:
4  postgres:
5    container_name: listings-postgres
6    image: postgres:10-alpine
7    mem_limit: 500m
8    networks:
9      - listings-stack
10    ports:
11      - "5432:5432"
12    environment:
13      POSTGRES_DB: listings
14      POSTGRES_PASSWORD: listings
15      POSTGRES_USER: listings
16      PGUSER: listings
17    healthcheck:
18      test: ["CMD", "pg_isready"]
19      interval: 1s
20      timeout: 3s
21      retries: 30
22
23  listings-zookeeper:
24    container_name: listings-zookeeper
25    image: confluentinc/cp-zookeeper:6.2.0
26    environment:
27      ZOOKEEPER_CLIENT_PORT: 2181
28      ZOOKEEPER_TICK_TIME: 2000
29    networks:
30      - listings-stack
31    ports:
32      - "2181:2181"
33    healthcheck:
34      test: nc -z localhost 2181 || exit -1
35      interval: 10s
36      timeout: 5s
37      retries: 10
38
39  listings-kafka:
40    container_name: listings-kafka
41    image: confluentinc/cp-kafka:6.2.0
42    depends_on:
43      listings-zookeeper:
44        condition: service_healthy
45    environment:
46      KAFKA_BROKER_ID: 1
47      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://listings-kafka:9092,PLAINTEXT_HOST://localhost:29092
48      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
49      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
50      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
51      KAFKA_ZOOKEEPER_CONNECT: listings-zookeeper:2181
52    volumes:
53      - /var/run/docker.sock:/var/run/docker.sock
54    networks:
55      - listings-stack
56    ports:
57      - "29092:29092"
58    healthcheck:
59      test: kafka-topics --bootstrap-server 127.0.0.1:9092 --list
60      interval: 10s
61      timeout: 10s
62      retries: 50
63
64networks: {listings-stack: {}}
65

1version: '2.1'
2
3services:
4  postgres:
5    container_name: listings-postgres
6    image: postgres:10-alpine
7    mem_limit: 500m
8    networks:
9      - listings-stack
10    ports:
11      - "5432:5432"
12    environment:
13      POSTGRES_DB: listings
14      POSTGRES_PASSWORD: listings
15      POSTGRES_USER: listings
16      PGUSER: listings
17    healthcheck:
18      test: ["CMD", "pg_isready"]
19      interval: 1s
20      timeout: 3s
21      retries: 30
22
23  listings-zookeeper:
24    container_name: listings-zookeeper
25    image: confluentinc/cp-zookeeper:6.2.0
26    environment:
27      ZOOKEEPER_CLIENT_PORT: 2181
28      ZOOKEEPER_TICK_TIME: 2000
29    networks:
30      - listings-stack
31    ports:
32      - "2181:2181"
33    healthcheck:
34      test: nc -z localhost 2181 || exit -1
35      interval: 10s
36      timeout: 5s
37      retries: 10
38
39  listings-kafka:
40    container_name: listings-kafka
41    image: confluentinc/cp-kafka:6.2.0
42    depends_on:
43      listings-zookeeper:
44        condition: service_healthy
45    environment:
46      KAFKA_BROKER_ID: 1
47      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://listings-kafka:9092,PLAINTEXT_HOST://localhost:29092
48      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
49      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
50      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
51      KAFKA_ZOOKEEPER_CONNECT: listings-zookeeper:2181
52    volumes:
53      - /var/run/docker.sock:/var/run/docker.sock
54    networks:
55      - listings-stack
56    ports:
57      - "29092:29092"
58    healthcheck:
59      test: kafka-topics --bootstrap-server 127.0.0.1:9092 --list
60      interval: 10s
61      timeout: 10s
62      retries: 50
63
64networks: {listings-stack: {}}
65name: Build
66
67on: [ pull_request ]
68
69env:
70  AWS_ACCESS_KEY_ID: ${{ secrets.TUNNEL_AWS_ACCESS_KEY_ID }}
71  AWS_SECRET_ACCESS_KEY: ${{ secrets.TUNNEL_AWS_SECRET_ACCESS_KEY }}
72  AWS_DEFAULT_REGION: 'us-east-1'
73  CIRCLECI_KEY_TUNNEL: ${{ secrets.ID_RSA_CIRCLECI_TUNNEL }}
74
75jobs:
76  build:
77    name: Listings-API Build
78    runs-on: [ self-hosted, zap ]
79
80    steps:
81      - uses: actions/checkout@v2
82        with:
83          token: ${{ secrets.GH_OLXBR_PAT }}
84          submodules: recursive
85          path: ./repo
86          fetch-depth: 0
87
88      - name: Set up JDK 11
89        uses: actions/setup-java@v2
90        with:
91          distribution: 'adopt'
92          java-version: '11'
93          architecture: x64
94          cache: 'gradle'
95
96      - name: Docker up
97        working-directory: ./repo
98        run: docker-compose up -d
99
100      - name: Build with Gradle
101        working-directory: ./repo
102        run: ./gradlew build -Dhttps.protocols=TLSv1,TLSv1.1,TLSv1.2 -x integrationTest
103
104      - name: Integration tests with Gradle
105        working-directory: ./repo
106        run: ./gradlew integrationTest -Dhttps.protocols=TLSv1,TLSv1.1,TLSv1.2
107
108      - name: Sonarqube
109        working-directory: ./repo
110        env:
111          GITHUB_TOKEN: ${{ secrets.GH_OLXBR_PAT }}
112          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
113        run: ./gradlew sonarqube --info -Dhttps.protocols=TLSv1,TLSv1.1,TLSv1.2
114
115      - name: Docker down
116        if: always()
117        working-directory: ./repo
118        run: docker-compose down --remove-orphans
119
120      - name: Cleanup Gradle Cache
121        # Remove some files from the Gradle cache, so they aren't cached by GitHub Actions.
122        # Restoring these files from a GitHub Actions cache might cause problems for future builds.
123        run: |
124          rm -f ${{ env.HOME }}/.gradle/caches/modules-2/modules-2.lock
125          rm -f ${{ env.HOME }}/.gradle/caches/modules-2/gc.properties
126
127

1version: '2.1'
2
3services:
4  postgres:
5    container_name: listings-postgres
6    image: postgres:10-alpine
7    mem_limit: 500m
8    networks:
9      - listings-stack
10    ports:
11      - "5432:5432"
12    environment:
13      POSTGRES_DB: listings
14      POSTGRES_PASSWORD: listings
15      POSTGRES_USER: listings
16      PGUSER: listings
17    healthcheck:
18      test: ["CMD", "pg_isready"]
19      interval: 1s
20      timeout: 3s
21      retries: 30
22
23  listings-zookeeper:
24    container_name: listings-zookeeper
25    image: confluentinc/cp-zookeeper:6.2.0
26    environment:
27      ZOOKEEPER_CLIENT_PORT: 2181
28      ZOOKEEPER_TICK_TIME: 2000
29    networks:
30      - listings-stack
31    ports:
32      - "2181:2181"
33    healthcheck:
34      test: nc -z localhost 2181 || exit -1
35      interval: 10s
36      timeout: 5s
37      retries: 10
38
39  listings-kafka:
40    container_name: listings-kafka
41    image: confluentinc/cp-kafka:6.2.0
42    depends_on:
43      listings-zookeeper:
44        condition: service_healthy
45    environment:
46      KAFKA_BROKER_ID: 1
47      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://listings-kafka:9092,PLAINTEXT_HOST://localhost:29092
48      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
49      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
50      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
51      KAFKA_ZOOKEEPER_CONNECT: listings-zookeeper:2181
52    volumes:
53      - /var/run/docker.sock:/var/run/docker.sock
54    networks:
55      - listings-stack
56    ports:
57      - "29092:29092"
58    healthcheck:
59      test: kafka-topics --bootstrap-server 127.0.0.1:9092 --list
60      interval: 10s
61      timeout: 10s
62      retries: 50
63
64networks: {listings-stack: {}}
65name: Build
66
67on: [ pull_request ]
68
69env:
70  AWS_ACCESS_KEY_ID: ${{ secrets.TUNNEL_AWS_ACCESS_KEY_ID }}
71  AWS_SECRET_ACCESS_KEY: ${{ secrets.TUNNEL_AWS_SECRET_ACCESS_KEY }}
72  AWS_DEFAULT_REGION: 'us-east-1'
73  CIRCLECI_KEY_TUNNEL: ${{ secrets.ID_RSA_CIRCLECI_TUNNEL }}
74
75jobs:
76  build:
77    name: Listings-API Build
78    runs-on: [ self-hosted, zap ]
79
80    steps:
81      - uses: actions/checkout@v2
82        with:
83          token: ${{ secrets.GH_OLXBR_PAT }}
84          submodules: recursive
85          path: ./repo
86          fetch-depth: 0
87
88      - name: Set up JDK 11
89        uses: actions/setup-java@v2
90        with:
91          distribution: 'adopt'
92          java-version: '11'
93          architecture: x64
94          cache: 'gradle'
95
96      - name: Docker up
97        working-directory: ./repo
98        run: docker-compose up -d
99
100      - name: Build with Gradle
101        working-directory: ./repo
102        run: ./gradlew build -Dhttps.protocols=TLSv1,TLSv1.1,TLSv1.2 -x integrationTest
103
104      - name: Integration tests with Gradle
105        working-directory: ./repo
106        run: ./gradlew integrationTest -Dhttps.protocols=TLSv1,TLSv1.1,TLSv1.2
107
108      - name: Sonarqube
109        working-directory: ./repo
110        env:
111          GITHUB_TOKEN: ${{ secrets.GH_OLXBR_PAT }}
112          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
113        run: ./gradlew sonarqube --info -Dhttps.protocols=TLSv1,TLSv1.1,TLSv1.2
114
115      - name: Docker down
116        if: always()
117        working-directory: ./repo
118        run: docker-compose down --remove-orphans
119
120      - name: Cleanup Gradle Cache
121        # Remove some files from the Gradle cache, so they aren't cached by GitHub Actions.
122        # Restoring these files from a GitHub Actions cache might cause problems for future builds.
123        run: |
124          rm -f ${{ env.HOME }}/.gradle/caches/modules-2/modules-2.lock
125          rm -f ${{ env.HOME }}/.gradle/caches/modules-2/gc.properties
126
127  boolean compareRecordSend(String topicName, int expected) {
128    def condition = new PollingConditions()
129    condition.within(kafkaProperties.listener.pollTimeout.getSeconds() * 5) {
130      assert expected == getRecordSendTotal(topicName)
131    }
132    return true
133  }
134
135  int getRecordSendTotal(String topicName) {
136    kafkaTemplate.flush()
137    return kafkaTemplate.metrics().find {
138      it.key.name() == "record-send-total" && it.key.tags().get("topic") == topicName
139    }?.value?.metricValue() ?: 0
140  }
141

1version: '2.1'
2
3services:
4  postgres:
5    container_name: listings-postgres
6    image: postgres:10-alpine
7    mem_limit: 500m
8    networks:
9      - listings-stack
10    ports:
11      - "5432:5432"
12    environment:
13      POSTGRES_DB: listings
14      POSTGRES_PASSWORD: listings
15      POSTGRES_USER: listings
16      PGUSER: listings
17    healthcheck:
18      test: ["CMD", "pg_isready"]
19      interval: 1s
20      timeout: 3s
21      retries: 30
22
23  listings-zookeeper:
24    container_name: listings-zookeeper
25    image: confluentinc/cp-zookeeper:6.2.0
26    environment:
27      ZOOKEEPER_CLIENT_PORT: 2181
28      ZOOKEEPER_TICK_TIME: 2000
29    networks:
30      - listings-stack
31    ports:
32      - "2181:2181"
33    healthcheck:
34      test: nc -z localhost 2181 || exit -1
35      interval: 10s
36      timeout: 5s
37      retries: 10
38
39  listings-kafka:
40    container_name: listings-kafka
41    image: confluentinc/cp-kafka:6.2.0
42    depends_on:
43      listings-zookeeper:
44        condition: service_healthy
45    environment:
46      KAFKA_BROKER_ID: 1
47      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://listings-kafka:9092,PLAINTEXT_HOST://localhost:29092
48      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
49      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
50      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
51      KAFKA_ZOOKEEPER_CONNECT: listings-zookeeper:2181
52    volumes:
53      - /var/run/docker.sock:/var/run/docker.sock
54    networks:
55      - listings-stack
56    ports:
57      - "29092:29092"
58    healthcheck:
59      test: kafka-topics --bootstrap-server 127.0.0.1:9092 --list
60      interval: 10s
61      timeout: 10s
62      retries: 50
63
64networks: {listings-stack: {}}
65name: Build
66
67on: [ pull_request ]
68
69env:
70  AWS_ACCESS_KEY_ID: ${{ secrets.TUNNEL_AWS_ACCESS_KEY_ID }}
71  AWS_SECRET_ACCESS_KEY: ${{ secrets.TUNNEL_AWS_SECRET_ACCESS_KEY }}
72  AWS_DEFAULT_REGION: 'us-east-1'
73  CIRCLECI_KEY_TUNNEL: ${{ secrets.ID_RSA_CIRCLECI_TUNNEL }}
74
75jobs:
76  build:
77    name: Listings-API Build
78    runs-on: [ self-hosted, zap ]
79
80    steps:
81      - uses: actions/checkout@v2
82        with:
83          token: ${{ secrets.GH_OLXBR_PAT }}
84          submodules: recursive
85          path: ./repo
86          fetch-depth: 0
87
88      - name: Set up JDK 11
89        uses: actions/setup-java@v2
90        with:
91          distribution: 'adopt'
92          java-version: '11'
93          architecture: x64
94          cache: 'gradle'
95
96      - name: Docker up
97        working-directory: ./repo
98        run: docker-compose up -d
99
100      - name: Build with Gradle
101        working-directory: ./repo
102        run: ./gradlew build -Dhttps.protocols=TLSv1,TLSv1.1,TLSv1.2 -x integrationTest
103
104      - name: Integration tests with Gradle
105        working-directory: ./repo
106        run: ./gradlew integrationTest -Dhttps.protocols=TLSv1,TLSv1.1,TLSv1.2
107
108      - name: Sonarqube
109        working-directory: ./repo
110        env:
111          GITHUB_TOKEN: ${{ secrets.GH_OLXBR_PAT }}
112          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
113        run: ./gradlew sonarqube --info -Dhttps.protocols=TLSv1,TLSv1.1,TLSv1.2
114
115      - name: Docker down
116        if: always()
117        working-directory: ./repo
118        run: docker-compose down --remove-orphans
119
120      - name: Cleanup Gradle Cache
121        # Remove some files from the Gradle cache, so they aren't cached by GitHub Actions.
122        # Restoring these files from a GitHub Actions cache might cause problems for future builds.
123        run: |
124          rm -f ${{ env.HOME }}/.gradle/caches/modules-2/modules-2.lock
125          rm -f ${{ env.HOME }}/.gradle/caches/modules-2/gc.properties
126
127  boolean compareRecordSend(String topicName, int expected) {
128    def condition = new PollingConditions()
129    condition.within(kafkaProperties.listener.pollTimeout.getSeconds() * 5) {
130      assert expected == getRecordSendTotal(topicName)
131    }
132    return true
133  }
134
135  int getRecordSendTotal(String topicName) {
136    kafkaTemplate.flush()
137    return kafkaTemplate.metrics().find {
138      it.key.name() == "record-send-total" && it.key.tags().get("topic") == topicName
139    }?.value?.metricValue() ?: 0
140  }
141Condition not satisfied after 50.00 seconds and 496 attempts
142    at spock.util.concurrent.PollingConditions.within(PollingConditions.java:185)
143    at com.company.listings.KafkaAwareBaseSpec.compareRecordSend(KafkaAwareBaseSpec.groovy:31)
144    at com.company.listings.application.worker.listener.notifier.ListingNotifierITSpec.should notify listings(ListingNotifierITSpec.groovy:44)
145
146    Caused by:
147    Condition not satisfied:
148
149    expected == getRecordSendTotal(topicName)
150    |        |  |                  |
151    10       |  0                  v4
152                false
153

1version: '2.1'
2
3services:
4  postgres:
5    container_name: listings-postgres
6    image: postgres:10-alpine
7    mem_limit: 500m
8    networks:
9      - listings-stack
10    ports:
11      - "5432:5432"
12    environment:
13      POSTGRES_DB: listings
14      POSTGRES_PASSWORD: listings
15      POSTGRES_USER: listings
16      PGUSER: listings
17    healthcheck:
18      test: ["CMD", "pg_isready"]
19      interval: 1s
20      timeout: 3s
21      retries: 30
22
23  listings-zookeeper:
24    container_name: listings-zookeeper
25    image: confluentinc/cp-zookeeper:6.2.0
26    environment:
27      ZOOKEEPER_CLIENT_PORT: 2181
28      ZOOKEEPER_TICK_TIME: 2000
29    networks:
30      - listings-stack
31    ports:
32      - "2181:2181"
33    healthcheck:
34      test: nc -z localhost 2181 || exit -1
35      interval: 10s
36      timeout: 5s
37      retries: 10
38
39  listings-kafka:
40    container_name: listings-kafka
41    image: confluentinc/cp-kafka:6.2.0
42    depends_on:
43      listings-zookeeper:
44        condition: service_healthy
45    environment:
46      KAFKA_BROKER_ID: 1
47      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://listings-kafka:9092,PLAINTEXT_HOST://localhost:29092
48      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
49      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
50      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
51      KAFKA_ZOOKEEPER_CONNECT: listings-zookeeper:2181
52    volumes:
53      - /var/run/docker.sock:/var/run/docker.sock
54    networks:
55      - listings-stack
56    ports:
57      - "29092:29092"
58    healthcheck:
59      test: kafka-topics --bootstrap-server 127.0.0.1:9092 --list
60      interval: 10s
61      timeout: 10s
62      retries: 50
63
64networks: {listings-stack: {}}
65name: Build
66
67on: [ pull_request ]
68
69env:
70  AWS_ACCESS_KEY_ID: ${{ secrets.TUNNEL_AWS_ACCESS_KEY_ID }}
71  AWS_SECRET_ACCESS_KEY: ${{ secrets.TUNNEL_AWS_SECRET_ACCESS_KEY }}
72  AWS_DEFAULT_REGION: 'us-east-1'
73  CIRCLECI_KEY_TUNNEL: ${{ secrets.ID_RSA_CIRCLECI_TUNNEL }}
74
75jobs:
76  build:
77    name: Listings-API Build
78    runs-on: [ self-hosted, zap ]
79
80    steps:
81      - uses: actions/checkout@v2
82        with:
83          token: ${{ secrets.GH_OLXBR_PAT }}
84          submodules: recursive
85          path: ./repo
86          fetch-depth: 0
87
88      - name: Set up JDK 11
89        uses: actions/setup-java@v2
90        with:
91          distribution: 'adopt'
92          java-version: '11'
93          architecture: x64
94          cache: 'gradle'
95
96      - name: Docker up
97        working-directory: ./repo
98        run: docker-compose up -d
99
100      - name: Build with Gradle
101        working-directory: ./repo
102        run: ./gradlew build -Dhttps.protocols=TLSv1,TLSv1.1,TLSv1.2 -x integrationTest
103
104      - name: Integration tests with Gradle
105        working-directory: ./repo
106        run: ./gradlew integrationTest -Dhttps.protocols=TLSv1,TLSv1.1,TLSv1.2
107
108      - name: Sonarqube
109        working-directory: ./repo
110        env:
111          GITHUB_TOKEN: ${{ secrets.GH_OLXBR_PAT }}
112          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
113        run: ./gradlew sonarqube --info -Dhttps.protocols=TLSv1,TLSv1.1,TLSv1.2
114
115      - name: Docker down
116        if: always()
117        working-directory: ./repo
118        run: docker-compose down --remove-orphans
119
120      - name: Cleanup Gradle Cache
121        # Remove some files from the Gradle cache, so they aren't cached by GitHub Actions.
122        # Restoring these files from a GitHub Actions cache might cause problems for future builds.
123        run: |
124          rm -f ${{ env.HOME }}/.gradle/caches/modules-2/modules-2.lock
125          rm -f ${{ env.HOME }}/.gradle/caches/modules-2/gc.properties
126
127  boolean compareRecordSend(String topicName, int expected) {
128    def condition = new PollingConditions()
129    condition.within(kafkaProperties.listener.pollTimeout.getSeconds() * 5) {
130      assert expected == getRecordSendTotal(topicName)
131    }
132    return true
133  }
134
135  int getRecordSendTotal(String topicName) {
136    kafkaTemplate.flush()
137    return kafkaTemplate.metrics().find {
138      it.key.name() == "record-send-total" && it.key.tags().get("topic") == topicName
139    }?.value?.metricValue() ?: 0
140  }
141Condition not satisfied after 50.00 seconds and 496 attempts
142    at spock.util.concurrent.PollingConditions.within(PollingConditions.java:185)
143    at com.company.listings.KafkaAwareBaseSpec.compareRecordSend(KafkaAwareBaseSpec.groovy:31)
144    at com.company.listings.application.worker.listener.notifier.ListingNotifierITSpec.should notify listings(ListingNotifierITSpec.groovy:44)
145
146    Caused by:
147    Condition not satisfied:
148
149    expected == getRecordSendTotal(topicName)
150    |        |  |                  |
151    10       |  0                  v4
152                false
153spring:
154  kafka:
155    bootstrap-servers: localhost:29092
156    producer:
157      batch-size: 262144
158      buffer-memory: 536870912
159      retries: 1
160      key-serializer: org.apache.kafka.common.serialization.StringSerializer
161      value-serializer: org.apache.kafka.common.serialization.ByteArraySerializer
162      acks: all
163      properties:
164        linger.ms: 0
165

1{"known_type": "test", 
2 "unknown_type": {"attr1": true, "attr2": "value2", "attr3": 777}}
3
4{"known_type": "test", 
5 "unknown_type": {"field1":[1,3,4,7], "field2": {"sub_field": "test"}}}
6
7{"known_type": "test", 
8"unknown_type": {"param": "some_value"}}
9

1{"known_type": "test", 
2 "unknown_type": {"attr1": true, "attr2": "value2", "attr3": 777}}
3
4{"known_type": "test", 
5 "unknown_type": {"field1":[1,3,4,7], "field2": {"sub_field": "test"}}}
6
7{"known_type": "test", 
8"unknown_type": {"param": "some_value"}}
9Example(test,Map(attr1 -> true, attr2 -> "value2", attr3 -> 777))
10
11Example(test,Map(field1 -> [1,3,4,7], field2 -> {"sub_field" : "sub_value"}))
12
13Example(test,Map(param -> "some_value"))
14

1{"known_type": "test", 
2 "unknown_type": {"attr1": true, "attr2": "value2", "attr3": 777}}
3
4{"known_type": "test", 
5 "unknown_type": {"field1":[1,3,4,7], "field2": {"sub_field": "test"}}}
6
7{"known_type": "test", 
8"unknown_type": {"param": "some_value"}}
9Example(test,Map(attr1 -> true, attr2 -> "value2", attr3 -> 777))
10
11Example(test,Map(field1 -> [1,3,4,7], field2 -> {"sub_field" : "sub_value"}))
12
13Example(test,Map(param -> "some_value"))
14case class Example(known_type: String = "", unknown_type: Map[String, String])
15
16val result: Option[Example] = for {
17    json       <- parse(rawJson2).toOption
18    t          <- json.hcursor.downField("known_type").as[String].toOption
19    map        <- json.hcursor.downField("unknown_type").as[JsonObject].toOption
20    m = map.toMap.map { case (name, value) => name -> value.toString() }
21} yield Example(t, m)
22