kafdrop | web UI for viewing Kafka topics and browsing consumer groups | Pub Sub library

pyspark.sql.utils.AnalysisException:  Failed to find data source: kafka. Please deploy the application as per the deployment section of "Structured Streaming + Kafka Integration Guide".

---
version: '3.7'

services:
  zookeeper:
    image: bitnami/zookeeper:3
    ports:
      - 2181:2181
    environment:
      ALLOW_ANONYMOUS_LOGIN: "yes"
  kafka:
    image: bitnami/kafka:2
    ports:
      - 9092:9092
    environment:
      KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181
      ALLOW_PLAINTEXT_LISTENER: "yes"
      KAFKA_LISTENERS: >-
          INTERNAL://:29092,EXTERNAL://:9092
      KAFKA_ADVERTISED_LISTENERS: >-
          INTERNAL://kafka:29092,EXTERNAL://localhost:9092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: >-
          INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: "INTERNAL"
    depends_on:
      - zookeeper

  spark:
    image: docker.io/bitnami/spark:3-debian-10
    environment:
      - SPARK_MODE=master
      - SPARK_RPC_AUTHENTICATION_ENABLED=no
      - SPARK_RPC_ENCRYPTION_ENABLED=no
      - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
      - SPARK_SSL_ENABLED=no
    ports:
      - '8080:8080'
    volumes:
      - ./:/home/workspace/
      - ./spark/jars:/opt/bitnami/spark/.ivy2 

  spark-worker-1:
    image: docker.io/bitnami/spark:3-debian-10
    environment:
      - SPARK_MODE=worker
      - SPARK_MASTER_URL=spark://spark:7077
      - SPARK_WORKER_MEMORY=1G
      - SPARK_WORKER_CORES=1
      - SPARK_RPC_AUTHENTICATION_ENABLED=no
      - SPARK_RPC_ENCRYPTION_ENABLED=no
      - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
      - SPARK_SSL_ENABLED=no
    volumes:
      - ./:/home/workspace/
      - ./spark/jars:/opt/bitnami/spark/.ivy2 
      
  kafdrop:
    image: obsidiandynamics/kafdrop:latest
    ports:
      - 9000:9000
    environment:
      KAFKA_BROKERCONNECT: kafka:29092
    depends_on:
      - kafka

from pyspark.sql import SparkSession
import os

#os.environ['PYSPARK_SUBMIT_ARGS'] = '--packages org.apache.spark:spark-sql-kafka-0-10_2.12:3.2.0,org.apache.kafka:kafka-clients:2.8.1'
# the source for this data pipeline is a kafka topic, defined below
spark = SparkSession.builder.appName("fuel-level").master("local[*]").getOrCreate()
spark.sparkContext.setLogLevel('WARN')

kafkaRawStreamingDF = spark                          \
    .readStream                                          \
    .format("kafka")                                     \
    .option("kafka.bootstrap.servers", "localhost:9092") \
    .option("subscribe","SimLab-KUKA")                  \
    .option("startingOffsets","earliest")\
    .load()                                     

#this is necessary for Kafka Data Frame to be readable, into a single column  value
kafkaStreamingDF = kafkaRawStreamingDF.selectExpr("cast(key as string) key", "cast(value as string) value")

kafkaStreamingDF.writeStream.outputMode("append").format("console").start().awaitTermination()

Error: Missing application resource.

Usage: spark-submit [options] <app jar | python file | R file> [app arguments]
Usage: spark-submit --kill [submission ID] --master [spark://...]
Usage: spark-submit --status [submission ID] --master [spark://...]
Usage: spark-submit run-example [options] example-class [example args]

Options:
  --master MASTER_URL         spark://host:port, mesos://host:port, yarn,
                              k8s://https://host:port, or local (Default: local[*]).
  --deploy-mode DEPLOY_MODE   Whether to launch the driver program locally ("client") or
                              on one of the worker machines inside the cluster ("cluster")
                              (Default: client).
  --class CLASS_NAME          Your application's main class (for Java / Scala apps).
  --name NAME                 A name of your application.
  --jars JARS                 Comma-separated list of jars to include on the driver
                              and executor classpaths.
  --packages                  Comma-separated list of maven coordinates of jars to include
                              on the driver and executor classpaths. Will search the local
                              maven repo, then maven central and any additional remote
                              repositories given by --repositories. The format for the
                              coordinates should be groupId:artifactId:version.
  --exclude-packages          Comma-separated list of groupId:artifactId, to exclude while
                              resolving the dependencies provided in --packages to avoid
                              dependency conflicts.
  --repositories              Comma-separated list of additional remote repositories to
                              search for the maven coordinates given with --packages.
  --py-files PY_FILES         Comma-separated list of .zip, .egg, or .py files to place
                              on the PYTHONPATH for Python apps.
  --files FILES               Comma-separated list of files to be placed in the working
                              directory of each executor. File paths of these files
                              in executors can be accessed via SparkFiles.get(fileName).
  --archives ARCHIVES         Comma-separated list of archives to be extracted into the
                              working directory of each executor.

  --conf, -c PROP=VALUE       Arbitrary Spark configuration property.
  --properties-file FILE      Path to a file from which to load extra properties. If not
                              specified, this will look for conf/spark-defaults.conf.

  --driver-memory MEM         Memory for driver (e.g. 1000M, 2G) (Default: 1024M).
  --driver-java-options       Extra Java options to pass to the driver.
  --driver-library-path       Extra library path entries to pass to the driver.
  --driver-class-path         Extra class path entries to pass to the driver. Note that
                              jars added with --jars are automatically included in the
                              classpath.

  --executor-memory MEM       Memory per executor (e.g. 1000M, 2G) (Default: 1G).

  --proxy-user NAME           User to impersonate when submitting the application.
                              This argument does not work with --principal / --keytab.

  --help, -h                  Show this help message and exit.
  --verbose, -v               Print additional debug output.
  --version,                  Print the version of current Spark.

 Cluster deploy mode only:
  --driver-cores NUM          Number of cores used by the driver, only in cluster mode
                              (Default: 1).

 Spark standalone or Mesos with cluster deploy mode only:
  --supervise                 If given, restarts the driver on failure.

 Spark standalone, Mesos or K8s with cluster deploy mode only:
  --kill SUBMISSION_ID        If given, kills the driver specified.
  --status SUBMISSION_ID      If given, requests the status of the driver specified.

 Spark standalone, Mesos and Kubernetes only:
  --total-executor-cores NUM  Total cores for all executors.

 Spark standalone, YARN and Kubernetes only:
  --executor-cores NUM        Number of cores used by each executor. (Default: 1 in
                              YARN and K8S modes, or all available cores on the worker
                              in standalone mode).

 Spark on YARN and Kubernetes only:
  --num-executors NUM         Number of executors to launch (Default: 2).
                              If dynamic allocation is enabled, the initial number of
                              executors will be at least NUM.
  --principal PRINCIPAL       Principal to be used to login to KDC.
  --keytab KEYTAB             The full path to the file that contains the keytab for the
                              principal specified above.

 Spark on YARN only:
  --queue QUEUE_NAME          The YARN queue to submit to (Default: "default").
      
Traceback (most recent call last):
  File "/Users/janikbischoff/Documents/Uni/PuL/BA/Code/Tests/spark-test.py", line 6, in <module>
    spark = SparkSession.builder.appName("fuel-level").master("local[*]").getOrCreate()
  File "/Users/janikbischoff/Library/Python/3.8/lib/python/site-packages/pyspark/sql/session.py", line 228, in getOrCreate
    sc = SparkContext.getOrCreate(sparkConf)
  File "/Users/janikbischoff/Library/Python/3.8/lib/python/site-packages/pyspark/context.py", line 392, in getOrCreate
    SparkContext(conf=conf or SparkConf())
  File "/Users/janikbischoff/Library/Python/3.8/lib/python/site-packages/pyspark/context.py", line 144, in __init__
    SparkContext._ensure_initialized(self, gateway=gateway, conf=conf)
  File "/Users/janikbischoff/Library/Python/3.8/lib/python/site-packages/pyspark/context.py", line 339, in _ensure_initialized
    SparkContext._gateway = gateway or launch_gateway(conf)
  File "/Users/janikbischoff/Library/Python/3.8/lib/python/site-packages/pyspark/java_gateway.py", line 108, in launch_gateway
    raise RuntimeError("Java gateway process exited before sending its port number")
RuntimeError: Java gateway process exited before sending its port number

log.retention.ms=1814400000
transaction.state.log.replication.factor=2
offsets.topic.replication.factor=2
auto.create.topics.enable=true
default.replication.factor=2
min.insync.replicas=1
num.io.threads=8
num.network.threads=5
num.partitions=1
num.replica.fetchers=2
replica.lag.time.max.ms=30000
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
socket.send.buffer.bytes=102400
unclean.leader.election.enable=true
zookeeper.session.timeout.ms=18000

bootstrap.servers=broker1:9092,broker2:9092
group.id=picturesGroup3
auto.offset.reset=earliest

bootstrap.servers=localhost:9092
max.in.flight.requests.per.connection=1
retries=2000000000
acks=all
max.block.ms=2000000000

version: '3.7'

services:
  zookeeper-1:
    image: confluentinc/cp-zookeeper:6.2.1
    hostname: zookeeper-1
    container_name: zookeeper-1
    ports:
      - "2181:2181"
      - "2888:2888"
      - "3888:3888"
    environment:
      ZOOKEEPER_SERVER_ID: 1
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_PEER_PORT: 2888
      ZOOKEEPER_LEADER_PORT: 3888
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_INIT_LIMIT: 5
      ZOOKEEPER_SYNC_LIMIT: 2
      ZOOKEEPER_SERVERS: "localhost:2888:3888;192.168.100.14:12888:13888;192.168.100.14:22888:23888"
    volumes:
      - ./kafka-data/zookeeper-1:/var/lib/zookeeper/data
      - ./kafka-data/zookeeper-logs-1:/var/lib/zookeeper/log
    networks:
      - mynet

  zookeeper-2:
    image: confluentinc/cp-zookeeper:6.2.1
    hostname: zookeeper-2
    container_name: zookeeper-2
    ports:
      - "12181:12181"
      - "12888:12888"
      - "13888:13888"
    environment:
      ZOOKEEPER_SERVER_ID: 2
      ZOOKEEPER_CLIENT_PORT: 12181
      ZOOKEEPER_PEER_PORT: 12888
      ZOOKEEPER_LEADER_PORT: 13888
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_INIT_LIMIT: 5
      ZOOKEEPER_SYNC_LIMIT: 2
      ZOOKEEPER_SERVERS: "192.168.100.14:2888:3888;localhost:12888:13888;192.168.100.14:22888:23888"
    volumes:
      - ./kafka-data/zookeeper-2:/var/lib/zookeeper/data
      - ./kafka-data/zookeeper-logs-2:/var/lib/zookeeper/log
    networks:
      - mynet

  zookeeper-3:
    image: confluentinc/cp-zookeeper:6.2.1
    hostname: zookeeper-3
    container_name: zookeeper-3
    ports:
      - "22181:22181"
      - "22888:22888"
      - "23888:23888"
    environment:
      ZOOKEEPER_SERVER_ID: 3
      ZOOKEEPER_CLIENT_PORT: 22181
      ZOOKEEPER_PEER_PORT: 22888
      ZOOKEEPER_LEADER_PORT: 23888
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_INIT_LIMIT: 5
      ZOOKEEPER_SYNC_LIMIT: 2
      ZOOKEEPER_SERVERS: "192.168.100.14:2888:3888;192.168.100.14:12888:13888;localhost:22888:23888"
    volumes:
      - ./kafka-data/zookeeper-3:/var/lib/zookeeper/data
      - ./kafka-data/zookeeper-logs-3:/var/lib/zookeeper/log
    networks:
      - mynet

  broker-1:
    image: confluentinc/cp-kafka:6.2.1
    hostname: broker-1
    container_name: broker-1
    depends_on:
      - zookeeper-1
      - zookeeper-2
      - zookeeper-3
    ports:
      - "9092:9092"
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_LISTENERS: OUTSIDE://0.0.0.0:9092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: OUTSIDE:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: OUTSIDE://192.168.100.14:9092
      KAFKA_INTER_BROKER_LISTENER_NAME: OUTSIDE
      KAFKA_ZOOKEEPER_CONNECT: "192.168.100.14:2181,192.168.100.14:12181,192.168.100.14:22181"
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_LOG_MESSAGE_TIMESTAMP_TYPE: 'LogAppendTime'
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_CONNECTIONS_MAX_IDLE_MS: 31536000000 # 1 year
    volumes:
      - ./kafka-data/kafka-1:/var/lib/kafka/data
    networks:
      - mynet

  # PORT 8081 reserved for Schema Registry

  kafka-rest-1:
    image: confluentinc/cp-kafka-rest:6.2.1
    hostname: kafka-rest-1
    container_name: kafka-rest-1
    depends_on:
      - broker-1
    ports:
      - "8082:8082"
    environment:
      KAFKA_REST_HOST_NAME: 192.168.100.14
      KAFKA_REST_LISTENERS: http://0.0.0.0:8082
      KAFKA_REST_BOOTSTRAP_SERVERS: 192.168.100.14:9092
    networks:
      - mynet

  # PORT 8083 reserved for Kafka-Connect REST API

  kafka-ui-1:
    image: provectuslabs/kafka-ui:0.2.1
    hostname: kafka-ui-1
    container_name: kafka-ui-1
    depends_on:
      - broker-1
    ports:
      - "8084:8080"
    environment:
      KAFKA_CLUSTERS_0_NAME: lab
      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: 192.168.100.14:9092
      KAFKA_CLUSTERS_0_ZOOKEEPER: "192.168.100.14:2181;192.168.100.14:12181;192.168.100.14:22181"
    networks:
      - mynet

  kafdrop-1:
    image: obsidiandynamics/kafdrop:3.27.0
    hostname: kafrop-1
    container_name: kafdrop-1
    depends_on:
      - broker-1
    ports:
      - "8085:9000"
    environment:
      KAFKA_BROKERCONNECT: 192.168.100.14:9092
      JVM_OPTS: "-Xms32M -Xmx64M"
      SERVER_SERVLET_CONTEXTPATH: "/"
    networks:
      - mynet

networks:
  mynet:
    driver: bridge

# docker-compose up broker-1
zookeeper-1 is up-to-date
Starting broker-1 ... done
Attaching to broker-1
broker-1        | ===> User
broker-1        | uid=1000(appuser) gid=1000(appuser) groups=1000(appuser)
broker-1        | ===> Configuring ...
broker-1        | ===> Running preflight checks ...
broker-1        | ===> Check if /var/lib/kafka/data is writable ...
broker-1        | ===> Check if Zookeeper is healthy ...
broker-1        | SLF4J: Class path contains multiple SLF4J bindings.
broker-1        | SLF4J: Found binding in [jar:file:/usr/share/java/cp-base-new/slf4j-log4j12-1.7.30.jar!/org/slf4j/impl/StaticLoggerBinder.class]
broker-1        | SLF4J: Found binding in [jar:file:/usr/share/java/cp-base-new/slf4j-simple-1.7.30.jar!/org/slf4j/impl/StaticLoggerBinder.class]
broker-1        | SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
broker-1        | SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory]
broker-1        | log4j:WARN No appenders could be found for logger (io.confluent.admin.utils.cli.ZookeeperReadyCommand).
broker-1        | log4j:WARN Please initialize the log4j system properly.
broker-1        | log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.

broker-1 exited with code 1

version: '3.7'

services:
  zookeeper-1:
    image: confluentinc/cp-zookeeper:6.2.1
    hostname: zookeeper-1
    container_name: zookeeper-1
    ports:
      - "2181:2181"
      - "2888:2888"
      - "3888:3888"
    environment:
      ZOOKEEPER_SERVER_ID: 1
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_PEER_PORT: 2888
      ZOOKEEPER_LEADER_PORT: 3888
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_INIT_LIMIT: 5
      ZOOKEEPER_SYNC_LIMIT: 2
      ZOOKEEPER_SERVERS: "localhost:2888:3888" #;192.168.100.14:12888:13888;192.168.100.14:22888:23888"
    volumes:
      - ./kafka-data/zookeeper-1:/var/lib/zookeeper/data
      - ./kafka-data/zookeeper-logs-1:/var/lib/zookeeper/log
    networks:
      - mynet

#  zookeeper-2:
#    image: confluentinc/cp-zookeeper:6.2.1
#    hostname: zookeeper-2
#    container_name: zookeeper-2
#    ports:
#      - "12181:12181"
#      - "12888:12888"
#      - "13888:13888"
#    environment:
#      ZOOKEEPER_SERVER_ID: 2
#      ZOOKEEPER_CLIENT_PORT: 12181
#      ZOOKEEPER_PEER_PORT: 12888
#      ZOOKEEPER_LEADER_PORT: 13888
#      ZOOKEEPER_TICK_TIME: 2000
#      ZOOKEEPER_INIT_LIMIT: 5
#      ZOOKEEPER_SYNC_LIMIT: 2
#      ZOOKEEPER_SERVERS: "192.168.100.14:2888:3888;localhost:12888:13888;192.168.100.14:22888:23888"
#    volumes:
#      - ./kafka-data/zookeeper-2:/var/lib/zookeeper/data
#      - ./kafka-data/zookeeper-logs-2:/var/lib/zookeeper/log
#    networks:
#      - mynet
#
#  zookeeper-3:
#    image: confluentinc/cp-zookeeper:6.2.1
#    hostname: zookeeper-3
#    container_name: zookeeper-3
#    ports:
#      - "22181:22181"
#      - "22888:22888"
#      - "23888:23888"
#    environment:
#      ZOOKEEPER_SERVER_ID: 3
#      ZOOKEEPER_CLIENT_PORT: 22181
#      ZOOKEEPER_PEER_PORT: 22888
#      ZOOKEEPER_LEADER_PORT: 23888
#      ZOOKEEPER_TICK_TIME: 2000
#      ZOOKEEPER_INIT_LIMIT: 5
#      ZOOKEEPER_SYNC_LIMIT: 2
#      ZOOKEEPER_SERVERS: "192.168.100.14:2888:3888;192.168.100.14:12888:13888;localhost:22888:23888"
#    volumes:
#      - ./kafka-data/zookeeper-3:/var/lib/zookeeper/data
#      - ./kafka-data/zookeeper-logs-3:/var/lib/zookeeper/log
#    networks:
#      - mynet

  broker-1:
    image: confluentinc/cp-kafka:6.2.1
    hostname: broker-1
    container_name: broker-1
    depends_on:
      - zookeeper-1
#      - zookeeper-2
#      - zookeeper-3
    ports:
      - "9092:9092"
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_LISTENERS: OUTSIDE://0.0.0.0:9092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: OUTSIDE:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: OUTSIDE://192.168.100.14:9092
      KAFKA_INTER_BROKER_LISTENER_NAME: OUTSIDE
      KAFKA_ZOOKEEPER_CONNECT: "192.168.100.14:2181" #,192.168.100.14:12181,192.168.100.14:22181"
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_LOG_MESSAGE_TIMESTAMP_TYPE: 'LogAppendTime'
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_CONNECTIONS_MAX_IDLE_MS: 31536000000 # 1 year
    volumes:
      - ./kafka-data/kafka-1:/var/lib/kafka/data
    networks:
      - mynet

  # PORT 8081 reserved for Schema Registry

  kafka-rest-1:
    image: confluentinc/cp-kafka-rest:6.2.1
    hostname: kafka-rest-1
    container_name: kafka-rest-1
    depends_on:
      - broker-1
    ports:
      - "8082:8082"
    environment:
      KAFKA_REST_HOST_NAME: 192.168.100.14
      KAFKA_REST_LISTENERS: http://0.0.0.0:8082
      KAFKA_REST_BOOTSTRAP_SERVERS: 192.168.100.14:9092
    networks:
      - mynet

  # PORT 8083 reserved for Kafka-Connect REST API

  kafka-ui-1:
    image: provectuslabs/kafka-ui:0.2.1
    hostname: kafka-ui-1
    container_name: kafka-ui-1
    depends_on:
      - broker-1
    ports:
      - "8084:8080"
    environment:
      KAFKA_CLUSTERS_0_NAME: lab
      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: 192.168.100.14:9092
      KAFKA_CLUSTERS_0_ZOOKEEPER: "192.168.100.14:2181" #;192.168.100.14:12181;192.168.100.14:22181"
    networks:
      - mynet

  kafdrop-1:
    image: obsidiandynamics/kafdrop:3.27.0
    hostname: kafrop-1
    container_name: kafdrop-1
    depends_on:
      - broker-1
    ports:
      - "8085:9000"
    environment:
      KAFKA_BROKERCONNECT: 192.168.100.14:9092
      JVM_OPTS: "-Xms32M -Xmx64M"
      SERVER_SERVLET_CONTEXTPATH: "/"
    networks:
      - mynet

networks:
  mynet:
    driver: bridge

mvn quarkus:dev

####
# This Dockerfile is used in order to build a container that runs the Quarkus application in JVM mode
#
# Before building the container image run:
#
# mvn package
#
# Then, build the image with:
#
# docker build -f src/main/docker/Dockerfile.jvm -t connector .
#
# Then run the container using:
#
# docker run -i --rm -p 8080:8080 connector
#
# If you want to include the debug port into your docker image
# you will have to expose the debug port (default 5005) like this :  EXPOSE 8080 5050
#
# Then run the container using :
#
# docker run -i --rm -p 8080:8080 -p 5005:5005 -e JAVA_ENABLE_DEBUG="true" connector
#
###
FROM docker.internal/library/quarkus-base:latest

ARG RUN_JAVA_VERSION=1.3.8
ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en'

USER root
RUN apk update && apk add libstdc++

# Configure the JAVA_OPTIONS, you can add -XshowSettings:vm to also display the heap size.
ENV JAVA_OPTIONS="-Dquarkus.http.host=0.0.0.0 -Djava.util.logging.manager=org.jboss.logmanager.LogManager"

#ENV QUARKUS_LAUNCH_DEVMODE=true \
#    JAVA_ENABLE_DEBUG=true
# -Dquarkus.package.type=mutable-jar

# We make four distinct layers so if there are application changes the library layers can be re-used
COPY --chown=1001 target/quarkus-app/lib/ ${APP_HOME}/lib/
COPY --chown=1001 target/quarkus-app/*-run.jar ${APP_HOME}/app.jar
COPY --chown=1001 target/quarkus-app/app/ ${APP_HOME}/app/
COPY --chown=1001 target/quarkus-app/quarkus/ ${APP_HOME}/quarkus/

EXPOSE 8080
USER 1001

#ENTRYPOINT [ "/deployments/run-java.sh" ]

version: '2'

services:
  zookeeper:
    container_name: zookeeper
    image: confluentinc/cp-zookeeper
    ports:
      - "2181:2181"
      - "2888:2888"
      - "3888:3888"
    environment:
      - ZOOKEEPER_CLIENT_PORT=2181
      - ZOOKEEPER_TICK_TIME=2000
    networks:
      - kafkastreams-network

  kafka:
    container_name: kafka
    image: confluentinc/cp-kafka
    ports:
      - "9092:9092"
    depends_on:
      - zookeeper
    environment:
      - KAFKA_BROKER_ID=1
      - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
      - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      - KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT
      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
      - KAFKA_AUTO_CREATE_TOPICS_ENABLE=true
      - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
      - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR=1
      - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1
      - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=100
    networks:
      - kafkastreams-network

  connect:
    container_name: connect
    image: debezium/connect
    ports:
      - "8083:8083"
    depends_on:
      - kafka
    environment:
      - BOOTSTRAP_SERVERS=kafka:29092
      - GROUP_ID=1
      - CONFIG_STORAGE_TOPIC=my_connect_configs
      - OFFSET_STORAGE_TOPIC=my_connect_offsets
      - STATUS_STORAGE_TOPIC=my_connect_statuses
    networks:
      - kafkastreams-network

  schema-registry:
    image: confluentinc/cp-schema-registry:5.5.0
    container_name: schema-registry
    ports:
      - "8081:8081"
    depends_on:
      - zookeeper
      - kafka
    environment:
      SCHEMA_REGISTRY_HOST_NAME: schema-registry
      SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: zookeeper:2181
    networks:
      - kafkastreams-network

  kafdrop:
    image: obsidiandynamics/kafdrop
    container_name: kafdrop
    restart: "no"
    ports:
      - "9001:9000"
    environment:
      KAFKA_BROKERCONNECT: kafka:29092
      JVM_OPTS: "-Xms16M -Xmx48M -Xss180K -XX:-TieredCompilation -XX:+UseStringDeduplication -noverify"
    depends_on:
      - kafka
      - schema-registry
    networks:
      - kafkastreams-network

  connector:
    image: connector
    depends_on:
      - zookeeper
      - kafka
      - connect
    environment:
      QUARKUS_KAFKA_STREAMS_BOOTSTRAP_SERVERS: kafka:9092
    networks:
      - kafkastreams-network

networks:
  kafkastreams-network:
    name: ks

2021-08-05 11:52:35,433 WARN  [org.apa.kaf.cli.NetworkClient] (kafka-admin-client-thread | connector-18d10d7d-b619-4715-a219-2557d70e0479-admin) [AdminClient clientId=connector-18d10d7d-b619-4715-a219-2557d70e0479-admin] Connection to node -1 (kafka/172.21.0.3:9092) could not be established. Broker may not be available.

version: "3.8"
services:
    zookeeper:
      container_name: zookeeper
      image: bitnami/zookeeper:latest
      ports:
        - 2181:2181
      environment:
        - ALLOW_ANONYMOUS_LOGIN=yes
    kafka4userpass: 
      container_name: kafka4userpass
      image: bitnami/kafka:latest
      ports:
        - '9096:9096'
      environment:
        - KAFKA_BROKER_ID=4
        - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181
        - KAFKA_ZOOKEEPER_PROTOCOL:SASL_PLAINTEXT
        - ALLOW_PLAINTEXT_LISTENER=yes
        - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:SASL_PLAINTEXT
        - KAFKA_CFG_LISTENERS=CLIENT://:9092,EXTERNAL://:9096
        - KAFKA_CFG_ADVERTISED_LISTENERS=CLIENT://kafka4userpass:9092,EXTERNAL://localhost:9096
        - KAFKA_INTER_BROKER_LISTENER_NAME=CLIENT
        - KAFKA_CLIENT_USERS=kafkauser
        - KAFKA_CLIENT_PASSWORDS=kafkapassword
      volumes:
        - './test/docker-compose/kafka_jaas.conf:/opt/bitnami/kafka/conf/kafka_jaas.conf:ro'
      depends_on:
        - zookeeper
    kafdrop:
      container_name: kafdrop
      image: obsidiandynamics/kafdrop
      restart: "no"
      ports:
        - "9000:9000"
      environment:
        KAFKA_BROKERCONNECT: "CLIENT://kafka1:9092,CLIENT://kafka2:9092,CLIENT://kafka3:9092,CLIENT://kafka4userpass:9092"
        JVM_OPTS: "-Xms16M -Xmx48M -Xss180K -XX:-TieredCompilation -XX:+UseStringDeduplication -noverify"
      depends_on:
        #- "kafka1"
        #- "kafka2"
        #- "kafka3"
        - "kafka4userpass"

KafkaServer {
    org.apache.kafka.common.security.plain.PlainLoginModule required
    username="kafkauser"
    password="kafkapassword";
};


Client {
    org.apache.zookeeper.server.auth.PlainLoginModule required
    username="kafkauser"
    password="kafkapassword";
};

172.21.0.3:9092/bootstrap: Connect to ipv4#172.21.0.3:9092 failed: Unknown error (after 21502ms in state CONNECT)
Error: 1/1 brokers are down %3|1620652406.633|FAIL|rdkafka#consumer-1| [thrd:172.21.0.3:9092/bootstrap]: 172.21.0.3:9092/bootstrap: Connect to ipv4#172.21.0.3:9092 failed: Unknown error (after 21037ms in state CONNECT, 1 identical error(s) suppressed) 
Error: 172.21.0.3:9092/bootstrap: Connect to ipv4#172.21.0.3:9092 failed: Unknown error (after 21037ms in state CONNECT, 1 identical error(s) suppressed)

Error: kafka:9092/bootstrap: Failed to resolve 'kafka:9092': No such host is known.  (after 6817ms in state CONNECT, 7 identical error(s) suppressed)

version: '3.8'
services:
  zookeeper:
    #image: "debezium/zookeeper:${DEBEZIUM_VERSION}"
    image: "confluentinc/cp-zookeeper:5.5.0"
  #  ports:
  #    - 2181:2181
  #    - 2888:2888
  #    - 3888:3888
    ports:
      - 2181:2181
    environment: 
      ZOOKEEPER_CLIENT_PORT: 2181

  kafka:
    #image: "debezium/kafka:${DEBEZIUM_VERSION}"
    image: "confluentinc/cp-kafka"
    ports:
      - 9092:9092
      #- 29092:29092
    depends_on:
      - zookeeper
    environment:
     - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 
     - KAFKA_BROKERID=1
     - ALLOW_PLAINTEXT_LISTENER="yes"
     - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
     - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:29092,PLAINTEXT_HOST://kafka:9092
     - KAFKA_LISTENERS=PLAINTEXT://kafka:29092,PLAINTEXT_HOST://kafka:9092
     - KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT
     - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1

  connect:
    image: "debezium/connect:${DEBEZIUM_VERSION}"
    ports: 
      - 8083:8083
    depends_on:
      - kafka
      - zookeeper
    environment:
      - BOOTSTRAP_SERVERS=kafka:29092
      - GROUP_ID=1
      - CONFIG_STORAGE_TOPIC=my_connect_configs
      - OFFSET_STORAGE_TOPIC=my_connect_offsets
      - STATUS_STORAGE_TOPIC=my_source_connect_statuses

  kafdrop:
    image: "obsidiandynamics/kafdrop"
    ports:
      - 9000:9000
    depends_on: 
      - connect
    environment: 
      - KAFKA_BROKERCONNECT=kafka:29092

 var config = new ConsumerConfig
            {
                BootstrapServers = "kafka:9092",
                GroupId = "simple-dotnet-consumer",
                AutoOffsetReset = AutoOffsetReset.Earliest,
                EnablePartitionEof = true
            };
            using var consumer = new ConsumerBuilder<string, string>(config)
                .SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}"))
                .Build();

            consumer.Subscribe(new List<string>() { "DESKTOP-DBA3LAO.dbo.CashRegister" });
            var start = DateTime.Now;
            long messageCounter = 0;
            try
            {
                while (!(Console.KeyAvailable && Console.ReadKey().Key == ConsoleKey.Q))
                {
                    var result = consumer.Consume(TimeSpan.FromMilliseconds(100));
                    if (result == null) { continue; }
                    if (result.IsPartitionEOF) { break; }

                    ++messageCounter;
                    if (messageCounter % 1024 == 0) { Console.WriteLine($"Received message key: \"{result.Message.Key}\" value: {result.Message.Value}"); }
                }
            }
            catch (OperationCanceledException) { }

            consumer.Close();  // commit offset and unsubscribe

            var elapsed = DateTime.Now - start;
            Console.WriteLine("average throughput: {0:N3} msg/sec, {1} messages over {2:N3} sec", messageCounter / elapsed.TotalSeconds, messageCounter, elapsed.TotalSeconds);
        

docker ps -a | awk -F \t '{print $1","$2","$3","$4","$5","$6","$7","$8}'

CONTAINER ID        IMAGE                              COMMAND                  CREATED             STATUS                      PORTS                          NAMES
35fe9af4efa0        thomsch98/kafdrop:latest       "/usr/local/bin/mvn-…"      3 days ago          Exited (137) 3 days ago    PAServices_kafdrop.1       x2r2tuosozdd9uzfnq7ejdi70

CONTAINER ID,IMAGE,COMMAND,CREATED,STATUS,PORTS,NAMES
8c0e092b6815,thomsch98/kafdrop:latest,"/usr/local/bin/mvn-…",3 days ago,Up 3 days,PAServices_kafdrop.1,yen4hgju18kkfgq9bvud7e1w8

ARG BASE_PREFIX=confluentinc 
ARG CONNECT_IMAGE=cp-kafka-connect

FROM $BASE_PREFIX/$CONNECT_IMAGE:6.1.0

ENV CONNECT_PLUGIN_PATH="/usr/share/java,/usr/share/confluent-hub-components"

RUN confluent-hub install --no-prompt confluentinc/kafka-connect-datagen:0.4.0

docker exec 51e32e20b292 bash -c 'echo $CONNECT_PLUGIN_PATH'                                           
shows : /usr/share/java,/usr/share/confluent-hub-components

[appuser@88db8385b575 ~]$ ls -la /usr/share/confluent-hub-components/
total 20
drwxr-xr-x 1 appuser appuser 4096 Mar 26 21:19 .
drwxr-xr-x 1 root    root    4096 Feb  4 21:10 ..
drwxr-xr-x 6 appuser appuser 4096 Mar 26 18:00 confluentinc-kafka-connect-datagen

[appuser@88db8385b575 ~]$ ls -la /usr/share/confluent-hub-components/confluentinc-kafka-connect-datagen/
total 28
drwxr-xr-x 6 appuser appuser 4096 Mar 26 18:00 .
drwxr-xr-x 1 appuser appuser 4096 Mar 26 21:19 ..
drwxr-xr-x 2 appuser appuser 4096 Mar 26 18:00 assets
drwxr-xr-x 4 appuser appuser 4096 Mar 26 18:00 doc
drwxr-xr-x 2 appuser appuser 4096 Mar 26 18:00 etc
drwxr-xr-x 2 appuser appuser 4096 Mar 26 18:00 lib
-rw-r--r-- 1 appuser appuser 1380 Mar 26 18:00 manifest.json

docker logs 51e32e20b292 | grep "DatagenConnector"  
        
    "connector.class": "io.confluent.kafka.connect.datagen.DatagenConnector",
    "connector.class": "io.confluent.kafka.connect.datagen.DatagenConnector",

docker run -d --rm -p 9000:9000 \
    -e KAFKA_BROKERCONNECT=<localhost:9092> \
    -e JVM_OPTS="-Xms32M -Xmx64M" \
    -e SERVER_SERVLET_CONTEXTPATH="/" \
    obsidiandynamics/kafdrop

[AdminClient clientId=kafdrop-admin] Connection to node -1 (localhost/127.0.0.1:9092) could not be established. Broker may not be available.
2020-07-22 09:39:29.108  WARN 1 [| kafdrop-admin] o.a.k.c.NetworkClient 

version: '2'

networks:
  kafka-net:
    driver: bridge

services:
  zookeeper-server:
    image: 'bitnami/zookeeper:latest'
    networks:
      - kafka-net
    ports:
      - '2181:2181'
    environment:
      - ALLOW_ANONYMOUS_LOGIN=yes
  kafdrop:
    image: obsidiandynamics/kafdrop
    networks:
      - kafka-net
    restart: "no"
    ports:
      - "9000:9000"
    environment:
      KAFKA_BROKERCONNECT: "PLAINTEXT://localhost:9092,PLAINTEXT://localhost:9093,PLAINTEXT://localhost:9094"
      JVM_OPTS: "-Xms16M -Xmx48M -Xss180K -XX:-TieredCompilation -XX:+UseStringDeduplication -noverify"
    depends_on:
      - "kafka-server1"
      - "kafka-server2"
      - "kafka-server3"
  kafka-server1:
    image: 'bitnami/kafka:latest'
    networks:
      - kafka-net    
    ports:
      - '9092:9092'
    environment:
      - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper-server:2181
      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092
      - ALLOW_PLAINTEXT_LISTENER=yes
    depends_on:
      - zookeeper-server
  kafka-server2:
    image: 'bitnami/kafka:latest'
    networks:
      - kafka-net    
    ports:
      - '9093:9092'
    environment:
      - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper-server:2181
      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9093
      - ALLOW_PLAINTEXT_LISTENER=yes
    depends_on:
      - zookeeper-server
  kafka-server3:
    image: 'bitnami/kafka:latest'
    networks:
      - kafka-net    
    ports:
      - '9094:9092'
    environment:
      - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper-server:2181
      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9094
      - ALLOW_PLAINTEXT_LISTENER=yes
    depends_on:
      - zookeeper-server

020-08-26 10:53:53.517  WARN 1 [| kafdrop-admin] o.a.k.c.NetworkClient                    : [AdminClient clientId=kafdrop-admin] Connection to node -3 (localhost/127.0.0.1:9094) could not be established. Broker may not be available.
2020-08-26 10:53:53.522  WARN 1 [| kafdrop-admin] o.a.k.c.NetworkClient                    : [AdminClient clientId=kafdrop-admin] Connection to node -2 (localhost/127.0.0.1:9093) could not be established. Broker may not be available.
2020-08-26 10:53:53.526  WARN 1 [| kafdrop-admin] o.a.k.c.NetworkClient                    : [AdminClient clientId=kafdrop-admin] Connection to node -1 (localhost/127.0.0.1:9092) could not be established. Broker may not be available.
2020-08-26 10:53:53.627  WARN 1 [| kafdrop-admin] o.a.k.c.NetworkClient                    : [AdminClient clientId=kafdro