Cloud API

1appflow = flow.InstalledAppFlow.from_client_secrets_file("client_secrets.json",
2scopes=["https://www.googleapis.com/auth/cloud-platform"])
3
4appflow.run_console()
5
6credentials = appflow.credentials
7
8service = googleapiclient.discovery.build(
9    'cloudresourcemanager', 'v1',  credentials=credentials)
10
11operation1 = service.projects().list().execute()
12
13jason=json.dumps(
14    operation1,
15    sort_keys=True,
16    indent=3)
17data = json.loads(jason)
18
19#Gets the list of projects in a Python List object [Proj]
20
21proj=[]
22for mem in data['projects']:
23    print(mem['projectId'])
24    proj.append(mem['projectId'])
25
26for prj in proj:
27    resource = 'projects/' + prj
28    
29    response1 = service.projects().testIamPermissions(resource=resource, body=None, x__xgafv=None).execute()
30    
31    response2 = service.projects().listOrgPolicies(resource=resource, body=None, x__xgafv=None).execute()
32
33    response3 = service.projects().getIamPolicy(resource=resource, body=None, x__xgafv=None).execute()
34

1Karans-MacBook-Pro:keystore karanalang$ $CONFLUENT_HOME/bin/kafka-console-producer --broker-list localhost:9093 --topic karantest --producer.config $CONFLUENT_HOME/props/client-ssl.properties
2>[2021-11-10 13:15:09,824] ERROR [Producer clientId=console-producer] Connection to node -1 (localhost/127.0.0.1:9093) failed authentication due to: SSL handshake failed (org.apache.kafka.clients.NetworkClient)
3[2021-11-10 13:15:09,826] WARN [Producer clientId=console-producer] Bootstrap broker localhost:9093 (id: -1 rack: null) disconnected (org.apache.kafka.clients.NetworkClient)
4[2021-11-10 13:15:10,018] ERROR [Producer clientId=console-producer] Connection to node -1 (localhost/127.0.0.1:9093) failed authentication due to: SSL handshake failed (org.apache.kafka.clients.NetworkClient)
5[2021-11-10 13:15:10,019] WARN [Producer clientId=console-producer] Bootstrap broker localhost:9093 (id: -1 rack: null) disconnected (org.apache.kafka.clients.NetworkClient)
6[2021-11-10 13:15:10,195] ERROR [Producer clientId=console-producer] Connection to node -1 (localhost/127.0.0.1:9093) failed authentication due to: SSL handshake failed (org.apache.kafka.clients.NetworkClient)
7

1Karans-MacBook-Pro:keystore karanalang$ $CONFLUENT_HOME/bin/kafka-console-producer --broker-list localhost:9093 --topic karantest --producer.config $CONFLUENT_HOME/props/client-ssl.properties
2>[2021-11-10 13:15:09,824] ERROR [Producer clientId=console-producer] Connection to node -1 (localhost/127.0.0.1:9093) failed authentication due to: SSL handshake failed (org.apache.kafka.clients.NetworkClient)
3[2021-11-10 13:15:09,826] WARN [Producer clientId=console-producer] Bootstrap broker localhost:9093 (id: -1 rack: null) disconnected (org.apache.kafka.clients.NetworkClient)
4[2021-11-10 13:15:10,018] ERROR [Producer clientId=console-producer] Connection to node -1 (localhost/127.0.0.1:9093) failed authentication due to: SSL handshake failed (org.apache.kafka.clients.NetworkClient)
5[2021-11-10 13:15:10,019] WARN [Producer clientId=console-producer] Bootstrap broker localhost:9093 (id: -1 rack: null) disconnected (org.apache.kafka.clients.NetworkClient)
6[2021-11-10 13:15:10,195] ERROR [Producer clientId=console-producer] Connection to node -1 (localhost/127.0.0.1:9093) failed authentication due to: SSL handshake failed (org.apache.kafka.clients.NetworkClient)
7Karans-MacBook-Pro:keystore karanalang$ openssl s_client -debug -connect localhost:9093 -tls1
8CONNECTED(00000005)
9write to 0x13d7bdf90 [0x13e01ea03] (118 bytes => 118 (0x76))
100000 - 16 03 01 00 71 01 00 00-6d 03 01 81 e8 00 cd c4   ....q...m.......
110010 - 04 4b 64 86 3e 30 97 32-c3 66 3a 8c ed 05 bf 97   .Kd.>0.2.f:.....
120020 - ff d5 b2 a4 26 fe 99 c0-7f 94 a1 00 00 2e c0 14   ....&...........
130030 - c0 0a 00 39 ff 85 00 88-00 81 00 35 00 84 c0 13   ...9.......5....
14---
150076 - <SPACES/NULS>
16read from 0x13d7bdf90 [0x13e01a803] (5 bytes => 5 (0x5))
170005 - <SPACES/NULS>
184307385836:error:1400410B:SSL routines:CONNECT_CR_SRVR_HELLO:wrong version number:/System/Volumes/Data/SWE/macOS/BuildRoots/e90674e518/Library/Caches/com.apple.xbs/Sources/libressl/libressl-56.60.2/libressl-2.8/ssl/ssl_pkt.c:386:
19---
20no peer certificate available
21---
22No client certificate CA names sent
23---
24SSL handshake has read 5 bytes and written 0 bytes
25---
26New, (NONE), Cipher is (NONE)
27Secure Renegotiation IS NOT supported
28Compression: NONE
29Expansion: NONE
30No ALPN negotiated
31SSL-Session:
32    Protocol  : TLSv1
33    Cipher    : 0000
34    Session-ID: 
35    Session-ID-ctx: 
36    Master-Key: 
37    Start Time: 1636579015
38    Timeout   : 7200 (sec)
39    Verify return code: 0 (ok)
40---
41
42

1Karans-MacBook-Pro:keystore karanalang$ $CONFLUENT_HOME/bin/kafka-console-producer --broker-list localhost:9093 --topic karantest --producer.config $CONFLUENT_HOME/props/client-ssl.properties
2>[2021-11-10 13:15:09,824] ERROR [Producer clientId=console-producer] Connection to node -1 (localhost/127.0.0.1:9093) failed authentication due to: SSL handshake failed (org.apache.kafka.clients.NetworkClient)
3[2021-11-10 13:15:09,826] WARN [Producer clientId=console-producer] Bootstrap broker localhost:9093 (id: -1 rack: null) disconnected (org.apache.kafka.clients.NetworkClient)
4[2021-11-10 13:15:10,018] ERROR [Producer clientId=console-producer] Connection to node -1 (localhost/127.0.0.1:9093) failed authentication due to: SSL handshake failed (org.apache.kafka.clients.NetworkClient)
5[2021-11-10 13:15:10,019] WARN [Producer clientId=console-producer] Bootstrap broker localhost:9093 (id: -1 rack: null) disconnected (org.apache.kafka.clients.NetworkClient)
6[2021-11-10 13:15:10,195] ERROR [Producer clientId=console-producer] Connection to node -1 (localhost/127.0.0.1:9093) failed authentication due to: SSL handshake failed (org.apache.kafka.clients.NetworkClient)
7Karans-MacBook-Pro:keystore karanalang$ openssl s_client -debug -connect localhost:9093 -tls1
8CONNECTED(00000005)
9write to 0x13d7bdf90 [0x13e01ea03] (118 bytes => 118 (0x76))
100000 - 16 03 01 00 71 01 00 00-6d 03 01 81 e8 00 cd c4   ....q...m.......
110010 - 04 4b 64 86 3e 30 97 32-c3 66 3a 8c ed 05 bf 97   .Kd.>0.2.f:.....
120020 - ff d5 b2 a4 26 fe 99 c0-7f 94 a1 00 00 2e c0 14   ....&...........
130030 - c0 0a 00 39 ff 85 00 88-00 81 00 35 00 84 c0 13   ...9.......5....
14---
150076 - <SPACES/NULS>
16read from 0x13d7bdf90 [0x13e01a803] (5 bytes => 5 (0x5))
170005 - <SPACES/NULS>
184307385836:error:1400410B:SSL routines:CONNECT_CR_SRVR_HELLO:wrong version number:/System/Volumes/Data/SWE/macOS/BuildRoots/e90674e518/Library/Caches/com.apple.xbs/Sources/libressl/libressl-56.60.2/libressl-2.8/ssl/ssl_pkt.c:386:
19---
20no peer certificate available
21---
22No client certificate CA names sent
23---
24SSL handshake has read 5 bytes and written 0 bytes
25---
26New, (NONE), Cipher is (NONE)
27Secure Renegotiation IS NOT supported
28Compression: NONE
29Expansion: NONE
30No ALPN negotiated
31SSL-Session:
32    Protocol  : TLSv1
33    Cipher    : 0000
34    Session-ID: 
35    Session-ID-ctx: 
36    Master-Key: 
37    Start Time: 1636579015
38    Timeout   : 7200 (sec)
39    Verify return code: 0 (ok)
40---
41
42# Licensed to the Apache Software Foundation (ASF) under one or more
43# contributor license agreements.  See the NOTICE file distributed with
44# this work for additional information regarding copyright ownership.
45# The ASF licenses this file to You under the Apache License, Version 2.0
46# (the "License"); you may not use this file except in compliance with
47# the License.  You may obtain a copy of the License at
48#
49#    http://www.apache.org/licenses/LICENSE-2.0
50#
51# Unless required by applicable law or agreed to in writing, software
52# distributed under the License is distributed on an "AS IS" BASIS,
53# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
54# See the License for the specific language governing permissions and
55# limitations under the License.
56
57# see kafka.server.KafkaConfig for additional details and defaults
58
59############################# Server Basics #############################
60
61# The id of the broker. This must be set to a unique integer for each broker.
62broker.id=0
63
64############################# Socket Server Settings #############################
65
66# The address the socket server listens on. It will get the value returned from 
67# java.net.InetAddress.getCanonicalHostName() if not configured.
68#   FORMAT:
69#     listeners = listener_name://host_name:port
70#   EXAMPLE:
71#     listeners = PLAINTEXT://your.host.name:9092
72# SSL CHANGE
73listeners=PLAINTEXT://localhost:9092,SSL://localhost:9093
74
75# Hostname and port the broker will advertise to producers and consumers. If not set, 
76# it uses the value for "listeners" if configured.  Otherwise, it will use the value
77# returned from java.net.InetAddress.getCanonicalHostName().
78# SSL CHANGE
79advertised.listeners=PLAINTEXT://localhost:9092,SSL://localhost:9093
80ssl.client.auth=none
81
82# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details
83listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL
84
85# The number of threads that the server uses for receiving requests from the network and sending responses to the network
86num.network.threads=3
87
88# The number of threads that the server uses for processing requests, which may include disk I/O
89num.io.threads=8
90
91# The send buffer (SO_SNDBUF) used by the socket server
92socket.send.buffer.bytes=102400
93
94# The receive buffer (SO_RCVBUF) used by the socket server
95socket.receive.buffer.bytes=102400
96
97# The maximum size of a request that the socket server will accept (protection against OOM)
98socket.request.max.bytes=104857600
99
100
101############################# Log Basics #############################
102
103# A comma separated list of directories under which to store log files
104log.dirs=/tmp/kafka-logs
105
106# The default number of log partitions per topic. More partitions allow greater
107# parallelism for consumption, but this will also result in more files across
108# the brokers.
109num.partitions=1
110
111# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown.
112# This value is recommended to be increased for installations with data dirs located in RAID array.
113num.recovery.threads.per.data.dir=1
114
115############################# Internal Topic Settings  #############################
116# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state"
117# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3.
118offsets.topic.replication.factor=1
119transaction.state.log.replication.factor=1
120transaction.state.log.min.isr=1
121
122############################# Log Flush Policy #############################
123
124# Messages are immediately written to the filesystem but by default we only fsync() to sync
125# the OS cache lazily. The following configurations control the flush of data to disk.
126# There are a few important trade-offs here:
127#    1. Durability: Unflushed data may be lost if you are not using replication.
128#    2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush.
129#    3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks.
130# The settings below allow one to configure the flush policy to flush data after a period of time or
131# every N messages (or both). This can be done globally and overridden on a per-topic basis.
132
133# The number of messages to accept before forcing a flush of data to disk
134#log.flush.interval.messages=10000
135
136# The maximum amount of time a message can sit in a log before we force a flush
137#log.flush.interval.ms=1000
138
139############################# Log Retention Policy #############################
140
141# The following configurations control the disposal of log segments. The policy can
142# be set to delete segments after a period of time, or after a given size has accumulated.
143# A segment will be deleted whenever *either* of these criteria are met. Deletion always happens
144# from the end of the log.
145
146# The minimum age of a log file to be eligible for deletion due to age
147log.retention.hours=168
148
149# A size-based retention policy for logs. Segments are pruned from the log unless the remaining
150# segments drop below log.retention.bytes. Functions independently of log.retention.hours.
151#log.retention.bytes=1073741824
152
153# The maximum size of a log segment file. When this size is reached a new log segment will be created.
154log.segment.bytes=1073741824
155
156# The interval at which log segments are checked to see if they can be deleted according
157# to the retention policies
158log.retention.check.interval.ms=300000
159
160############################# Zookeeper #############################
161
162# Zookeeper connection string (see zookeeper docs for details).
163# This is a comma separated host:port pairs, each corresponding to a zk
164# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
165# You can also append an optional chroot string to the urls to specify the
166# root directory for all kafka znodes.
167zookeeper.connect=localhost:2181
168
169# Timeout in ms for connecting to zookeeper
170zookeeper.connection.timeout.ms=18000
171
172##################### Confluent Metrics Reporter #######################
173# Confluent Control Center and Confluent Auto Data Balancer integration
174#
175# Uncomment the following lines to publish monitoring data for
176# Confluent Control Center and Confluent Auto Data Balancer
177# If you are using a dedicated metrics cluster, also adjust the settings
178# to point to your metrics kakfa cluster.
179#metric.reporters=io.confluent.metrics.reporter.ConfluentMetricsReporter
180#confluent.metrics.reporter.bootstrap.servers=localhost:9092
181#
182# Uncomment the following line if the metrics cluster has a single broker
183#confluent.metrics.reporter.topic.replicas=1
184
185############################# Group Coordinator Settings #############################
186
187# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance.
188# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms.
189# The default value for this is 3 seconds.
190# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing.
191# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup.
192group.initial.rebalance.delay.ms=0
193
194
195############################# Confluent Authorizer Settings  #############################
196
197# Uncomment to enable Confluent Authorizer with support for ACLs, LDAP groups and RBAC
198#authorizer.class.name=io.confluent.kafka.security.authorizer.ConfluentServerAuthorizer
199# Semi-colon separated list of super users in the format <principalType>:<principalName>
200#super.users=
201# Specify a valid Confluent license. By default free-tier license will be used
202#confluent.license=
203# Replication factor for the topic used for licensing. Default is 3.
204confluent.license.topic.replication.factor=1
205
206# Uncomment the following lines and specify values where required to enable CONFLUENT provider for RBAC and centralized ACLs
207# Enable CONFLUENT provider 
208#confluent.authorizer.access.rule.providers=ZK_ACL,CONFLUENT
209# Bootstrap servers for RBAC metadata. Must be provided if this broker is not in the metadata cluster
210#confluent.metadata.bootstrap.servers=PLAINTEXT://127.0.0.1:9092
211# Replication factor for the metadata topic used for authorization. Default is 3.
212confluent.metadata.topic.replication.factor=1
213
214# Replication factor for the topic used for audit logs. Default is 3.
215confluent.security.event.logger.exporter.kafka.topic.replicas=1
216
217# Listeners for metadata server
218#confluent.metadata.server.listeners=http://0.0.0.0:8090
219# Advertised listeners for metadata server
220#confluent.metadata.server.advertised.listeners=http://127.0.0.1:8090
221
222############################# Confluent Data Balancer Settings  #############################
223
224# The Confluent Data Balancer is used to measure the load across the Kafka cluster and move data
225# around as necessary. Comment out this line to disable the Data Balancer.
226confluent.balancer.enable=true
227
228# By default, the Data Balancer will only move data when an empty broker (one with no partitions on it)
229# is added to the cluster or a broker failure is detected. Comment out this line to allow the Data
230# Balancer to balance load across the cluster whenever an imbalance is detected.
231#confluent.balancer.heal.uneven.load.trigger=ANY_UNEVEN_LOAD
232
233# The default time to declare a broker permanently failed is 1 hour (3600000 ms).
234# Uncomment this line to turn off broker failure detection, or adjust the threshold
235# to change the duration before a broker is declared failed.
236#confluent.balancer.heal.broker.failure.threshold.ms=-1
237
238# Edit and uncomment the following line to limit the network bandwidth used by data balancing operations.
239# This value is in bytes/sec/broker. The default is 10MB/sec.
240#confluent.balancer.throttle.bytes.per.second=10485760
241
242# Capacity Limits -- when set to positive values, the Data Balancer will attempt to keep
243# resource usage per-broker below these limits.
244# Edit and uncomment this line to limit the maximum number of replicas per broker. Default is unlimited.
245#confluent.balancer.max.replicas=10000
246
247# Edit and uncomment this line to limit what fraction of the log disk (0-1.0) is used before rebalancing.
248# The default (below) is 85% of the log disk.
249#confluent.balancer.disk.max.load=0.85
250
251# Edit and uncomment these lines to define a maximum network capacity per broker, in bytes per
252# second. The Data Balancer will attempt to ensure that brokers are using less than this amount
253# of network bandwidth when rebalancing.
254# Here, 10MB/s. The default is unlimited capacity.
255#confluent.balancer.network.in.max.bytes.per.second=10485760
256#confluent.balancer.network.out.max.bytes.per.second=10485760
257
258# Edit and uncomment this line to identify specific topics that should not be moved by the data balancer.
259# Removal operations always move topics regardless of this setting.
260#confluent.balancer.exclude.topic.names=
261
262# Edit and uncomment this line to identify topic prefixes that should not be moved by the data balancer.
263# (For example, a "confluent.balancer" prefix will match all of "confluent.balancer.a", "confluent.balancer.b",
264# "confluent.balancer.c", and so on.)
265# Removal operations always move topics regardless of this setting.
266#confluent.balancer.exclude.topic.prefixes=
267
268# The replication factor for the topics the Data Balancer uses to store internal state.
269# For anything other than development testing, a value greater than 1 is recommended to ensure availability.
270# The default value is 3.
271confluent.balancer.topic.replication.factor=1
272
273################################## Confluent Telemetry Settings  ##################################
274
275# To start using Telemetry, first generate a Confluent Cloud API key/secret. This can be done with
276# instructions at https://docs.confluent.io/current/cloud/using/api-keys.html. Note that you should
277# be using the '--resource cloud' flag.
278#
279# After generating an API key/secret, to enable Telemetry uncomment the lines below and paste
280# in your API key/secret.
281#
282#confluent.telemetry.enabled=true
283#confluent.telemetry.api.key=<CLOUD_API_KEY>
284#confluent.telemetry.api.secret=<CCLOUD_API_SECRET>
285
286############ SSL #################
287
288ssl.truststore.location=/Users/karanalang/Documents/Technology/confluent-6.2.1/ssl_certs/truststore/kafka.truststore.jks
289ssl.truststore.password=test123
290ssl.keystore.location=/Users/karanalang/Documents/Technology/confluent-6.2.1/ssl_certs/keystore/kafka.keystore.jks
291ssl.keystore.password=test123
292ssl.key.password=test123
293
294# confluent.metrics.reporter.bootstrap.servers=localhost:9093
295# confluent.metrics.reporter.security.protocol=SSL
296# confluent.metrics.reporter.ssl.truststore.location=/Users/karanalang/Documents/Technology/confluent-6.2.1/ssl_certs/truststore/kafka.truststore.jks
297# confluent.metrics.reporter.ssl.truststore.password=test123
298# confluent.metrics.reporter.ssl.keystore.location=/Users/karanalang/Documents/Technology/confluent-6.2.1/ssl_certs/keystore/kafka.keystore.jks
299# confluent.metrics.reporter.ssl.keystore.password=test123
300# confluent.metrics.reporter.ssl.key.password=test123
301
302

1Karans-MacBook-Pro:keystore karanalang$ $CONFLUENT_HOME/bin/kafka-console-producer --broker-list localhost:9093 --topic karantest --producer.config $CONFLUENT_HOME/props/client-ssl.properties
2>[2021-11-10 13:15:09,824] ERROR [Producer clientId=console-producer] Connection to node -1 (localhost/127.0.0.1:9093) failed authentication due to: SSL handshake failed (org.apache.kafka.clients.NetworkClient)
3[2021-11-10 13:15:09,826] WARN [Producer clientId=console-producer] Bootstrap broker localhost:9093 (id: -1 rack: null) disconnected (org.apache.kafka.clients.NetworkClient)
4[2021-11-10 13:15:10,018] ERROR [Producer clientId=console-producer] Connection to node -1 (localhost/127.0.0.1:9093) failed authentication due to: SSL handshake failed (org.apache.kafka.clients.NetworkClient)
5[2021-11-10 13:15:10,019] WARN [Producer clientId=console-producer] Bootstrap broker localhost:9093 (id: -1 rack: null) disconnected (org.apache.kafka.clients.NetworkClient)
6[2021-11-10 13:15:10,195] ERROR [Producer clientId=console-producer] Connection to node -1 (localhost/127.0.0.1:9093) failed authentication due to: SSL handshake failed (org.apache.kafka.clients.NetworkClient)
7Karans-MacBook-Pro:keystore karanalang$ openssl s_client -debug -connect localhost:9093 -tls1
8CONNECTED(00000005)
9write to 0x13d7bdf90 [0x13e01ea03] (118 bytes => 118 (0x76))
100000 - 16 03 01 00 71 01 00 00-6d 03 01 81 e8 00 cd c4   ....q...m.......
110010 - 04 4b 64 86 3e 30 97 32-c3 66 3a 8c ed 05 bf 97   .Kd.>0.2.f:.....
120020 - ff d5 b2 a4 26 fe 99 c0-7f 94 a1 00 00 2e c0 14   ....&...........
130030 - c0 0a 00 39 ff 85 00 88-00 81 00 35 00 84 c0 13   ...9.......5....
14---
150076 - <SPACES/NULS>
16read from 0x13d7bdf90 [0x13e01a803] (5 bytes => 5 (0x5))
170005 - <SPACES/NULS>
184307385836:error:1400410B:SSL routines:CONNECT_CR_SRVR_HELLO:wrong version number:/System/Volumes/Data/SWE/macOS/BuildRoots/e90674e518/Library/Caches/com.apple.xbs/Sources/libressl/libressl-56.60.2/libressl-2.8/ssl/ssl_pkt.c:386:
19---
20no peer certificate available
21---
22No client certificate CA names sent
23---
24SSL handshake has read 5 bytes and written 0 bytes
25---
26New, (NONE), Cipher is (NONE)
27Secure Renegotiation IS NOT supported
28Compression: NONE
29Expansion: NONE
30No ALPN negotiated
31SSL-Session:
32    Protocol  : TLSv1
33    Cipher    : 0000
34    Session-ID: 
35    Session-ID-ctx: 
36    Master-Key: 
37    Start Time: 1636579015
38    Timeout   : 7200 (sec)
39    Verify return code: 0 (ok)
40---
41
42# Licensed to the Apache Software Foundation (ASF) under one or more
43# contributor license agreements.  See the NOTICE file distributed with
44# this work for additional information regarding copyright ownership.
45# The ASF licenses this file to You under the Apache License, Version 2.0
46# (the "License"); you may not use this file except in compliance with
47# the License.  You may obtain a copy of the License at
48#
49#    http://www.apache.org/licenses/LICENSE-2.0
50#
51# Unless required by applicable law or agreed to in writing, software
52# distributed under the License is distributed on an "AS IS" BASIS,
53# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
54# See the License for the specific language governing permissions and
55# limitations under the License.
56
57# see kafka.server.KafkaConfig for additional details and defaults
58
59############################# Server Basics #############################
60
61# The id of the broker. This must be set to a unique integer for each broker.
62broker.id=0
63
64############################# Socket Server Settings #############################
65
66# The address the socket server listens on. It will get the value returned from 
67# java.net.InetAddress.getCanonicalHostName() if not configured.
68#   FORMAT:
69#     listeners = listener_name://host_name:port
70#   EXAMPLE:
71#     listeners = PLAINTEXT://your.host.name:9092
72# SSL CHANGE
73listeners=PLAINTEXT://localhost:9092,SSL://localhost:9093
74
75# Hostname and port the broker will advertise to producers and consumers. If not set, 
76# it uses the value for "listeners" if configured.  Otherwise, it will use the value
77# returned from java.net.InetAddress.getCanonicalHostName().
78# SSL CHANGE
79advertised.listeners=PLAINTEXT://localhost:9092,SSL://localhost:9093
80ssl.client.auth=none
81
82# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details
83listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL
84
85# The number of threads that the server uses for receiving requests from the network and sending responses to the network
86num.network.threads=3
87
88# The number of threads that the server uses for processing requests, which may include disk I/O
89num.io.threads=8
90
91# The send buffer (SO_SNDBUF) used by the socket server
92socket.send.buffer.bytes=102400
93
94# The receive buffer (SO_RCVBUF) used by the socket server
95socket.receive.buffer.bytes=102400
96
97# The maximum size of a request that the socket server will accept (protection against OOM)
98socket.request.max.bytes=104857600
99
100
101############################# Log Basics #############################
102
103# A comma separated list of directories under which to store log files
104log.dirs=/tmp/kafka-logs
105
106# The default number of log partitions per topic. More partitions allow greater
107# parallelism for consumption, but this will also result in more files across
108# the brokers.
109num.partitions=1
110
111# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown.
112# This value is recommended to be increased for installations with data dirs located in RAID array.
113num.recovery.threads.per.data.dir=1
114
115############################# Internal Topic Settings  #############################
116# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state"
117# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3.
118offsets.topic.replication.factor=1
119transaction.state.log.replication.factor=1
120transaction.state.log.min.isr=1
121
122############################# Log Flush Policy #############################
123
124# Messages are immediately written to the filesystem but by default we only fsync() to sync
125# the OS cache lazily. The following configurations control the flush of data to disk.
126# There are a few important trade-offs here:
127#    1. Durability: Unflushed data may be lost if you are not using replication.
128#    2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush.
129#    3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks.
130# The settings below allow one to configure the flush policy to flush data after a period of time or
131# every N messages (or both). This can be done globally and overridden on a per-topic basis.
132
133# The number of messages to accept before forcing a flush of data to disk
134#log.flush.interval.messages=10000
135
136# The maximum amount of time a message can sit in a log before we force a flush
137#log.flush.interval.ms=1000
138
139############################# Log Retention Policy #############################
140
141# The following configurations control the disposal of log segments. The policy can
142# be set to delete segments after a period of time, or after a given size has accumulated.
143# A segment will be deleted whenever *either* of these criteria are met. Deletion always happens
144# from the end of the log.
145
146# The minimum age of a log file to be eligible for deletion due to age
147log.retention.hours=168
148
149# A size-based retention policy for logs. Segments are pruned from the log unless the remaining
150# segments drop below log.retention.bytes. Functions independently of log.retention.hours.
151#log.retention.bytes=1073741824
152
153# The maximum size of a log segment file. When this size is reached a new log segment will be created.
154log.segment.bytes=1073741824
155
156# The interval at which log segments are checked to see if they can be deleted according
157# to the retention policies
158log.retention.check.interval.ms=300000
159
160############################# Zookeeper #############################
161
162# Zookeeper connection string (see zookeeper docs for details).
163# This is a comma separated host:port pairs, each corresponding to a zk
164# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
165# You can also append an optional chroot string to the urls to specify the
166# root directory for all kafka znodes.
167zookeeper.connect=localhost:2181
168
169# Timeout in ms for connecting to zookeeper
170zookeeper.connection.timeout.ms=18000
171
172##################### Confluent Metrics Reporter #######################
173# Confluent Control Center and Confluent Auto Data Balancer integration
174#
175# Uncomment the following lines to publish monitoring data for
176# Confluent Control Center and Confluent Auto Data Balancer
177# If you are using a dedicated metrics cluster, also adjust the settings
178# to point to your metrics kakfa cluster.
179#metric.reporters=io.confluent.metrics.reporter.ConfluentMetricsReporter
180#confluent.metrics.reporter.bootstrap.servers=localhost:9092
181#
182# Uncomment the following line if the metrics cluster has a single broker
183#confluent.metrics.reporter.topic.replicas=1
184
185############################# Group Coordinator Settings #############################
186
187# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance.
188# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms.
189# The default value for this is 3 seconds.
190# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing.
191# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup.
192group.initial.rebalance.delay.ms=0
193
194
195############################# Confluent Authorizer Settings  #############################
196
197# Uncomment to enable Confluent Authorizer with support for ACLs, LDAP groups and RBAC
198#authorizer.class.name=io.confluent.kafka.security.authorizer.ConfluentServerAuthorizer
199# Semi-colon separated list of super users in the format <principalType>:<principalName>
200#super.users=
201# Specify a valid Confluent license. By default free-tier license will be used
202#confluent.license=
203# Replication factor for the topic used for licensing. Default is 3.
204confluent.license.topic.replication.factor=1
205
206# Uncomment the following lines and specify values where required to enable CONFLUENT provider for RBAC and centralized ACLs
207# Enable CONFLUENT provider 
208#confluent.authorizer.access.rule.providers=ZK_ACL,CONFLUENT
209# Bootstrap servers for RBAC metadata. Must be provided if this broker is not in the metadata cluster
210#confluent.metadata.bootstrap.servers=PLAINTEXT://127.0.0.1:9092
211# Replication factor for the metadata topic used for authorization. Default is 3.
212confluent.metadata.topic.replication.factor=1
213
214# Replication factor for the topic used for audit logs. Default is 3.
215confluent.security.event.logger.exporter.kafka.topic.replicas=1
216
217# Listeners for metadata server
218#confluent.metadata.server.listeners=http://0.0.0.0:8090
219# Advertised listeners for metadata server
220#confluent.metadata.server.advertised.listeners=http://127.0.0.1:8090
221
222############################# Confluent Data Balancer Settings  #############################
223
224# The Confluent Data Balancer is used to measure the load across the Kafka cluster and move data
225# around as necessary. Comment out this line to disable the Data Balancer.
226confluent.balancer.enable=true
227
228# By default, the Data Balancer will only move data when an empty broker (one with no partitions on it)
229# is added to the cluster or a broker failure is detected. Comment out this line to allow the Data
230# Balancer to balance load across the cluster whenever an imbalance is detected.
231#confluent.balancer.heal.uneven.load.trigger=ANY_UNEVEN_LOAD
232
233# The default time to declare a broker permanently failed is 1 hour (3600000 ms).
234# Uncomment this line to turn off broker failure detection, or adjust the threshold
235# to change the duration before a broker is declared failed.
236#confluent.balancer.heal.broker.failure.threshold.ms=-1
237
238# Edit and uncomment the following line to limit the network bandwidth used by data balancing operations.
239# This value is in bytes/sec/broker. The default is 10MB/sec.
240#confluent.balancer.throttle.bytes.per.second=10485760
241
242# Capacity Limits -- when set to positive values, the Data Balancer will attempt to keep
243# resource usage per-broker below these limits.
244# Edit and uncomment this line to limit the maximum number of replicas per broker. Default is unlimited.
245#confluent.balancer.max.replicas=10000
246
247# Edit and uncomment this line to limit what fraction of the log disk (0-1.0) is used before rebalancing.
248# The default (below) is 85% of the log disk.
249#confluent.balancer.disk.max.load=0.85
250
251# Edit and uncomment these lines to define a maximum network capacity per broker, in bytes per
252# second. The Data Balancer will attempt to ensure that brokers are using less than this amount
253# of network bandwidth when rebalancing.
254# Here, 10MB/s. The default is unlimited capacity.
255#confluent.balancer.network.in.max.bytes.per.second=10485760
256#confluent.balancer.network.out.max.bytes.per.second=10485760
257
258# Edit and uncomment this line to identify specific topics that should not be moved by the data balancer.
259# Removal operations always move topics regardless of this setting.
260#confluent.balancer.exclude.topic.names=
261
262# Edit and uncomment this line to identify topic prefixes that should not be moved by the data balancer.
263# (For example, a "confluent.balancer" prefix will match all of "confluent.balancer.a", "confluent.balancer.b",
264# "confluent.balancer.c", and so on.)
265# Removal operations always move topics regardless of this setting.
266#confluent.balancer.exclude.topic.prefixes=
267
268# The replication factor for the topics the Data Balancer uses to store internal state.
269# For anything other than development testing, a value greater than 1 is recommended to ensure availability.
270# The default value is 3.
271confluent.balancer.topic.replication.factor=1
272
273################################## Confluent Telemetry Settings  ##################################
274
275# To start using Telemetry, first generate a Confluent Cloud API key/secret. This can be done with
276# instructions at https://docs.confluent.io/current/cloud/using/api-keys.html. Note that you should
277# be using the '--resource cloud' flag.
278#
279# After generating an API key/secret, to enable Telemetry uncomment the lines below and paste
280# in your API key/secret.
281#
282#confluent.telemetry.enabled=true
283#confluent.telemetry.api.key=<CLOUD_API_KEY>
284#confluent.telemetry.api.secret=<CCLOUD_API_SECRET>
285
286############ SSL #################
287
288ssl.truststore.location=/Users/karanalang/Documents/Technology/confluent-6.2.1/ssl_certs/truststore/kafka.truststore.jks
289ssl.truststore.password=test123
290ssl.keystore.location=/Users/karanalang/Documents/Technology/confluent-6.2.1/ssl_certs/keystore/kafka.keystore.jks
291ssl.keystore.password=test123
292ssl.key.password=test123
293
294# confluent.metrics.reporter.bootstrap.servers=localhost:9093
295# confluent.metrics.reporter.security.protocol=SSL
296# confluent.metrics.reporter.ssl.truststore.location=/Users/karanalang/Documents/Technology/confluent-6.2.1/ssl_certs/truststore/kafka.truststore.jks
297# confluent.metrics.reporter.ssl.truststore.password=test123
298# confluent.metrics.reporter.ssl.keystore.location=/Users/karanalang/Documents/Technology/confluent-6.2.1/ssl_certs/keystore/kafka.keystore.jks
299# confluent.metrics.reporter.ssl.keystore.password=test123
300# confluent.metrics.reporter.ssl.key.password=test123
301
302bootstrap.servers=localhost:9093
303security.protocol=SSL
304ssl.truststore.location=/Users/karanalang/Documents/Technology/confluent-6.2.1/ssl_certs/truststore/kafka.truststore.jks
305ssl.truststore.password=test123
306ssl.keystore.location=/Users/karanalang/Documents/Technology/confluent-6.2.1/ssl_certs/keystore/kafka.keystore.jks
307ssl.keystore.password=test123
308ssl.key.password=test123
309

1Karans-MacBook-Pro:keystore karanalang$ $CONFLUENT_HOME/bin/kafka-console-producer --broker-list localhost:9093 --topic karantest --producer.config $CONFLUENT_HOME/props/client-ssl.properties
2>[2021-11-10 13:15:09,824] ERROR [Producer clientId=console-producer] Connection to node -1 (localhost/127.0.0.1:9093) failed authentication due to: SSL handshake failed (org.apache.kafka.clients.NetworkClient)
3[2021-11-10 13:15:09,826] WARN [Producer clientId=console-producer] Bootstrap broker localhost:9093 (id: -1 rack: null) disconnected (org.apache.kafka.clients.NetworkClient)
4[2021-11-10 13:15:10,018] ERROR [Producer clientId=console-producer] Connection to node -1 (localhost/127.0.0.1:9093) failed authentication due to: SSL handshake failed (org.apache.kafka.clients.NetworkClient)
5[2021-11-10 13:15:10,019] WARN [Producer clientId=console-producer] Bootstrap broker localhost:9093 (id: -1 rack: null) disconnected (org.apache.kafka.clients.NetworkClient)
6[2021-11-10 13:15:10,195] ERROR [Producer clientId=console-producer] Connection to node -1 (localhost/127.0.0.1:9093) failed authentication due to: SSL handshake failed (org.apache.kafka.clients.NetworkClient)
7Karans-MacBook-Pro:keystore karanalang$ openssl s_client -debug -connect localhost:9093 -tls1
8CONNECTED(00000005)
9write to 0x13d7bdf90 [0x13e01ea03] (118 bytes => 118 (0x76))
100000 - 16 03 01 00 71 01 00 00-6d 03 01 81 e8 00 cd c4   ....q...m.......
110010 - 04 4b 64 86 3e 30 97 32-c3 66 3a 8c ed 05 bf 97   .Kd.>0.2.f:.....
120020 - ff d5 b2 a4 26 fe 99 c0-7f 94 a1 00 00 2e c0 14   ....&...........
130030 - c0 0a 00 39 ff 85 00 88-00 81 00 35 00 84 c0 13   ...9.......5....
14---
150076 - <SPACES/NULS>
16read from 0x13d7bdf90 [0x13e01a803] (5 bytes => 5 (0x5))
170005 - <SPACES/NULS>
184307385836:error:1400410B:SSL routines:CONNECT_CR_SRVR_HELLO:wrong version number:/System/Volumes/Data/SWE/macOS/BuildRoots/e90674e518/Library/Caches/com.apple.xbs/Sources/libressl/libressl-56.60.2/libressl-2.8/ssl/ssl_pkt.c:386:
19---
20no peer certificate available
21---
22No client certificate CA names sent
23---
24SSL handshake has read 5 bytes and written 0 bytes
25---
26New, (NONE), Cipher is (NONE)
27Secure Renegotiation IS NOT supported
28Compression: NONE
29Expansion: NONE
30No ALPN negotiated
31SSL-Session:
32    Protocol  : TLSv1
33    Cipher    : 0000
34    Session-ID: 
35    Session-ID-ctx: 
36    Master-Key: 
37    Start Time: 1636579015
38    Timeout   : 7200 (sec)
39    Verify return code: 0 (ok)
40---
41
42# Licensed to the Apache Software Foundation (ASF) under one or more
43# contributor license agreements.  See the NOTICE file distributed with
44# this work for additional information regarding copyright ownership.
45# The ASF licenses this file to You under the Apache License, Version 2.0
46# (the "License"); you may not use this file except in compliance with
47# the License.  You may obtain a copy of the License at
48#
49#    http://www.apache.org/licenses/LICENSE-2.0
50#
51# Unless required by applicable law or agreed to in writing, software
52# distributed under the License is distributed on an "AS IS" BASIS,
53# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
54# See the License for the specific language governing permissions and
55# limitations under the License.
56
57# see kafka.server.KafkaConfig for additional details and defaults
58
59############################# Server Basics #############################
60
61# The id of the broker. This must be set to a unique integer for each broker.
62broker.id=0
63
64############################# Socket Server Settings #############################
65
66# The address the socket server listens on. It will get the value returned from 
67# java.net.InetAddress.getCanonicalHostName() if not configured.
68#   FORMAT:
69#     listeners = listener_name://host_name:port
70#   EXAMPLE:
71#     listeners = PLAINTEXT://your.host.name:9092
72# SSL CHANGE
73listeners=PLAINTEXT://localhost:9092,SSL://localhost:9093
74
75# Hostname and port the broker will advertise to producers and consumers. If not set, 
76# it uses the value for "listeners" if configured.  Otherwise, it will use the value
77# returned from java.net.InetAddress.getCanonicalHostName().
78# SSL CHANGE
79advertised.listeners=PLAINTEXT://localhost:9092,SSL://localhost:9093
80ssl.client.auth=none
81
82# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details
83listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL
84
85# The number of threads that the server uses for receiving requests from the network and sending responses to the network
86num.network.threads=3
87
88# The number of threads that the server uses for processing requests, which may include disk I/O
89num.io.threads=8
90
91# The send buffer (SO_SNDBUF) used by the socket server
92socket.send.buffer.bytes=102400
93
94# The receive buffer (SO_RCVBUF) used by the socket server
95socket.receive.buffer.bytes=102400
96
97# The maximum size of a request that the socket server will accept (protection against OOM)
98socket.request.max.bytes=104857600
99
100
101############################# Log Basics #############################
102
103# A comma separated list of directories under which to store log files
104log.dirs=/tmp/kafka-logs
105
106# The default number of log partitions per topic. More partitions allow greater
107# parallelism for consumption, but this will also result in more files across
108# the brokers.
109num.partitions=1
110
111# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown.
112# This value is recommended to be increased for installations with data dirs located in RAID array.
113num.recovery.threads.per.data.dir=1
114
115############################# Internal Topic Settings  #############################
116# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state"
117# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3.
118offsets.topic.replication.factor=1
119transaction.state.log.replication.factor=1
120transaction.state.log.min.isr=1
121
122############################# Log Flush Policy #############################
123
124# Messages are immediately written to the filesystem but by default we only fsync() to sync
125# the OS cache lazily. The following configurations control the flush of data to disk.
126# There are a few important trade-offs here:
127#    1. Durability: Unflushed data may be lost if you are not using replication.
128#    2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush.
129#    3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks.
130# The settings below allow one to configure the flush policy to flush data after a period of time or
131# every N messages (or both). This can be done globally and overridden on a per-topic basis.
132
133# The number of messages to accept before forcing a flush of data to disk
134#log.flush.interval.messages=10000
135
136# The maximum amount of time a message can sit in a log before we force a flush
137#log.flush.interval.ms=1000
138
139############################# Log Retention Policy #############################
140
141# The following configurations control the disposal of log segments. The policy can
142# be set to delete segments after a period of time, or after a given size has accumulated.
143# A segment will be deleted whenever *either* of these criteria are met. Deletion always happens
144# from the end of the log.
145
146# The minimum age of a log file to be eligible for deletion due to age
147log.retention.hours=168
148
149# A size-based retention policy for logs. Segments are pruned from the log unless the remaining
150# segments drop below log.retention.bytes. Functions independently of log.retention.hours.
151#log.retention.bytes=1073741824
152
153# The maximum size of a log segment file. When this size is reached a new log segment will be created.
154log.segment.bytes=1073741824
155
156# The interval at which log segments are checked to see if they can be deleted according
157# to the retention policies
158log.retention.check.interval.ms=300000
159
160############################# Zookeeper #############################
161
162# Zookeeper connection string (see zookeeper docs for details).
163# This is a comma separated host:port pairs, each corresponding to a zk
164# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
165# You can also append an optional chroot string to the urls to specify the
166# root directory for all kafka znodes.
167zookeeper.connect=localhost:2181
168
169# Timeout in ms for connecting to zookeeper
170zookeeper.connection.timeout.ms=18000
171
172##################### Confluent Metrics Reporter #######################
173# Confluent Control Center and Confluent Auto Data Balancer integration
174#
175# Uncomment the following lines to publish monitoring data for
176# Confluent Control Center and Confluent Auto Data Balancer
177# If you are using a dedicated metrics cluster, also adjust the settings
178# to point to your metrics kakfa cluster.
179#metric.reporters=io.confluent.metrics.reporter.ConfluentMetricsReporter
180#confluent.metrics.reporter.bootstrap.servers=localhost:9092
181#
182# Uncomment the following line if the metrics cluster has a single broker
183#confluent.metrics.reporter.topic.replicas=1
184
185############################# Group Coordinator Settings #############################
186
187# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance.
188# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms.
189# The default value for this is 3 seconds.
190# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing.
191# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup.
192group.initial.rebalance.delay.ms=0
193
194
195############################# Confluent Authorizer Settings  #############################
196
197# Uncomment to enable Confluent Authorizer with support for ACLs, LDAP groups and RBAC
198#authorizer.class.name=io.confluent.kafka.security.authorizer.ConfluentServerAuthorizer
199# Semi-colon separated list of super users in the format <principalType>:<principalName>
200#super.users=
201# Specify a valid Confluent license. By default free-tier license will be used
202#confluent.license=
203# Replication factor for the topic used for licensing. Default is 3.
204confluent.license.topic.replication.factor=1
205
206# Uncomment the following lines and specify values where required to enable CONFLUENT provider for RBAC and centralized ACLs
207# Enable CONFLUENT provider 
208#confluent.authorizer.access.rule.providers=ZK_ACL,CONFLUENT
209# Bootstrap servers for RBAC metadata. Must be provided if this broker is not in the metadata cluster
210#confluent.metadata.bootstrap.servers=PLAINTEXT://127.0.0.1:9092
211# Replication factor for the metadata topic used for authorization. Default is 3.
212confluent.metadata.topic.replication.factor=1
213
214# Replication factor for the topic used for audit logs. Default is 3.
215confluent.security.event.logger.exporter.kafka.topic.replicas=1
216
217# Listeners for metadata server
218#confluent.metadata.server.listeners=http://0.0.0.0:8090
219# Advertised listeners for metadata server
220#confluent.metadata.server.advertised.listeners=http://127.0.0.1:8090
221
222############################# Confluent Data Balancer Settings  #############################
223
224# The Confluent Data Balancer is used to measure the load across the Kafka cluster and move data
225# around as necessary. Comment out this line to disable the Data Balancer.
226confluent.balancer.enable=true
227
228# By default, the Data Balancer will only move data when an empty broker (one with no partitions on it)
229# is added to the cluster or a broker failure is detected. Comment out this line to allow the Data
230# Balancer to balance load across the cluster whenever an imbalance is detected.
231#confluent.balancer.heal.uneven.load.trigger=ANY_UNEVEN_LOAD
232
233# The default time to declare a broker permanently failed is 1 hour (3600000 ms).
234# Uncomment this line to turn off broker failure detection, or adjust the threshold
235# to change the duration before a broker is declared failed.
236#confluent.balancer.heal.broker.failure.threshold.ms=-1
237
238# Edit and uncomment the following line to limit the network bandwidth used by data balancing operations.
239# This value is in bytes/sec/broker. The default is 10MB/sec.
240#confluent.balancer.throttle.bytes.per.second=10485760
241
242# Capacity Limits -- when set to positive values, the Data Balancer will attempt to keep
243# resource usage per-broker below these limits.
244# Edit and uncomment this line to limit the maximum number of replicas per broker. Default is unlimited.
245#confluent.balancer.max.replicas=10000
246
247# Edit and uncomment this line to limit what fraction of the log disk (0-1.0) is used before rebalancing.
248# The default (below) is 85% of the log disk.
249#confluent.balancer.disk.max.load=0.85
250
251# Edit and uncomment these lines to define a maximum network capacity per broker, in bytes per
252# second. The Data Balancer will attempt to ensure that brokers are using less than this amount
253# of network bandwidth when rebalancing.
254# Here, 10MB/s. The default is unlimited capacity.
255#confluent.balancer.network.in.max.bytes.per.second=10485760
256#confluent.balancer.network.out.max.bytes.per.second=10485760
257
258# Edit and uncomment this line to identify specific topics that should not be moved by the data balancer.
259# Removal operations always move topics regardless of this setting.
260#confluent.balancer.exclude.topic.names=
261
262# Edit and uncomment this line to identify topic prefixes that should not be moved by the data balancer.
263# (For example, a "confluent.balancer" prefix will match all of "confluent.balancer.a", "confluent.balancer.b",
264# "confluent.balancer.c", and so on.)
265# Removal operations always move topics regardless of this setting.
266#confluent.balancer.exclude.topic.prefixes=
267
268# The replication factor for the topics the Data Balancer uses to store internal state.
269# For anything other than development testing, a value greater than 1 is recommended to ensure availability.
270# The default value is 3.
271confluent.balancer.topic.replication.factor=1
272
273################################## Confluent Telemetry Settings  ##################################
274
275# To start using Telemetry, first generate a Confluent Cloud API key/secret. This can be done with
276# instructions at https://docs.confluent.io/current/cloud/using/api-keys.html. Note that you should
277# be using the '--resource cloud' flag.
278#
279# After generating an API key/secret, to enable Telemetry uncomment the lines below and paste
280# in your API key/secret.
281#
282#confluent.telemetry.enabled=true
283#confluent.telemetry.api.key=<CLOUD_API_KEY>
284#confluent.telemetry.api.secret=<CCLOUD_API_SECRET>
285
286############ SSL #################
287
288ssl.truststore.location=/Users/karanalang/Documents/Technology/confluent-6.2.1/ssl_certs/truststore/kafka.truststore.jks
289ssl.truststore.password=test123
290ssl.keystore.location=/Users/karanalang/Documents/Technology/confluent-6.2.1/ssl_certs/keystore/kafka.keystore.jks
291ssl.keystore.password=test123
292ssl.key.password=test123
293
294# confluent.metrics.reporter.bootstrap.servers=localhost:9093
295# confluent.metrics.reporter.security.protocol=SSL
296# confluent.metrics.reporter.ssl.truststore.location=/Users/karanalang/Documents/Technology/confluent-6.2.1/ssl_certs/truststore/kafka.truststore.jks
297# confluent.metrics.reporter.ssl.truststore.password=test123
298# confluent.metrics.reporter.ssl.keystore.location=/Users/karanalang/Documents/Technology/confluent-6.2.1/ssl_certs/keystore/kafka.keystore.jks
299# confluent.metrics.reporter.ssl.keystore.password=test123
300# confluent.metrics.reporter.ssl.key.password=test123
301
302bootstrap.servers=localhost:9093
303security.protocol=SSL
304ssl.truststore.location=/Users/karanalang/Documents/Technology/confluent-6.2.1/ssl_certs/truststore/kafka.truststore.jks
305ssl.truststore.password=test123
306ssl.keystore.location=/Users/karanalang/Documents/Technology/confluent-6.2.1/ssl_certs/keystore/kafka.keystore.jks
307ssl.keystore.password=test123
308ssl.key.password=test123
309$CONFLUENT_HOME/bin/kafka-console-producer --broker-list localhost:9093 --topic karantest --producer.config $CONFLUENT_HOME/props/client-ssl.properties
310$CONFLUENT_HOME/bin/kafka-console-consumer --bootstrap-server localhost:9093 --topic karantest --consumer.config $CONFLUENT_HOME/props/client-ssl.properties --from-beginning
311

1Karans-MacBook-Pro:keystore karanalang$ $CONFLUENT_HOME/bin/kafka-console-producer --broker-list localhost:9093 --topic karantest --producer.config $CONFLUENT_HOME/props/client-ssl.properties
2>[2021-11-10 13:15:09,824] ERROR [Producer clientId=console-producer] Connection to node -1 (localhost/127.0.0.1:9093) failed authentication due to: SSL handshake failed (org.apache.kafka.clients.NetworkClient)
3[2021-11-10 13:15:09,826] WARN [Producer clientId=console-producer] Bootstrap broker localhost:9093 (id: -1 rack: null) disconnected (org.apache.kafka.clients.NetworkClient)
4[2021-11-10 13:15:10,018] ERROR [Producer clientId=console-producer] Connection to node -1 (localhost/127.0.0.1:9093) failed authentication due to: SSL handshake failed (org.apache.kafka.clients.NetworkClient)
5[2021-11-10 13:15:10,019] WARN [Producer clientId=console-producer] Bootstrap broker localhost:9093 (id: -1 rack: null) disconnected (org.apache.kafka.clients.NetworkClient)
6[2021-11-10 13:15:10,195] ERROR [Producer clientId=console-producer] Connection to node -1 (localhost/127.0.0.1:9093) failed authentication due to: SSL handshake failed (org.apache.kafka.clients.NetworkClient)
7Karans-MacBook-Pro:keystore karanalang$ openssl s_client -debug -connect localhost:9093 -tls1
8CONNECTED(00000005)
9write to 0x13d7bdf90 [0x13e01ea03] (118 bytes => 118 (0x76))
100000 - 16 03 01 00 71 01 00 00-6d 03 01 81 e8 00 cd c4   ....q...m.......
110010 - 04 4b 64 86 3e 30 97 32-c3 66 3a 8c ed 05 bf 97   .Kd.>0.2.f:.....
120020 - ff d5 b2 a4 26 fe 99 c0-7f 94 a1 00 00 2e c0 14   ....&...........
130030 - c0 0a 00 39 ff 85 00 88-00 81 00 35 00 84 c0 13   ...9.......5....
14---
150076 - <SPACES/NULS>
16read from 0x13d7bdf90 [0x13e01a803] (5 bytes => 5 (0x5))
170005 - <SPACES/NULS>
184307385836:error:1400410B:SSL routines:CONNECT_CR_SRVR_HELLO:wrong version number:/System/Volumes/Data/SWE/macOS/BuildRoots/e90674e518/Library/Caches/com.apple.xbs/Sources/libressl/libressl-56.60.2/libressl-2.8/ssl/ssl_pkt.c:386:
19---
20no peer certificate available
21---
22No client certificate CA names sent
23---
24SSL handshake has read 5 bytes and written 0 bytes
25---
26New, (NONE), Cipher is (NONE)
27Secure Renegotiation IS NOT supported
28Compression: NONE
29Expansion: NONE
30No ALPN negotiated
31SSL-Session:
32    Protocol  : TLSv1
33    Cipher    : 0000
34    Session-ID: 
35    Session-ID-ctx: 
36    Master-Key: 
37    Start Time: 1636579015
38    Timeout   : 7200 (sec)
39    Verify return code: 0 (ok)
40---
41
42# Licensed to the Apache Software Foundation (ASF) under one or more
43# contributor license agreements.  See the NOTICE file distributed with
44# this work for additional information regarding copyright ownership.
45# The ASF licenses this file to You under the Apache License, Version 2.0
46# (the "License"); you may not use this file except in compliance with
47# the License.  You may obtain a copy of the License at
48#
49#    http://www.apache.org/licenses/LICENSE-2.0
50#
51# Unless required by applicable law or agreed to in writing, software
52# distributed under the License is distributed on an "AS IS" BASIS,
53# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
54# See the License for the specific language governing permissions and
55# limitations under the License.
56
57# see kafka.server.KafkaConfig for additional details and defaults
58
59############################# Server Basics #############################
60
61# The id of the broker. This must be set to a unique integer for each broker.
62broker.id=0
63
64############################# Socket Server Settings #############################
65
66# The address the socket server listens on. It will get the value returned from 
67# java.net.InetAddress.getCanonicalHostName() if not configured.
68#   FORMAT:
69#     listeners = listener_name://host_name:port
70#   EXAMPLE:
71#     listeners = PLAINTEXT://your.host.name:9092
72# SSL CHANGE
73listeners=PLAINTEXT://localhost:9092,SSL://localhost:9093
74
75# Hostname and port the broker will advertise to producers and consumers. If not set, 
76# it uses the value for "listeners" if configured.  Otherwise, it will use the value
77# returned from java.net.InetAddress.getCanonicalHostName().
78# SSL CHANGE
79advertised.listeners=PLAINTEXT://localhost:9092,SSL://localhost:9093
80ssl.client.auth=none
81
82# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details
83listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL
84
85# The number of threads that the server uses for receiving requests from the network and sending responses to the network
86num.network.threads=3
87
88# The number of threads that the server uses for processing requests, which may include disk I/O
89num.io.threads=8
90
91# The send buffer (SO_SNDBUF) used by the socket server
92socket.send.buffer.bytes=102400
93
94# The receive buffer (SO_RCVBUF) used by the socket server
95socket.receive.buffer.bytes=102400
96
97# The maximum size of a request that the socket server will accept (protection against OOM)
98socket.request.max.bytes=104857600
99
100
101############################# Log Basics #############################
102
103# A comma separated list of directories under which to store log files
104log.dirs=/tmp/kafka-logs
105
106# The default number of log partitions per topic. More partitions allow greater
107# parallelism for consumption, but this will also result in more files across
108# the brokers.
109num.partitions=1
110
111# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown.
112# This value is recommended to be increased for installations with data dirs located in RAID array.
113num.recovery.threads.per.data.dir=1
114
115############################# Internal Topic Settings  #############################
116# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state"
117# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3.
118offsets.topic.replication.factor=1
119transaction.state.log.replication.factor=1
120transaction.state.log.min.isr=1
121
122############################# Log Flush Policy #############################
123
124# Messages are immediately written to the filesystem but by default we only fsync() to sync
125# the OS cache lazily. The following configurations control the flush of data to disk.
126# There are a few important trade-offs here:
127#    1. Durability: Unflushed data may be lost if you are not using replication.
128#    2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush.
129#    3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks.
130# The settings below allow one to configure the flush policy to flush data after a period of time or
131# every N messages (or both). This can be done globally and overridden on a per-topic basis.
132
133# The number of messages to accept before forcing a flush of data to disk
134#log.flush.interval.messages=10000
135
136# The maximum amount of time a message can sit in a log before we force a flush
137#log.flush.interval.ms=1000
138
139############################# Log Retention Policy #############################
140
141# The following configurations control the disposal of log segments. The policy can
142# be set to delete segments after a period of time, or after a given size has accumulated.
143# A segment will be deleted whenever *either* of these criteria are met. Deletion always happens
144# from the end of the log.
145
146# The minimum age of a log file to be eligible for deletion due to age
147log.retention.hours=168
148
149# A size-based retention policy for logs. Segments are pruned from the log unless the remaining
150# segments drop below log.retention.bytes. Functions independently of log.retention.hours.
151#log.retention.bytes=1073741824
152
153# The maximum size of a log segment file. When this size is reached a new log segment will be created.
154log.segment.bytes=1073741824
155
156# The interval at which log segments are checked to see if they can be deleted according
157# to the retention policies
158log.retention.check.interval.ms=300000
159
160############################# Zookeeper #############################
161
162# Zookeeper connection string (see zookeeper docs for details).
163# This is a comma separated host:port pairs, each corresponding to a zk
164# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
165# You can also append an optional chroot string to the urls to specify the
166# root directory for all kafka znodes.
167zookeeper.connect=localhost:2181
168
169# Timeout in ms for connecting to zookeeper
170zookeeper.connection.timeout.ms=18000
171
172##################### Confluent Metrics Reporter #######################
173# Confluent Control Center and Confluent Auto Data Balancer integration
174#
175# Uncomment the following lines to publish monitoring data for
176# Confluent Control Center and Confluent Auto Data Balancer
177# If you are using a dedicated metrics cluster, also adjust the settings
178# to point to your metrics kakfa cluster.
179#metric.reporters=io.confluent.metrics.reporter.ConfluentMetricsReporter
180#confluent.metrics.reporter.bootstrap.servers=localhost:9092
181#
182# Uncomment the following line if the metrics cluster has a single broker
183#confluent.metrics.reporter.topic.replicas=1
184
185############################# Group Coordinator Settings #############################
186
187# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance.
188# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms.
189# The default value for this is 3 seconds.
190# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing.
191# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup.
192group.initial.rebalance.delay.ms=0
193
194
195############################# Confluent Authorizer Settings  #############################
196
197# Uncomment to enable Confluent Authorizer with support for ACLs, LDAP groups and RBAC
198#authorizer.class.name=io.confluent.kafka.security.authorizer.ConfluentServerAuthorizer
199# Semi-colon separated list of super users in the format <principalType>:<principalName>
200#super.users=
201# Specify a valid Confluent license. By default free-tier license will be used
202#confluent.license=
203# Replication factor for the topic used for licensing. Default is 3.
204confluent.license.topic.replication.factor=1
205
206# Uncomment the following lines and specify values where required to enable CONFLUENT provider for RBAC and centralized ACLs
207# Enable CONFLUENT provider 
208#confluent.authorizer.access.rule.providers=ZK_ACL,CONFLUENT
209# Bootstrap servers for RBAC metadata. Must be provided if this broker is not in the metadata cluster
210#confluent.metadata.bootstrap.servers=PLAINTEXT://127.0.0.1:9092
211# Replication factor for the metadata topic used for authorization. Default is 3.
212confluent.metadata.topic.replication.factor=1
213
214# Replication factor for the topic used for audit logs. Default is 3.
215confluent.security.event.logger.exporter.kafka.topic.replicas=1
216
217# Listeners for metadata server
218#confluent.metadata.server.listeners=http://0.0.0.0:8090
219# Advertised listeners for metadata server
220#confluent.metadata.server.advertised.listeners=http://127.0.0.1:8090
221
222############################# Confluent Data Balancer Settings  #############################
223
224# The Confluent Data Balancer is used to measure the load across the Kafka cluster and move data
225# around as necessary. Comment out this line to disable the Data Balancer.
226confluent.balancer.enable=true
227
228# By default, the Data Balancer will only move data when an empty broker (one with no partitions on it)
229# is added to the cluster or a broker failure is detected. Comment out this line to allow the Data
230# Balancer to balance load across the cluster whenever an imbalance is detected.
231#confluent.balancer.heal.uneven.load.trigger=ANY_UNEVEN_LOAD
232
233# The default time to declare a broker permanently failed is 1 hour (3600000 ms).
234# Uncomment this line to turn off broker failure detection, or adjust the threshold
235# to change the duration before a broker is declared failed.
236#confluent.balancer.heal.broker.failure.threshold.ms=-1
237
238# Edit and uncomment the following line to limit the network bandwidth used by data balancing operations.
239# This value is in bytes/sec/broker. The default is 10MB/sec.
240#confluent.balancer.throttle.bytes.per.second=10485760
241
242# Capacity Limits -- when set to positive values, the Data Balancer will attempt to keep
243# resource usage per-broker below these limits.
244# Edit and uncomment this line to limit the maximum number of replicas per broker. Default is unlimited.
245#confluent.balancer.max.replicas=10000
246
247# Edit and uncomment this line to limit what fraction of the log disk (0-1.0) is used before rebalancing.
248# The default (below) is 85% of the log disk.
249#confluent.balancer.disk.max.load=0.85
250
251# Edit and uncomment these lines to define a maximum network capacity per broker, in bytes per
252# second. The Data Balancer will attempt to ensure that brokers are using less than this amount
253# of network bandwidth when rebalancing.
254# Here, 10MB/s. The default is unlimited capacity.
255#confluent.balancer.network.in.max.bytes.per.second=10485760
256#confluent.balancer.network.out.max.bytes.per.second=10485760
257
258# Edit and uncomment this line to identify specific topics that should not be moved by the data balancer.
259# Removal operations always move topics regardless of this setting.
260#confluent.balancer.exclude.topic.names=
261
262# Edit and uncomment this line to identify topic prefixes that should not be moved by the data balancer.
263# (For example, a "confluent.balancer" prefix will match all of "confluent.balancer.a", "confluent.balancer.b",
264# "confluent.balancer.c", and so on.)
265# Removal operations always move topics regardless of this setting.
266#confluent.balancer.exclude.topic.prefixes=
267
268# The replication factor for the topics the Data Balancer uses to store internal state.
269# For anything other than development testing, a value greater than 1 is recommended to ensure availability.
270# The default value is 3.
271confluent.balancer.topic.replication.factor=1
272
273################################## Confluent Telemetry Settings  ##################################
274
275# To start using Telemetry, first generate a Confluent Cloud API key/secret. This can be done with
276# instructions at https://docs.confluent.io/current/cloud/using/api-keys.html. Note that you should
277# be using the '--resource cloud' flag.
278#
279# After generating an API key/secret, to enable Telemetry uncomment the lines below and paste
280# in your API key/secret.
281#
282#confluent.telemetry.enabled=true
283#confluent.telemetry.api.key=<CLOUD_API_KEY>
284#confluent.telemetry.api.secret=<CCLOUD_API_SECRET>
285
286############ SSL #################
287
288ssl.truststore.location=/Users/karanalang/Documents/Technology/confluent-6.2.1/ssl_certs/truststore/kafka.truststore.jks
289ssl.truststore.password=test123
290ssl.keystore.location=/Users/karanalang/Documents/Technology/confluent-6.2.1/ssl_certs/keystore/kafka.keystore.jks
291ssl.keystore.password=test123
292ssl.key.password=test123
293
294# confluent.metrics.reporter.bootstrap.servers=localhost:9093
295# confluent.metrics.reporter.security.protocol=SSL
296# confluent.metrics.reporter.ssl.truststore.location=/Users/karanalang/Documents/Technology/confluent-6.2.1/ssl_certs/truststore/kafka.truststore.jks
297# confluent.metrics.reporter.ssl.truststore.password=test123
298# confluent.metrics.reporter.ssl.keystore.location=/Users/karanalang/Documents/Technology/confluent-6.2.1/ssl_certs/keystore/kafka.keystore.jks
299# confluent.metrics.reporter.ssl.keystore.password=test123
300# confluent.metrics.reporter.ssl.key.password=test123
301
302bootstrap.servers=localhost:9093
303security.protocol=SSL
304ssl.truststore.location=/Users/karanalang/Documents/Technology/confluent-6.2.1/ssl_certs/truststore/kafka.truststore.jks
305ssl.truststore.password=test123
306ssl.keystore.location=/Users/karanalang/Documents/Technology/confluent-6.2.1/ssl_certs/keystore/kafka.keystore.jks
307ssl.keystore.password=test123
308ssl.key.password=test123
309$CONFLUENT_HOME/bin/kafka-console-producer --broker-list localhost:9093 --topic karantest --producer.config $CONFLUENT_HOME/props/client-ssl.properties
310$CONFLUENT_HOME/bin/kafka-console-consumer --bootstrap-server localhost:9093 --topic karantest --consumer.config $CONFLUENT_HOME/props/client-ssl.properties --from-beginning
311javax.net.ssl|DEBUG|0E|kafka-producer-network-thread | console-producer|2021-11-10 14:04:26.107 PST|SSLExtensions.java:173|Ignore unavailable extension: status_request
312javax.net.ssl|DEBUG|0E|kafka-producer-network-thread | console-producer|2021-11-10 14:04:26.107 PST|SSLExtensions.java:173|Ignore unavailable extension: status_request
313javax.net.ssl|ERROR|0E|kafka-producer-network-thread | console-producer|2021-11-10 14:04:26.108 PST|TransportContext.java:341|Fatal (CERTIFICATE_UNKNOWN): No name matching localhost found (
314"throwable" : {
315  java.security.cert.CertificateException: No name matching localhost found
316    at java.base/sun.security.util.HostnameChecker.matchDNS(HostnameChecker.java:234)
317    at java.base/sun.security.util.HostnameChecker.match(HostnameChecker.java:103)
318    at java.base/sun.security.ssl.X509TrustManagerImpl.checkIdentity(X509TrustManagerImpl.java:455)
319    at java.base/sun.security.ssl.X509TrustManagerImpl.checkIdentity(X509TrustManagerImpl.java:429)
320    at java.base/sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:283)
321    at java.base/sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:141)
322    at java.base/sun.security.ssl.CertificateMessage$T13CertificateConsumer.checkServerCerts(CertificateMessage.java:1335)
323    at java.base/sun.security.ssl.CertificateMessage$T13CertificateConsumer.onConsumeCertificate(CertificateMessage.java:1232)
324    at java.base/sun.security.ssl.CertificateMessage$T13CertificateConsumer.consume(CertificateMessage.java:1175)
325    at java.base/sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:392)
326    at java.base/sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:443)
327    at java.base/sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1074)
328    at java.base/sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1061)
329    at java.base/java.security.AccessController.doPrivileged(Native Method)
330    at java.base/sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1008)
331    at org.apache.kafka.common.network.SslTransportLayer.runDelegatedTasks(SslTransportLayer.java:509)
332    at org.apache.kafka.common.network.SslTransportLayer.handshakeUnwrap(SslTransportLayer.java:601)
333    at org.apache.kafka.common.network.SslTransportLayer.doHandshake(SslTransportLayer.java:447)
334    at org.apache.kafka.common.network.SslTransportLayer.handshake(SslTransportLayer.java:332)
335    at org.apache.kafka.common.network.KafkaChannel.prepare(KafkaChannel.java:229)
336    at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:563)
337    at org.apache.kafka.common.network.Selector.poll(Selector.java:499)
338    at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:639)
339    at org.apache.kafka.clients.producer.internals.Sender.runOnce(Sender.java:327)
340    at org.apache.kafka.clients.producer.internals.Sender.run(Sender.java:242)
341    at java.base/java.lang.Thread.run(Thread.java:829)}
342
343

1// Copyright 2021 Google LLC
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     https://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package compute
16
17// [START compute_instances_list_all]
18import (
19    "context"
20    "fmt"
21    "io"
22
23    compute "cloud.google.com/go/compute/apiv1"
24    "google.golang.org/api/iterator"
25    computepb "google.golang.org/genproto/googleapis/cloud/compute/v1"
26    "google.golang.org/protobuf/proto"
27)
28
29// listAllInstances prints all instances present in a project, grouped by their zone.
30func ListAllInstances(w io.Writer, projectID string) error {
31    // projectID := "your_project_id"
32    ctx := context.Background()
33    instancesClient, err := compute.NewInstancesRESTClient(ctx)
34    // instancesClient, err := compute.NewInstancesRESTClient(ctx, option.WithCredentialsFile(`C:\path\to\jsonkey.json`))
35
36    if err != nil {
37        return fmt.Errorf("NewInstancesRESTClient: %v", err)
38    }
39    defer instancesClient.Close()
40
41    // Use the `MaxResults` parameter to limit the number of results that the API returns per response page.
42    req := &computepb.AggregatedListInstancesRequest{
43        Project:    projectID,
44        MaxResults: proto.Uint32(6),
45    }
46
47    it := instancesClient.AggregatedList(ctx, req)
48    fmt.Fprintf(w, "Instances found:\n")
49    // Despite using the `MaxResults` parameter, you don't need to handle the pagination
50    // yourself. The returned iterator object handles pagination
51    // automatically, returning separated pages as you iterate over the results.
52    for {
53        pair, err := it.Next()
54        if err == iterator.Done {
55            break
56        }
57        if err != nil {
58            return err
59        }
60        instances := pair.Value.Instances
61        if len(instances) > 0 {
62            fmt.Fprintf(w, "%s\n", pair.Key)
63            for _, instance := range instances {
64                fmt.Fprintf(w, "- %s %s\n", instance.GetName(), instance.GetMachineType())
65            }
66        }
67    }
68
69    return nil
70}
71
72// [END compute_instances_list_all]
73
74

1// Copyright 2021 Google LLC
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     https://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package compute
16
17// [START compute_instances_list_all]
18import (
19    "context"
20    "fmt"
21    "io"
22
23    compute "cloud.google.com/go/compute/apiv1"
24    "google.golang.org/api/iterator"
25    computepb "google.golang.org/genproto/googleapis/cloud/compute/v1"
26    "google.golang.org/protobuf/proto"
27)
28
29// listAllInstances prints all instances present in a project, grouped by their zone.
30func ListAllInstances(w io.Writer, projectID string) error {
31    // projectID := "your_project_id"
32    ctx := context.Background()
33    instancesClient, err := compute.NewInstancesRESTClient(ctx)
34    // instancesClient, err := compute.NewInstancesRESTClient(ctx, option.WithCredentialsFile(`C:\path\to\jsonkey.json`))
35
36    if err != nil {
37        return fmt.Errorf("NewInstancesRESTClient: %v", err)
38    }
39    defer instancesClient.Close()
40
41    // Use the `MaxResults` parameter to limit the number of results that the API returns per response page.
42    req := &computepb.AggregatedListInstancesRequest{
43        Project:    projectID,
44        MaxResults: proto.Uint32(6),
45    }
46
47    it := instancesClient.AggregatedList(ctx, req)
48    fmt.Fprintf(w, "Instances found:\n")
49    // Despite using the `MaxResults` parameter, you don't need to handle the pagination
50    // yourself. The returned iterator object handles pagination
51    // automatically, returning separated pages as you iterate over the results.
52    for {
53        pair, err := it.Next()
54        if err == iterator.Done {
55            break
56        }
57        if err != nil {
58            return err
59        }
60        instances := pair.Value.Instances
61        if len(instances) > 0 {
62            fmt.Fprintf(w, "%s\n", pair.Key)
63            for _, instance := range instances {
64                fmt.Fprintf(w, "- %s %s\n", instance.GetName(), instance.GetMachineType())
65            }
66        }
67    }
68
69    return nil
70}
71
72// [END compute_instances_list_all]
73
74package main
75
76import (
77    "fmt"
78
79    "example.com/compute"
80
81    "bytes"
82)
83
84func main() {
85    buf := new(bytes.Buffer)
86
87    // Get a message and print it.
88    respone := compute.ListAllInstances(buf, "project-unique-id")
89    fmt.Println(respone)
90}
91
92

1implementation 'org.springframework.boot:spring-boot-starter-security'
2implementation 'org.springframework.boot:spring-boot-starter-oauth2-resource-server'
3

1implementation 'org.springframework.boot:spring-boot-starter-security'
2implementation 'org.springframework.boot:spring-boot-starter-oauth2-resource-server'
3@EnableWebFluxSecurity
4public class WebSecurityConfiguration {
5
6  // ...
7
8  @Bean
9  @Order(1)
10  public SecurityWebFilterChain iamAuthFilterChain(ServerHttpSecurity http) {
11    ServerWebExchangeMatcher matcher = exchange -> {
12      HttpHeaders headers = exchange.getRequest().getHeaders();
13      List<String> strings = headers.get(SurpriseHeaders.IDP_AUTH_TOKEN_HEADER_NAME);
14      return strings != null && strings.size() > 0
15          ? MatchResult.match() : MatchResult.notMatch();
16    };
17
18    http
19        .securityMatcher(matcher)
20        .csrf().disable()
21        .authorizeExchange()
22          .pathMatchers(navigationService.getAuthFreeEndpoints()).permitAll()
23          .anyExchange().authenticated()
24        .and()
25          .oauth2ResourceServer(OAuth2ResourceServerSpec::jwt)
26          .oauth2ResourceServer().jwt().jwkSetUri(getJwkUri())
27          .and()
28        .and()
29          .addFilterAt(new LoggingFilter("idpAuthFilterChain"), SecurityWebFiltersOrder.FIRST)
30          .addFilterAfter(new IdpTokenExchangeFilter(authClientService), SecurityWebFiltersOrder.AUTHENTICATION)
31    ;    
32    return http.build();
33  }
34
35}
36

1gcloud api-gateway api-configs list --api=$API --project=$PROJECT_ID --format="table(name)"
2

1gcloud api-gateway api-configs list --api=$API --project=$PROJECT_ID --format="table(name)"
2steps:
3  - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
4    entrypoint: "bash"
5    args:
6      - "-c"
7      - |
8        gcloud api-gateway api-configs list --api=logistics-homolog --project=$PROJECT_ID \
9        --filter=serviceConfigId:logistics-mobile-places-* --format="table(name)"
10

1gcloud api-gateway api-configs list --api=$API --project=$PROJECT_ID --format="table(name)"
2steps:
3  - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
4    entrypoint: "bash"
5    args:
6      - "-c"
7      - |
8        gcloud api-gateway api-configs list --api=logistics-homolog --project=$PROJECT_ID \
9        --filter=serviceConfigId:logistics-mobile-places-* --format="table(name)"
10ERROR: (gcloud.api-gateway.api-configs.list) PERMISSION_DENIED: Permission 'apigateway.apiconfigs.list' denied on 'projects/$PROJECT_ID/locations/global/apis/logistics-homolog/configs'
11

1gcloud identity groups memberships list --group-email=abc@xyz.com
2