Pub Sub

1p = beam.Pipeline(options=options)
2
3json_string_output =   (
4                          p
5                          | 'Read from BQ' >> beam.io.ReadFromBigQuery(
6                                query='SELECT * FROM '\
7                                 '`project.dataset.table_name`',
8                                 use_standard_sql=True)
9                          | 'convert to json' >> beam.Map(lambda record: json.dumps(record))
10                          | 'Write results' >> beam.io.WriteToText(outputs_prefix)
11                      )
12
13p.run()
14

1p = beam.Pipeline(options=options)
2
3json_string_output =   (
4                          p
5                          | 'Read from BQ' >> beam.io.ReadFromBigQuery(
6                                query='SELECT * FROM '\
7                                 '`project.dataset.table_name`',
8                                 use_standard_sql=True)
9                          | 'convert to json' >> beam.Map(lambda record: json.dumps(record))
10                          | 'Write results' >> beam.io.WriteToText(outputs_prefix)
11                      )
12
13p.run()
14 # create publisher
15    publisher = pubsub_v1.PublisherClient()
16
17    with open(input_file, 'rb') as ifp:
18        header = ifp.readline()  
19        # loop over each record
20        for line in ifp:
21            event_data = line   # entire line of input file is the message
22            print('Publishing {0} to {1}'.format(event_data, pubsub_topic))
23            publisher.publish(pubsub_topic, event_data)
24

1resource "google_pubsub_lite_reservation" "pubsub_lite_reservation" {
2  name = var.lite_reservation_name
3  project = var.project
4  region  = var.region
5  throughput_capacity = var.throughput_capacity
6}
7
8resource "google_pubsub_lite_topic" "pubsub_lite_topic" {
9  name    = var.topic_name
10  project = var.project
11  region  = var.region
12  zone    = var.zone
13  partition_config {
14    count = var.partitions_count
15    capacity {
16      publish_mib_per_sec   = var.publish_mib_per_sec
17      subscribe_mib_per_sec = var.subscribe_mib_per_sec
18    }
19  }
20
21  retention_config {
22    per_partition_bytes = var.per_partition_bytes
23    period              = var.period
24  }
25
26  reservation_config {
27    throughput_reservation = google_pubsub_lite_reservation.pubsub_lite_reservation.name
28  }
29
30}
31

196f4d12cdd02:~# nsc edit user RequestCacheService --allow-pubsub "$JS.API.STREAM.INFO.*"
2[ OK ] added pub pub ".API.STREAM.INFO.*"
3[ OK ] added sub ".API.STREAM.INFO.*"
4[ OK ] generated user creds file `/nsc/nkeys/creds/Client/Client/RequestCacheService.creds`
5[ OK ] edited user "RequestCacheService"
6

1manual_scaling:
2  instances: 1
3

1def registration_body():
2    return {  # An instruction to Classroom to send notifications from the `feed` to the
3        # provided destination.
4        "feed": {  # Information about a `Feed` with a `feed_type` of `COURSE_WORK_CHANGES`.
5            "feedType": "COURSE_WORK_CHANGES",  # Information about a `Feed` with a `feed_type` of `COURSE_WORK_CHANGES`.
6            "courseWorkChangesInfo": { 
7                # This field must be specified if `feed_type` is `COURSE_WORK_CHANGES`.
8                "courseId": "xxxxxxxxxxxx",  # The `course_id` of the course to subscribe to work changes for.
9            },
10        },
11        "cloudPubsubTopic": {
12            "topicName": "projects/xxxxx/topics/gcr-course",  # The `name` field of a Cloud Pub/Sub
13        },
14    }
15
16
17def create_registration(service):
18    """
19    Creates a registration to the Google Classroom Service which will listen
20    for updates on Google Classroom according to the requested body.
21    Pub Sub will emit a payload to subscribers when a classroom upate occurs.
22
23    Args:
24        service (Google Classroom Service): Google Classroom Service as retrieved
25        from the Google Aclassroom Service builder
26
27    Returns:
28        Registration: Google Classroom Registration
29    """
30
31    body = registration_body()
32    try:
33        registration = service.registrations().create(body=body).execute()
34        print(f"Registration to Google CLassroom Created\n{registration}")
35        return registration
36    except Exception as e:
37        print(e)
38        raise
39

1def registration_body():
2    return {  # An instruction to Classroom to send notifications from the `feed` to the
3        # provided destination.
4        "feed": {  # Information about a `Feed` with a `feed_type` of `COURSE_WORK_CHANGES`.
5            "feedType": "COURSE_WORK_CHANGES",  # Information about a `Feed` with a `feed_type` of `COURSE_WORK_CHANGES`.
6            "courseWorkChangesInfo": { 
7                # This field must be specified if `feed_type` is `COURSE_WORK_CHANGES`.
8                "courseId": "xxxxxxxxxxxx",  # The `course_id` of the course to subscribe to work changes for.
9            },
10        },
11        "cloudPubsubTopic": {
12            "topicName": "projects/xxxxx/topics/gcr-course",  # The `name` field of a Cloud Pub/Sub
13        },
14    }
15
16
17def create_registration(service):
18    """
19    Creates a registration to the Google Classroom Service which will listen
20    for updates on Google Classroom according to the requested body.
21    Pub Sub will emit a payload to subscribers when a classroom upate occurs.
22
23    Args:
24        service (Google Classroom Service): Google Classroom Service as retrieved
25        from the Google Aclassroom Service builder
26
27    Returns:
28        Registration: Google Classroom Registration
29    """
30
31    body = registration_body()
32    try:
33        registration = service.registrations().create(body=body).execute()
34        print(f"Registration to Google CLassroom Created\n{registration}")
35        return registration
36    except Exception as e:
37        print(e)
38        raise
39def init_subscription():
40    # [INIT PUBSUB SUBSCRIBER AND CALLBACKS]
41    subscriber = pubsub_v1.SubscriberClient()
42    subscription_path = subscriber.subscription_path(
43        "x-student-portal", "gcr-course-sub"
44    )
45    future = subscriber.subscribe(subscription_path, callback)
46    with subscriber:
47        try:
48            future.result()
49        except TimeoutError:
50            future.cancel()
51            future.result()
52
53
54def callback(message):
55    print("message recieved")
56    # do_stuff(message)
57    print(message)
58    message.ack()
59

1def registration_body():
2    return {  # An instruction to Classroom to send notifications from the `feed` to the
3        # provided destination.
4        "feed": {  # Information about a `Feed` with a `feed_type` of `COURSE_WORK_CHANGES`.
5            "feedType": "COURSE_WORK_CHANGES",  # Information about a `Feed` with a `feed_type` of `COURSE_WORK_CHANGES`.
6            "courseWorkChangesInfo": { 
7                # This field must be specified if `feed_type` is `COURSE_WORK_CHANGES`.
8                "courseId": "xxxxxxxxxxxx",  # The `course_id` of the course to subscribe to work changes for.
9            },
10        },
11        "cloudPubsubTopic": {
12            "topicName": "projects/xxxxx/topics/gcr-course",  # The `name` field of a Cloud Pub/Sub
13        },
14    }
15
16
17def create_registration(service):
18    """
19    Creates a registration to the Google Classroom Service which will listen
20    for updates on Google Classroom according to the requested body.
21    Pub Sub will emit a payload to subscribers when a classroom upate occurs.
22
23    Args:
24        service (Google Classroom Service): Google Classroom Service as retrieved
25        from the Google Aclassroom Service builder
26
27    Returns:
28        Registration: Google Classroom Registration
29    """
30
31    body = registration_body()
32    try:
33        registration = service.registrations().create(body=body).execute()
34        print(f"Registration to Google CLassroom Created\n{registration}")
35        return registration
36    except Exception as e:
37        print(e)
38        raise
39def init_subscription():
40    # [INIT PUBSUB SUBSCRIBER AND CALLBACKS]
41    subscriber = pubsub_v1.SubscriberClient()
42    subscription_path = subscriber.subscription_path(
43        "x-student-portal", "gcr-course-sub"
44    )
45    future = subscriber.subscribe(subscription_path, callback)
46    with subscriber:
47        try:
48            future.result()
49        except TimeoutError:
50            future.cancel()
51            future.result()
52
53
54def callback(message):
55    print("message recieved")
56    # do_stuff(message)
57    print(message)
58    message.ack()
59@app.on_event("startup")
60async def startup_event():
61    global db
62    global gc_api
63    global gc_service
64    global gc_registration
65    global future
66    global pub_sub_subscription_thread
67
68    db = firestore.FirestoreDatabase()
69    gc_service = get_service()
70    gc_api = ClassroomApi(service=gc_service)
71    gc_registration = publisher_client.create_registration(gc_service)
72
73    pub_sub_subscription_thread = multiprocessing.Process(
74        target=publisher_client.init_subscription
75    )
76    pub_sub_subscription_thread.start()
77

1fun subscribeToData() : Flow<List<MyData>> {
2    ....
3    //incoming data
4    emit(list)
5}
6
7fun getUptoFirst3Items() {
8   subscribeToData()
9   .take(ITEM_COUNT) // ITEM_COUNT is 3
10   .distinctUntilChange() //only proceed if the data is different from the previous top 3 items
11   .mapIndex {
12     //do transformation
13   }
14   .collect { transformedListOf3Elements ->
15    
16   }
17
18}
19

1Subscription sub = createSubscription();
2sub.addListener(new Listener() {
3        public void eventsReceived(List events) {
4                for (Event e : events)
5                        buffer.enqueue(e)
6        }
7});
8