GCP

1import pandas as pd
2
3# gcp / google sheets URL
4df_URL = "https://docs.google.com/spreadsheets/d/1wKtvNfWSjPNC1fNmTfUHm7sXiaPyOZMchjzQBt1y_f8/edit?usp=sharing"
5
6raw_dataset = pd.read_csv(df_URL, na_values='?',sep=';'
7                          , skipinitialspace=True, header=0, index_col=None)
8

1import pandas as pd
2
3# gcp / google sheets URL
4df_URL = "https://docs.google.com/spreadsheets/d/1wKtvNfWSjPNC1fNmTfUHm7sXiaPyOZMchjzQBt1y_f8/edit?usp=sharing"
5
6raw_dataset = pd.read_csv(df_URL, na_values='?',sep=';'
7                          , skipinitialspace=True, header=0, index_col=None)
8import pandas as pd
9from google.colab import auth
10auth.authenticate_user()
11import gspread
12from oauth2client.client import GoogleCredentials
13
14
15# gcp / google sheets url
16df_URL = "https://docs.google.com/spreadsheets/d/1wKtvNfWSjPNC1fNmTfUHm7sXiaPyOZMchjzQBt1y_f8/edit?usp=sharing"
17
18# importing the data from Google Drive  setup
19gc = gspread.authorize(GoogleCredentials.get_application_default())
20
21# read data and put it in dataframe
22g_sheets = gc.open_by_url(df_URL) 
23
24df = pd.DataFrame(g_sheets.get_worksheet(0).get_all_values())
25
26  
27# change first row to header
28df = df.rename(columns=df.iloc[0]) 
29
30# drop first row
31df.drop(index=df.index[0], axis=0, inplace=True) 
32

1try (SecretManagerServiceClient client = SecretManagerServiceClient.create()) {
2  // Build the parent name.
3  SecretName projectName = SecretName.of(projectId, secretId);
4
5  // Get all versions.
6  ListSecretVersionsPagedResponse pagedResponse = client.listSecretVersions(projectName);
7
8  // List all versions and their state.
9  pagedResponse
10      .iterateAll()
11      .forEach(
12          version -> {
13            System.out.printf("Secret version %s, %s\n", version.getName(), version.getState());
14          });
15}
16

1> ####@1.0.1 prepare /workspace
2> npm run gcp-build
3
4
5> ####@1.0.1 gcp-build /workspace
6> tsc -p .
7
8sh: 1: tsc: not found
9npm ERR! code ELIFECYCLE
10npm ERR! syscall spawn
11npm ERR! file sh
12npm ERR! errno ENOENT
13npm ERR! ####@1.0.1 gcp-build: `tsc -p .`
14npm ERR! spawn ENOENT
15npm ERR!
16npm ERR! Failed at the ####@1.0.1 gcp-build script.
17npm ERR! This is probably not a problem with npm. There is likely additional logging output above.
18

1> ####@1.0.1 prepare /workspace
2> npm run gcp-build
3
4
5> ####@1.0.1 gcp-build /workspace
6> tsc -p .
7
8sh: 1: tsc: not found
9npm ERR! code ELIFECYCLE
10npm ERR! syscall spawn
11npm ERR! file sh
12npm ERR! errno ENOENT
13npm ERR! ####@1.0.1 gcp-build: `tsc -p .`
14npm ERR! spawn ENOENT
15npm ERR!
16npm ERR! Failed at the ####@1.0.1 gcp-build script.
17npm ERR! This is probably not a problem with npm. There is likely additional logging output above.
18    "devDependencies": {
19        "@google-cloud/nodejs-repo-tools": "^3.3.0",
20        "@types/cors": "^2.8.12",
21        "@types/express": "^4.17.13",
22        "@types/pg": "^8.6.1",
23        "@types/request": "^2.48.7",
24        "tsc-watch": "^4.4.0",
25        "tslint": "^6.1.3",
26        "typescript": "^4.3.5"
27    },
28

1 // Configure the settings for the job controller client.
2    JobControllerSettings jobControllerSettings =
3        JobControllerSettings.newBuilder().setEndpoint(myEndpoint).build();
4

1 // Configure the settings for the job controller client.
2    JobControllerSettings jobControllerSettings =
3        JobControllerSettings.newBuilder().setEndpoint(myEndpoint).build();
4SEVERE: Servlet.service() for servlet [dispatcherServlet] in context with path [] threw exception [Handler dispatch failed; nested exception is java.lang.NoSuchMethodError: 'com.google.api.gax.core.GoogleCredentialsProvider$Builder com.google.api.gax.core.GoogleCredentialsProvider$Builder.setUseJwtAccessWithScope(boolean)'] with root cause
5java.lang.NoSuchMethodError: 'com.google.api.gax.core.GoogleCredentialsProvider$Builder com.google.api.gax.core.GoogleCredentialsProvider$Builder.setUseJwtAccessWithScope(boolean)'
6

1 // Configure the settings for the job controller client.
2    JobControllerSettings jobControllerSettings =
3        JobControllerSettings.newBuilder().setEndpoint(myEndpoint).build();
4SEVERE: Servlet.service() for servlet [dispatcherServlet] in context with path [] threw exception [Handler dispatch failed; nested exception is java.lang.NoSuchMethodError: 'com.google.api.gax.core.GoogleCredentialsProvider$Builder com.google.api.gax.core.GoogleCredentialsProvider$Builder.setUseJwtAccessWithScope(boolean)'] with root cause
5java.lang.NoSuchMethodError: 'com.google.api.gax.core.GoogleCredentialsProvider$Builder com.google.api.gax.core.GoogleCredentialsProvider$Builder.setUseJwtAccessWithScope(boolean)'
6<dependencyManagement>
7        <dependencies>
8            <dependency>
9                <groupId>com.google.cloud</groupId>
10                <artifactId>libraries-bom</artifactId>
11                <version>24.1.2</version>
12                <type>pom</type>
13                <scope>import</scope>
14            </dependency>
15
16            <dependency>
17                <groupId>org.springframework.cloud</groupId>
18                <artifactId>spring-cloud-gcp-dependencies</artifactId>
19                <version>1.2.8.RELEASE</version>
20                <type>pom</type>
21                <scope>import</scope>
22            </dependency>
23
24        </dependencies>
25    </dependencyManagement>
26

1 // Configure the settings for the job controller client.
2    JobControllerSettings jobControllerSettings =
3        JobControllerSettings.newBuilder().setEndpoint(myEndpoint).build();
4SEVERE: Servlet.service() for servlet [dispatcherServlet] in context with path [] threw exception [Handler dispatch failed; nested exception is java.lang.NoSuchMethodError: 'com.google.api.gax.core.GoogleCredentialsProvider$Builder com.google.api.gax.core.GoogleCredentialsProvider$Builder.setUseJwtAccessWithScope(boolean)'] with root cause
5java.lang.NoSuchMethodError: 'com.google.api.gax.core.GoogleCredentialsProvider$Builder com.google.api.gax.core.GoogleCredentialsProvider$Builder.setUseJwtAccessWithScope(boolean)'
6<dependencyManagement>
7        <dependencies>
8            <dependency>
9                <groupId>com.google.cloud</groupId>
10                <artifactId>libraries-bom</artifactId>
11                <version>24.1.2</version>
12                <type>pom</type>
13                <scope>import</scope>
14            </dependency>
15
16            <dependency>
17                <groupId>org.springframework.cloud</groupId>
18                <artifactId>spring-cloud-gcp-dependencies</artifactId>
19                <version>1.2.8.RELEASE</version>
20                <type>pom</type>
21                <scope>import</scope>
22            </dependency>
23
24        </dependencies>
25    </dependencyManagement>
26<dependency>
27            <groupId>com.google.cloud</groupId>
28            <artifactId>google-cloud-dataproc</artifactId>
29        </dependency>
30

1side_pipeline = (
2    p
3    | "periodic" >> PeriodicImpulse(fire_interval=3600, apply_windowing=True)
4    | "map to read request" >>
5        beam.Map(lambda x:beam.io.gcp.bigquery.ReadFromBigQueryRequest(table=side_table))
6    | beam.io.ReadAllFromBigQuery()
7)
8

1side_pipeline = (
2    p
3    | "periodic" >> PeriodicImpulse(fire_interval=3600, apply_windowing=True)
4    | "map to read request" >>
5        beam.Map(lambda x:beam.io.gcp.bigquery.ReadFromBigQueryRequest(table=side_table))
6    | beam.io.ReadAllFromBigQuery()
7)
8def enrich_payload(payload, equipments):
9    id = payload["id"]
10    for equipment in equipments:
11        if id == equipment["id"]:
12            payload["type"] = equipment["type"]
13            payload["brand"] = equipment["brand"]
14            payload["year"] = equipment["year"]
15
16            break
17
18    return payload
19

1side_pipeline = (
2    p
3    | "periodic" >> PeriodicImpulse(fire_interval=3600, apply_windowing=True)
4    | "map to read request" >>
5        beam.Map(lambda x:beam.io.gcp.bigquery.ReadFromBigQueryRequest(table=side_table))
6    | beam.io.ReadAllFromBigQuery()
7)
8def enrich_payload(payload, equipments):
9    id = payload["id"]
10    for equipment in equipments:
11        if id == equipment["id"]:
12            payload["type"] = equipment["type"]
13            payload["brand"] = equipment["brand"]
14            payload["year"] = equipment["year"]
15
16            break
17
18    return payload
19main_pipeline = (
20    p
21    | "read" >> beam.io.ReadFromPubSub(topic="projects/my-project/topics/topiq")
22    | "bytes to dict" >> beam.Map(lambda x: json.loads(x.decode("utf-8")))
23    | "transform" >> beam.Map(transform_function)
24    | "timestamping" >> beam.Map(lambda src: window.TimestampedValue(
25        src,
26        dt.datetime.fromisoformat(src["timestamp"]).timestamp()
27    ))
28    | "windowing" >> beam.WindowInto(window.FixedWindows(30))
29)
30
31final_pipeline = (
32    main_pipeline
33    | "enrich data" >> beam.Map(enrich_payload, equipments=beam.pvalue.AsIter(side_pipeline))
34    | "store" >> beam.io.WriteToBigQuery(bq_table)
35)
36
37result = p.run()
38result.wait_until_finish()
39

1echo Give yearmonth "yyyyMM"
2setlocal enabledelayedexpansion
3SET /p yearmonth= 
4SET ClientName[0]=abc
5SET ClientName[1]=def
6
7SET i = 0
8
9:myLoop
10if defined ClientName[%i%] (
11    call bq query --use_legacy_sql=false "CREATE EXTERNAL TABLE `test.!ClientName[%%i]!.%yearmonth%` OPTIONS (format = 'CSV',skip_leading_rows = 1 uris = ['gs://test/!ClientName[%%i]!/AWS/%yearmonth%/Metrics/data/*.csv'])"
12    set /a "i+=1"
13    GOTO :myLoop
14
15)
16

1from google_cloud_pipeline_components import aiplatform as gcc_aip
2from kfp.v2 import dsl
3
4from kfp.v2.dsl import component
5from kfp.v2.dsl import (
6    Output,
7    Artifact,
8    Model,
9)
10
11PROJECT_ID = 'my-gcp-project'
12BUCKET_NAME = "mybucket"
13PIPELINE_ROOT = "{}/pipeline_root".format(BUCKET_NAME)
14
15
16@component
17def get_input_data() -> str:
18    # getting data from API, save to Cloud Storage
19    # return GS URI
20    gcs_batch_input_path = 'gs://somebucket/file'
21    return gcs_batch_input_path
22
23
24@component(
25    base_image="python:3.9",
26    packages_to_install=['google-cloud-aiplatform==1.8.0']
27)
28def load_ml_model(project_id: str, model: Output[Artifact]):
29    """Load existing Vertex model"""
30    import google.cloud.aiplatform as aip
31
32    model_id = '1234'
33    model = aip.Model(model_name=model_id, project=project_id, location='us-central1')
34
35
36
37@dsl.pipeline(
38    name="batch-pipeline", pipeline_root=PIPELINE_ROOT,
39)
40def pipeline(gcp_project: str):
41    input_data = get_input_data()
42    ml_model = load_ml_model(gcp_project)
43
44    gcc_aip.ModelBatchPredictOp(
45        project=PROJECT_ID,
46        job_display_name=f'test-prediction',
47        model=ml_model.output,
48        gcs_source_uris=[input_data.output],  # this doesn't work
49        # gcs_source_uris=['gs://mybucket/output/'],  # hardcoded gs uri works
50        gcs_destination_output_uri_prefix=f'gs://{PIPELINE_ROOT}/prediction_output/'
51    )
52
53
54if __name__ == '__main__':
55    from kfp.v2 import compiler
56    import google.cloud.aiplatform as aip
57    pipeline_export_filepath = 'test-pipeline.json'
58    compiler.Compiler().compile(pipeline_func=pipeline,
59                                package_path=pipeline_export_filepath)
60    # pipeline_params = {
61    #     'gcp_project': PROJECT_ID,
62    # }
63    # job = aip.PipelineJob(
64    #     display_name='test-pipeline',
65    #     template_path=pipeline_export_filepath,
66    #     pipeline_root=f'gs://{PIPELINE_ROOT}',
67    #     project=PROJECT_ID,
68    #     parameter_values=pipeline_params,
69    # )
70
71    # job.run()
72

1from google_cloud_pipeline_components import aiplatform as gcc_aip
2from kfp.v2 import dsl
3
4from kfp.v2.dsl import component
5from kfp.v2.dsl import (
6    Output,
7    Artifact,
8    Model,
9)
10
11PROJECT_ID = 'my-gcp-project'
12BUCKET_NAME = "mybucket"
13PIPELINE_ROOT = "{}/pipeline_root".format(BUCKET_NAME)
14
15
16@component
17def get_input_data() -> str:
18    # getting data from API, save to Cloud Storage
19    # return GS URI
20    gcs_batch_input_path = 'gs://somebucket/file'
21    return gcs_batch_input_path
22
23
24@component(
25    base_image="python:3.9",
26    packages_to_install=['google-cloud-aiplatform==1.8.0']
27)
28def load_ml_model(project_id: str, model: Output[Artifact]):
29    """Load existing Vertex model"""
30    import google.cloud.aiplatform as aip
31
32    model_id = '1234'
33    model = aip.Model(model_name=model_id, project=project_id, location='us-central1')
34
35
36
37@dsl.pipeline(
38    name="batch-pipeline", pipeline_root=PIPELINE_ROOT,
39)
40def pipeline(gcp_project: str):
41    input_data = get_input_data()
42    ml_model = load_ml_model(gcp_project)
43
44    gcc_aip.ModelBatchPredictOp(
45        project=PROJECT_ID,
46        job_display_name=f'test-prediction',
47        model=ml_model.output,
48        gcs_source_uris=[input_data.output],  # this doesn't work
49        # gcs_source_uris=['gs://mybucket/output/'],  # hardcoded gs uri works
50        gcs_destination_output_uri_prefix=f'gs://{PIPELINE_ROOT}/prediction_output/'
51    )
52
53
54if __name__ == '__main__':
55    from kfp.v2 import compiler
56    import google.cloud.aiplatform as aip
57    pipeline_export_filepath = 'test-pipeline.json'
58    compiler.Compiler().compile(pipeline_func=pipeline,
59                                package_path=pipeline_export_filepath)
60    # pipeline_params = {
61    #     'gcp_project': PROJECT_ID,
62    # }
63    # job = aip.PipelineJob(
64    #     display_name='test-pipeline',
65    #     template_path=pipeline_export_filepath,
66    #     pipeline_root=f'gs://{PIPELINE_ROOT}',
67    #     project=PROJECT_ID,
68    #     parameter_values=pipeline_params,
69    # )
70
71    # job.run()
72   input_data = get_input_data2()
73   gcc_aip.ModelBatchPredictOp(
74        project=PROJECT_ID,
75        job_display_name=f'test-prediction',
76        model=ml_model.output,
77        gcs_source_uris=[input_data.output],  # this doesn't work
78        # gcs_source_uris=['gs://mybucket/output/'],  # hardcoded gs uri works
79        gcs_destination_output_uri_prefix=f'gs://{PIPELINE_ROOT}/prediction_output/'
80    )
81

1from google_cloud_pipeline_components import aiplatform as gcc_aip
2from kfp.v2 import dsl
3
4from kfp.v2.dsl import component
5from kfp.v2.dsl import (
6    Output,
7    Artifact,
8    Model,
9)
10
11PROJECT_ID = 'my-gcp-project'
12BUCKET_NAME = "mybucket"
13PIPELINE_ROOT = "{}/pipeline_root".format(BUCKET_NAME)
14
15
16@component
17def get_input_data() -> str:
18    # getting data from API, save to Cloud Storage
19    # return GS URI
20    gcs_batch_input_path = 'gs://somebucket/file'
21    return gcs_batch_input_path
22
23
24@component(
25    base_image="python:3.9",
26    packages_to_install=['google-cloud-aiplatform==1.8.0']
27)
28def load_ml_model(project_id: str, model: Output[Artifact]):
29    """Load existing Vertex model"""
30    import google.cloud.aiplatform as aip
31
32    model_id = '1234'
33    model = aip.Model(model_name=model_id, project=project_id, location='us-central1')
34
35
36
37@dsl.pipeline(
38    name="batch-pipeline", pipeline_root=PIPELINE_ROOT,
39)
40def pipeline(gcp_project: str):
41    input_data = get_input_data()
42    ml_model = load_ml_model(gcp_project)
43
44    gcc_aip.ModelBatchPredictOp(
45        project=PROJECT_ID,
46        job_display_name=f'test-prediction',
47        model=ml_model.output,
48        gcs_source_uris=[input_data.output],  # this doesn't work
49        # gcs_source_uris=['gs://mybucket/output/'],  # hardcoded gs uri works
50        gcs_destination_output_uri_prefix=f'gs://{PIPELINE_ROOT}/prediction_output/'
51    )
52
53
54if __name__ == '__main__':
55    from kfp.v2 import compiler
56    import google.cloud.aiplatform as aip
57    pipeline_export_filepath = 'test-pipeline.json'
58    compiler.Compiler().compile(pipeline_func=pipeline,
59                                package_path=pipeline_export_filepath)
60    # pipeline_params = {
61    #     'gcp_project': PROJECT_ID,
62    # }
63    # job = aip.PipelineJob(
64    #     display_name='test-pipeline',
65    #     template_path=pipeline_export_filepath,
66    #     pipeline_root=f'gs://{PIPELINE_ROOT}',
67    #     project=PROJECT_ID,
68    #     parameter_values=pipeline_params,
69    # )
70
71    # job.run()
72   input_data = get_input_data2()
73   gcc_aip.ModelBatchPredictOp(
74        project=PROJECT_ID,
75        job_display_name=f'test-prediction',
76        model=ml_model.output,
77        gcs_source_uris=[input_data.output],  # this doesn't work
78        # gcs_source_uris=['gs://mybucket/output/'],  # hardcoded gs uri works
79        gcs_destination_output_uri_prefix=f'gs://{PIPELINE_ROOT}/prediction_output/'
80    )
81google-cloud-aiplatform==1.8.0  
82google-cloud-pipeline-components==0.2.0  
83kfp==1.8.10  
84kfp-pipeline-spec==0.1.13  
85kfp-server-api==1.7.1
86

1from google_cloud_pipeline_components import aiplatform as gcc_aip
2from kfp.v2 import dsl
3
4from kfp.v2.dsl import component
5from kfp.v2.dsl import (
6    Output,
7    Artifact,
8    Model,
9)
10
11PROJECT_ID = 'my-gcp-project'
12BUCKET_NAME = "mybucket"
13PIPELINE_ROOT = "{}/pipeline_root".format(BUCKET_NAME)
14
15
16@component
17def get_input_data() -> str:
18    # getting data from API, save to Cloud Storage
19    # return GS URI
20    gcs_batch_input_path = 'gs://somebucket/file'
21    return gcs_batch_input_path
22
23
24@component(
25    base_image="python:3.9",
26    packages_to_install=['google-cloud-aiplatform==1.8.0']
27)
28def load_ml_model(project_id: str, model: Output[Artifact]):
29    """Load existing Vertex model"""
30    import google.cloud.aiplatform as aip
31
32    model_id = '1234'
33    model = aip.Model(model_name=model_id, project=project_id, location='us-central1')
34
35
36
37@dsl.pipeline(
38    name="batch-pipeline", pipeline_root=PIPELINE_ROOT,
39)
40def pipeline(gcp_project: str):
41    input_data = get_input_data()
42    ml_model = load_ml_model(gcp_project)
43
44    gcc_aip.ModelBatchPredictOp(
45        project=PROJECT_ID,
46        job_display_name=f'test-prediction',
47        model=ml_model.output,
48        gcs_source_uris=[input_data.output],  # this doesn't work
49        # gcs_source_uris=['gs://mybucket/output/'],  # hardcoded gs uri works
50        gcs_destination_output_uri_prefix=f'gs://{PIPELINE_ROOT}/prediction_output/'
51    )
52
53
54if __name__ == '__main__':
55    from kfp.v2 import compiler
56    import google.cloud.aiplatform as aip
57    pipeline_export_filepath = 'test-pipeline.json'
58    compiler.Compiler().compile(pipeline_func=pipeline,
59                                package_path=pipeline_export_filepath)
60    # pipeline_params = {
61    #     'gcp_project': PROJECT_ID,
62    # }
63    # job = aip.PipelineJob(
64    #     display_name='test-pipeline',
65    #     template_path=pipeline_export_filepath,
66    #     pipeline_root=f'gs://{PIPELINE_ROOT}',
67    #     project=PROJECT_ID,
68    #     parameter_values=pipeline_params,
69    # )
70
71    # job.run()
72   input_data = get_input_data2()
73   gcc_aip.ModelBatchPredictOp(
74        project=PROJECT_ID,
75        job_display_name=f'test-prediction',
76        model=ml_model.output,
77        gcs_source_uris=[input_data.output],  # this doesn't work
78        # gcs_source_uris=['gs://mybucket/output/'],  # hardcoded gs uri works
79        gcs_destination_output_uri_prefix=f'gs://{PIPELINE_ROOT}/prediction_output/'
80    )
81google-cloud-aiplatform==1.8.0  
82google-cloud-pipeline-components==0.2.0  
83kfp==1.8.10  
84kfp-pipeline-spec==0.1.13  
85kfp-server-api==1.7.1
86@component
87def load_ml_model(project_id: str, model: Output[Artifact]):
88    region = 'us-central1'
89    model_id = '1234'
90    model_uid = f'projects/{project_id}/locations/{region}/models/{model_id}'
91    model.uri = model_uid
92    model.metadata['resourceName'] = model_uid
93

1from google_cloud_pipeline_components import aiplatform as gcc_aip
2from kfp.v2 import dsl
3
4from kfp.v2.dsl import component
5from kfp.v2.dsl import (
6    Output,
7    Artifact,
8    Model,
9)
10
11PROJECT_ID = 'my-gcp-project'
12BUCKET_NAME = "mybucket"
13PIPELINE_ROOT = "{}/pipeline_root".format(BUCKET_NAME)
14
15
16@component
17def get_input_data() -> str:
18    # getting data from API, save to Cloud Storage
19    # return GS URI
20    gcs_batch_input_path = 'gs://somebucket/file'
21    return gcs_batch_input_path
22
23
24@component(
25    base_image="python:3.9",
26    packages_to_install=['google-cloud-aiplatform==1.8.0']
27)
28def load_ml_model(project_id: str, model: Output[Artifact]):
29    """Load existing Vertex model"""
30    import google.cloud.aiplatform as aip
31
32    model_id = '1234'
33    model = aip.Model(model_name=model_id, project=project_id, location='us-central1')
34
35
36
37@dsl.pipeline(
38    name="batch-pipeline", pipeline_root=PIPELINE_ROOT,
39)
40def pipeline(gcp_project: str):
41    input_data = get_input_data()
42    ml_model = load_ml_model(gcp_project)
43
44    gcc_aip.ModelBatchPredictOp(
45        project=PROJECT_ID,
46        job_display_name=f'test-prediction',
47        model=ml_model.output,
48        gcs_source_uris=[input_data.output],  # this doesn't work
49        # gcs_source_uris=['gs://mybucket/output/'],  # hardcoded gs uri works
50        gcs_destination_output_uri_prefix=f'gs://{PIPELINE_ROOT}/prediction_output/'
51    )
52
53
54if __name__ == '__main__':
55    from kfp.v2 import compiler
56    import google.cloud.aiplatform as aip
57    pipeline_export_filepath = 'test-pipeline.json'
58    compiler.Compiler().compile(pipeline_func=pipeline,
59                                package_path=pipeline_export_filepath)
60    # pipeline_params = {
61    #     'gcp_project': PROJECT_ID,
62    # }
63    # job = aip.PipelineJob(
64    #     display_name='test-pipeline',
65    #     template_path=pipeline_export_filepath,
66    #     pipeline_root=f'gs://{PIPELINE_ROOT}',
67    #     project=PROJECT_ID,
68    #     parameter_values=pipeline_params,
69    # )
70
71    # job.run()
72   input_data = get_input_data2()
73   gcc_aip.ModelBatchPredictOp(
74        project=PROJECT_ID,
75        job_display_name=f'test-prediction',
76        model=ml_model.output,
77        gcs_source_uris=[input_data.output],  # this doesn't work
78        # gcs_source_uris=['gs://mybucket/output/'],  # hardcoded gs uri works
79        gcs_destination_output_uri_prefix=f'gs://{PIPELINE_ROOT}/prediction_output/'
80    )
81google-cloud-aiplatform==1.8.0  
82google-cloud-pipeline-components==0.2.0  
83kfp==1.8.10  
84kfp-pipeline-spec==0.1.13  
85kfp-server-api==1.7.1
86@component
87def load_ml_model(project_id: str, model: Output[Artifact]):
88    region = 'us-central1'
89    model_id = '1234'
90    model_uid = f'projects/{project_id}/locations/{region}/models/{model_id}'
91    model.uri = model_uid
92    model.metadata['resourceName'] = model_uid
93batch_predict_op = gcc_aip.ModelBatchPredictOp(
94        project=gcp_project,
95        job_display_name=f'batch-prediction-test',
96        model=ml_model.outputs['model'],
97        gcs_source_uris=[input_batch_gcs_path],
98gcs_destination_output_uri_prefix=f'gs://{BUCKET_NAME}/prediction_output/test'
99    )
100

1from google_cloud_pipeline_components import aiplatform as gcc_aip
2from kfp.v2 import dsl
3
4from kfp.v2.dsl import component
5from kfp.v2.dsl import (
6    Output,
7    Artifact,
8    Model,
9)
10
11PROJECT_ID = 'my-gcp-project'
12BUCKET_NAME = "mybucket"
13PIPELINE_ROOT = "{}/pipeline_root".format(BUCKET_NAME)
14
15
16@component
17def get_input_data() -> str:
18    # getting data from API, save to Cloud Storage
19    # return GS URI
20    gcs_batch_input_path = 'gs://somebucket/file'
21    return gcs_batch_input_path
22
23
24@component(
25    base_image="python:3.9",
26    packages_to_install=['google-cloud-aiplatform==1.8.0']
27)
28def load_ml_model(project_id: str, model: Output[Artifact]):
29    """Load existing Vertex model"""
30    import google.cloud.aiplatform as aip
31
32    model_id = '1234'
33    model = aip.Model(model_name=model_id, project=project_id, location='us-central1')
34
35
36
37@dsl.pipeline(
38    name="batch-pipeline", pipeline_root=PIPELINE_ROOT,
39)
40def pipeline(gcp_project: str):
41    input_data = get_input_data()
42    ml_model = load_ml_model(gcp_project)
43
44    gcc_aip.ModelBatchPredictOp(
45        project=PROJECT_ID,
46        job_display_name=f'test-prediction',
47        model=ml_model.output,
48        gcs_source_uris=[input_data.output],  # this doesn't work
49        # gcs_source_uris=['gs://mybucket/output/'],  # hardcoded gs uri works
50        gcs_destination_output_uri_prefix=f'gs://{PIPELINE_ROOT}/prediction_output/'
51    )
52
53
54if __name__ == '__main__':
55    from kfp.v2 import compiler
56    import google.cloud.aiplatform as aip
57    pipeline_export_filepath = 'test-pipeline.json'
58    compiler.Compiler().compile(pipeline_func=pipeline,
59                                package_path=pipeline_export_filepath)
60    # pipeline_params = {
61    #     'gcp_project': PROJECT_ID,
62    # }
63    # job = aip.PipelineJob(
64    #     display_name='test-pipeline',
65    #     template_path=pipeline_export_filepath,
66    #     pipeline_root=f'gs://{PIPELINE_ROOT}',
67    #     project=PROJECT_ID,
68    #     parameter_values=pipeline_params,
69    # )
70
71    # job.run()
72   input_data = get_input_data2()
73   gcc_aip.ModelBatchPredictOp(
74        project=PROJECT_ID,
75        job_display_name=f'test-prediction',
76        model=ml_model.output,
77        gcs_source_uris=[input_data.output],  # this doesn't work
78        # gcs_source_uris=['gs://mybucket/output/'],  # hardcoded gs uri works
79        gcs_destination_output_uri_prefix=f'gs://{PIPELINE_ROOT}/prediction_output/'
80    )
81google-cloud-aiplatform==1.8.0  
82google-cloud-pipeline-components==0.2.0  
83kfp==1.8.10  
84kfp-pipeline-spec==0.1.13  
85kfp-server-api==1.7.1
86@component
87def load_ml_model(project_id: str, model: Output[Artifact]):
88    region = 'us-central1'
89    model_id = '1234'
90    model_uid = f'projects/{project_id}/locations/{region}/models/{model_id}'
91    model.uri = model_uid
92    model.metadata['resourceName'] = model_uid
93batch_predict_op = gcc_aip.ModelBatchPredictOp(
94        project=gcp_project,
95        job_display_name=f'batch-prediction-test',
96        model=ml_model.outputs['model'],
97        gcs_source_uris=[input_batch_gcs_path],
98gcs_destination_output_uri_prefix=f'gs://{BUCKET_NAME}/prediction_output/test'
99    )
100@dsl.pipeline(
101    name="my-pipeline",
102    pipeline_root=PIPELINE_ROOT,
103)
104def pipeline(
105        gcp_project: str,
106        region: str,
107        bucket: str
108):
109    ts = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
110    
111    gcs_prediction_input_path = f'gs://{BUCKET_NAME}/prediction_input/video_batch_prediction_input_{ts}.jsonl'
112    batch_input_data_op = get_input_data(gcs_prediction_input_path)  # this loads input data to GCS path
113
114    batch_predict_op = gcc_aip.ModelBatchPredictOp(
115        project=gcp_project,
116        model=training_job_run_op.outputs["model"],
117        job_display_name='batch-prediction',
118        # gcs_source_uris=[batch_input_data_op.output],
119        gcs_source_uris=[gcs_prediction_input_path],
120        gcs_destination_output_uri_prefix=f'gs://{BUCKET_NAME}/prediction_output/',
121    ).after(batch_input_data_op)  # we need to add 'after' so it runs after input data is prepared since get_input_data doesn't returns anything
122
123