python-docs-samples | Code samples used on cloud.google.com | GCP library

'HOST':  '/cloudsql/asim800:us-central1:django-mysql-1'

OperationalError: (2002, "Can't connect to local MySQL server through socket '/cloudsql/asim800:us-central1:django-mysql-1'

    DATABASES = {
        'default': {
                'ENGINE': 'django.db.backends.mysql',
                'HOST': '/cloudsql/asim800:us-central1:django-mysql-1', 
                'NAME': 'myproject',
                'USER': 'pblogsu',
                'PASSWORD': '****',
        }
    }

runtime: python38

Flask==2.0.0;
google-cloud
google-cloud-storage

OSError: [Errno 30] Read-only file system: 'file.xlsx'

from flask import Flask, request
from werkzeug.utils import secure_filename
from google.cloud import storage
import os

app = Flask(__name__)
app.config['UPLOAD_FOLDER'] = "/tmp/"

@app.route('/')
def index():
    page = '''
    <form action="/upload" method="post" enctype="multipart/form-data">
        <input name="file" class="custom-file-input" type="file">
        <button type="submit" class="btn btn-success">Submit</button>
    </form>
    '''
    return page

@app.route('/upload',methods=["POST"])
def upload():
    try:
        if request.method == 'POST':
            f = request.files['file']
            f.save(os.path.join(app.config['UPLOAD_FOLDER'],secure_filename(f.filename)))
            
            #Start Do Main Process Here
            TMP_FILE_NAME = os.path.join(app.config['UPLOAD_FOLDER'],secure_filename(f.filename))
            BUCKET_NAME = 'fsample123'
            result = upload_blob(BUCKET_NAME, TMP_FILE_NAME, f"tmp/{secure_filename(f.filename)}")
            #End Do Main Process Here
            return result
            
    except Exception as e:
        return str(e)

def upload_blob(bucket_name, source_file_name, destination_blob_name):
    """Uploads a file to the bucket."""
    # SOURCE:
    # https://cloud.google.com/storage/docs/uploading-objects#uploading-an-object

    # The ID of your GCS bucket
    # bucket_name = "your-bucket-name"
    # The path to your file to upload
    # source_file_name = "local/path/to/file"
    # The ID of your GCS object
    # destination_blob_name = "storage-object-name"

    storage_client = storage.Client()
    bucket = storage_client.bucket(bucket_name)
    blob = bucket.blob(destination_blob_name)
    blob.upload_from_filename(source_file_name)
    return "File {} uploaded to {}.".format(source_file_name, destination_blob_name)

if __name__ == '__main__':
   app.run()

Broken DAG: [/home/airflow/gcs/dags/composer-dataproc-dag.py] cannot import name 'email' 

import datetime

from airflow import models
from airflow.contrib.operators import dataproc_operator
from airflow.utils.dates import days_ago
from airflow.operators import email

project_id = models.Variable.get("project_id")
yesterday = datetime.datetime.now() - datetime.timedelta(days=1)

default_args = {
    'owner': 'airflow',
    'start_date': yesterday,
    'depends_on_past': False,
    'email': models.Variable.get("email"),
    'email_on_failure': True,
    'email_on_retry': False,
    'retries': 1,
    'retry_delay': datetime.timedelta(minutes=5),
}

# Define a DAG (directed acyclic graph) of tasks.
# Any task you create within the context manager is automatically added to the
# DAG object.
with models.DAG(
    # The id you will see in the DAG airflow page
    "dataproc_workflow_dag",
    default_args=default_args,
    # The interval with which to schedule the DAG
    schedule_interval=datetime.timedelta(days=1),  # Override to match your needs
) as dag:

    start_template_job = dataproc_operator.DataprocWorkflowTemplateInstantiateOperator(
        # The task id of your job
        task_id="dataproc_workflow_dag",
        # The template id of your workflow
        template_id="mywf1",
        project_id=project_id,
        # The region for the template
        # For more info on regions where Dataflow is available see:
        # https://cloud.google.com/dataflow/docs/resources/locations
        region="us-central1",
    )
    email_summary = email.EmailOperator(
        task_id='email_summary',
        to=models.Variable.get('email'),
        subject='testing email service of gcp',
        html_content="""
        Analyzed the Requirements and have created a test POC.Lets see If it works out!
        """
        )

stream {
    upstream google_mqtt {
            server mgtt.googleapis.com:8883;
    }
    server {
            listen 8883;
            proxy_pass google_mqtt;
    }
}

ssl.SSLCertVerificationError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: Hostname mismatch, certificate is not valid for 'fake.mqtt.com' 

def create_jwt(project_id, private_key_file, algorithm):
    token = {
        # The time that the token was issued at
        "iat": datetime.datetime.utcnow(),
        # The time the token expires.
        "exp": datetime.datetime.utcnow() + datetime.timedelta(minutes=20),
        # The audience field should always be set to the GCP project id.
        "aud": project_id,
    }

    # Read the private key file.
    with open(private_key_file, "r") as f:
        private_key = f.read()

    print(
        "Creating JWT using {} from private key file {}".format(
            algorithm, private_key_file
        )
    )
    return jwt.encode(token, private_key, algorithm=algorithm)

client.username_pw_set(
            username='unused',
            password=create_jwt(project_id, private_key_file, algorithm))

client.tls_set(ca_certs='roots.pem', tls_version=ssl.PROTOCOL_TLSv1_2,)

Processing /var/opt/google/dataflow/setuptools-51.1.1.tar.gz
ERROR: Could not find a version that satisfies the requirement wheel (from versions: none)
ERROR: No matching distribution found for wheel
ERROR: Command errored out with exit status 1: /usr/local/bin/python3 /usr/local/lib/python3.7/site-packages/pip install --ignore-installed --no-user --prefix /tmp/pip-build-env-rvfx0hn9/overlay --no-warn-script-location --no-binary :none: --only-binary :none: --no-index --find-links /var/opt/google/dataflow -- 'setuptools>=40.8.0' wheel Check the logs for full command output."

ndjson==0.3.1
apache-beam[gcp]==2.26.0
dataclasses-json==0.5.2
google-cloud-storage==1.35.0
google-cloud-secret-manager==2.0.0

FROM gcr.io/dataflow-templates-base/python3-template-launcher-base

ARG WORKDIR=/dataflow/template
RUN mkdir -p ${WORKDIR}
WORKDIR ${WORKDIR}

# Due to a change in the Beam base image in version 2.24, we need to install
# libffi-dev manually as a dependency. For more information:
#   https://github.com/GoogleCloudPlatform/python-docs-samples/issues/4891
RUN apt-get update && apt-get install -y libffi-dev git && rm -rf /var/lib/apt/lists/*

ADD utils utils
COPY requirements.txt .
COPY main.py .

ENV FLEX_TEMPLATE_PYTHON_REQUIREMENTS_FILE="${WORKDIR}/requirements.txt"
ENV FLEX_TEMPLATE_PYTHON_PY_FILE="${WORKDIR}/main.py"

RUN pip install -U apache-beam[gcp]==2.24.0
RUN pip install -U -r ./requirements.txt

# Specify a network interface with NAT to access the public
# internet.
'networkInterfaces': [            
        {
        "kind": "compute#networkInterface",
        "subnetwork": "projects/PROJECT-NAME/regions/europe-west3/subnetworks/default",
        "aliasIpRanges": []
        }
],

usage: id.py [-h] project_id location composer_environment
id.py: error: the following arguments are required: project_id, location, composer_environment

python3 id.py --project_id project_id --location location --composer_environment composer_environment

python3 id.py --project_id 'project_id' --location 'location' --composer_environment 'composer_environment'

python3 id.py --project_id --location --composer_environment

"""Get the client ID associated with a Cloud Composer environment."""

import argparse


def get_client_id(project_id, location, composer_environment):
    # [START composer_get_environment_client_id]
    import google.auth
    import google.auth.transport.requests
    import requests
    import six.moves.urllib.parse

    # Authenticate with Google Cloud.
    # See: https://cloud.google.com/docs/authentication/getting-started
    credentials, _ = google.auth.default(
        scopes=['https://www.googleapis.com/auth/cloud-platform'])
    authed_session = google.auth.transport.requests.AuthorizedSession(
        credentials)

    # project_id = 'YOUR_PROJECT_ID'
    # location = 'us-central1'
    # composer_environment = 'YOUR_COMPOSER_ENVIRONMENT_NAME'

    environment_url = (
        'https://composer.googleapis.com/v1beta1/projects/{}/locations/{}'
        '/environments/{}').format(project_id, location, composer_environment)
    composer_response = authed_session.request('GET', environment_url)
    environment_data = composer_response.json()
    airflow_uri = environment_data['config']['airflowUri']

    # The Composer environment response does not include the IAP client ID.
    # Make a second, unauthenticated HTTP request to the web server to get the
    # redirect URI.
    redirect_response = requests.get(airflow_uri, allow_redirects=False)
    redirect_location = redirect_response.headers['location']

    # Extract the client_id query parameter from the redirect.
    parsed = six.moves.urllib.parse.urlparse(redirect_location)
    query_string = six.moves.urllib.parse.parse_qs(parsed.query)
    print(query_string['client_id'][0])
    # [END composer_get_environment_client_id]


if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument('project_id', help='Your Project ID.')
    parser.add_argument(
        'location', help='Region of the Cloud Composer environent.')
    parser.add_argument(
        'composer_environment', help='Name of the Cloud Composer environent.')

    args = parser.parse_args()
    get_client_id(
        args.project_id, args.location, args.composer_environment)

Processing ./dist/imt_ai_predict_batch-0.1.tar.gz
    ERROR: Command errored out with exit status 1:
     command: /Users/amrit.tiwary/anaconda3/envs/mini_imt2/bin/python -c 'import sys, setuptools, tokenize; sys.argv[0] = '"'"'/private/var/folders/mj/b_g4b4yj26q278g9k58w92fck5d9wj/T/pip-req-build-9dmmje_z/setup.py'"'"'; __file__='"'"'/private/var/folders/mj/b_g4b4yj26q278g9k58w92fck5d9wj/T/pip-req-build-9dmmje_z/setup.py'"'"';f=getattr(tokenize, '"'"'open'"'"', open)(__file__);code=f.read().replace('"'"'\r\n'"'"', '"'"'\n'"'"');f.close();exec(compile(code, __file__, '"'"'exec'"'"'))' egg_info --egg-base /private/var/folders/mj/b_g4b4yj26q278g9k58w92fck5d9wj/T/pip-pip-egg-info-ic3qhcpo
         cwd: /private/var/folders/mj/b_g4b4yj26q278g9k58w92fck5d9wj/T/pip-req-build-9dmmje_z/
    Complete output (6 lines):
    Parent directory: /private/var/folders/mj/b_g4b4yj26q278g9k58w92fck5d9wj/T/pip-req-build-9dmmje_z
    Traceback (most recent call last):
      File "<string>", line 1, in <module>
      File "/private/var/folders/mj/b_g4b4yj26q278g9k58w92fck5d9wj/T/pip-req-build-9dmmje_z/setup.py", line 22, in <module>
        REQUIRED_PACKAGES = [line.strip() for line in open(base/"requirements.txt")] + \
    FileNotFoundError: [Errno 2] No such file or directory: '/private/var/folders/mj/b_g4b4yj26q278g9k58w92fck5d9wj/T/pip-req-build-9dmmje_z/requirements.txt'
    ----------------------------------------
ERROR: Command errored out with exit status 1: python setup.py egg_info Check the logs for full command output.

Parent directory: .

Packages: ['numpy==1.17.3', 'pandas==1.2.1', 'setuptools==53.0.0', 'Pillow==6.2.1', 'torchvision==0.5.0', 'torch @ https://download.pytorch.org/whl/cpu/torch-1.4.0%2Bcpu-cp37-cp37m-linux_x86_64.whl']


running egg_info
writing imt_ai_predict_batch.egg-info/PKG-INFO
writing dependency_links to imt_ai_predict_batch.egg-info/dependency_links.txt
writing requirements to imt_ai_predict_batch.egg-info/requires.txt
writing top-level names to imt_ai_predict_batch.egg-info/top_level.txt
reading manifest file 'imt_ai_predict_batch.egg-info/SOURCES.txt'
writing manifest file 'imt_ai_predict_batch.egg-info/SOURCES.txt'

from pathlib import Path
"""
Purpose: Package your Predictor and its dependencies by packaging  Predictor as a .tar.gz.

For deploying any PyTorch model, the PyTorch package is needed to be provided in the below-mentioned command. In this case, 
we have used the package:
torch-1.3.1+cpu-cp37-cp37m-linux_x86_64.whl
For getting pytorch packages according to your requirements,
Visit this link: Pytorch Packages(google search)
Download the package you require and store it in the GCS bucket.
The package we are using provides version resource with PyTorch 1.3.1 for Python 3.7, built to run on a CPU in Linux.

NOTE: We could have directly added ‘torch’ in the ‘REQUIRED_PACKAGES’ list in setup.py file in order to provide pytorch as a dependency to be installed while deployment.
This way, it uses the PyPI installation of pytorch.
But the issue with this approach is that it internally downloads PyPI package for pytorch which is of 720 MB Which causes the failure of our model deployment because the AI platform allows custom models of 500MB or below to be deployed on it.
So it is recommended to provide a pytorch package manually using gsutil command under — package-uris.
Try: pip install --target=/tmp/custom_lib --no-cache-dir -b /tmp/pip_builds imt_ai_predict-0.4.tar.gz for troubleshooting
"""
base = Path(__file__).parent
print(f"Parent directory: {base}")
REQUIRED_PACKAGES = [line.strip() for line in open(base/"requirements.txt")] + \
['torchvision==0.5.0', 'torch @ https://download.pytorch.org/whl/cpu/torch-1.4.0%2Bcpu-cp37-cp37m-linux_x86_64.whl']

print(f"\nPackages: {REQUIRED_PACKAGES}\n\n")

setup(description="Extract features of a image",
      author='Amrit',
      author_email='amrit.tiwary@mindshareworld.com',
      name='imt_ai_predict_batch',
      version='0.1',
      url='https://msww-uat-app-imagemapping.cl01.k8s.mindshareapps.com/',
      install_requires=REQUIRED_PACKAGES,
      project_urls={
          'Documentation': 'https://cloud.google.com/ai-platform/prediction/docs/custom-prediction-routines#tensorflow',
          'Deploy': 'https://cloud.google.com/ai-platform/prediction/docs/deploying-models#gcloud_1',
          'Ai_platform troubleshooting': 'https://cloud.google.com/ai-platform/training/docs/troubleshooting',
          'Say Thanks!': ['https://medium.com/searce/deploy-your-own-custom-model-on-gcps-ai-platform-7e42a5721b43',
                          'https://stackoverflow.com/questions/61933879/unexpected-error-when-loading-the-model-problem-in-predictor-modulenotfounder',
                          'https://stackoverflow.com/questions/56428037/what-is-the-best-way-to-save-pil-image-in-json'],
          'google Torch wheels': "http://storage.googleapis.com/cloud-ai-pytorch/readme.txt",
          'Torch & torchvision wheels': "https://download.pytorch.org/whl/torch_stable.html ",
          'Raised issue with google github branch': "https://github.com/GoogleCloudPlatform/python-docs-samples/issues/5257",
          "Stackoverflow": "https://stackoverflow.com/questions/65795374/gcp-ai-platform-error-when-creating-a-custom-predictor-model-version-trained"
      },
    python_requires='~=3.7',
    scripts=['predictor.py', 'preproc.py'])```
Have tried the usual upgrading python setuptools to 52.0.0, but can't seem to shake this error off. 

python3 cloudiot_pubsub_example_server.py --project_id=project-aura-249003 --pubsub_subscription=temp

python3 cloudiot_pubsub_example_mqtt_device.py --project_id=project-aura-249003 --registry_id=Linux_PC --device_id=linux_pc --private_key_file=rsa_private.pem --algorithm=RS256 --ca_certs=roots.pem --cloud_region=asia-east1