TFServing-setup-tutorial | basic tutorial for Tensorflow Serving | Learning library

grpc API for python. get model status and reload model using grpc.
setup environment pip install numpy tensorflow tensorflow-serving-api grpcio
grpc API for python grpc API predict.proto syntax = "proto3"; package tensorflow.serving; option cc_enable_arenas = true; import "tensorflow/core/framework/tensor.proto"; import "tensorflow_serving/apis/model.proto"; // PredictRequest specifies which TensorFlow model to run, as well as // how inputs are mapped to tensors and how outputs are filtered before // returning to user. message PredictRequest { // Model Specification. If version is not specified, will use the latest // (numerical) version. ModelSpec model_spec = 1; # for python `request.model_spec` // Input tensors. // Names of input tensor are alias names. The mapping from aliases to real // input tensor names is stored in the SavedModel export as a prediction // SignatureDef under the 'inputs' field. map<string, TensorProto> inputs = 2; # for python `request.input` dictionary // Output filter. // Names specified are alias names. The mapping from aliases to real output // tensor names is stored in the SavedModel export as a prediction // SignatureDef under the 'outputs' field. // Only tensors specified here will be run/fetched and returned, with the // exception that when none is specified, all tensors specified in the // named signature will be run/fetched and returned. repeated string output_filter = 3; # for python `request.output_filter` list need to append values. } // Response for PredictRequest on successful run. message PredictResponse { // Effective Model Specification used to process PredictRequest. ModelSpec model_spec = 2; // Output tensors. map<string, TensorProto> outputs = 1; } model.proto syntax = "proto3"; package tensorflow.serving; option cc_enable_arenas = true; import "google/protobuf/wrappers.proto"; // Metadata for an inference request such as the model name and version. message ModelSpec { // Required servable name. string name = 1; // Optional choice of which version of the model to use. // // Recommended to be left unset in the common case. Should be specified only // when there is a strong version consistency requirement. // // When left unspecified, the system will serve the best available version. // This is typically the latest version, though during version transitions, // notably when serving on a fleet of instances, may be either the previous or // new version. # for this `request.model_spec.version.value` # or `request.model_spec.version_label` oneof version_choice { // Use this specific version number. google.protobuf.Int64Value version = 2; // Use the version associated with the given label. string version_label = 4; } // A named signature to evaluate. If unspecified, the default signature will // be used. # for python `request.model_spec.signature_name` string signature_name = 3; }
run grpcRequest.py python3 grpcRequest.py # outputs { # key: "dense" # value { # dtype: DT_FLOAT # tensor_shape { # dim { # size: 2 # } # dim { # size: 1 # } # } # float_val: 0.9901617765426636 # float_val: 0.9934704303741455 # } # } # model_spec { # name: "Toy" # version { # value: 3 # } # signature_name: "serving_default" # }
get model status and reload model using grpc run server $ docker run --rm -p 8500:8500 -p 8501:8501 --mount type=bind,source=$(pwd)/save/,target=/models/save --mount type=bind,srce=$(pwd)/config/versionlabels.config,target=/models/versionctrl.config -it tensorflow/serving --model_config_file=/models/versionctrl.config --model_config_file_poll_wait_seconds=60 --allow_version_labels_for_unavailable_models obtain model status info $ python grpcModelStatus.py -m Toy -v 1 # model_version_status { # version: 1 # state: AVAILABLE # status { # } # } reload config file $ python grpcReloadModel.py -m Toy # model Toy reloaded sucessfully from Server # 2020-05-28 10:38:51.057588: I tensorflow_serving/model_servers/model_service_impl.cc:47] # Config entry # index : 0 # path : /models/save/Toy/ # name : Toy # platform : tensorflow # 2020-05-28 10:38:51.057775: I tensorflow_serving/model_servers/server_core.cc:462] Adding/updating models. # 2020-05-28 10:38:51.057843: I tensorflow_serving/model_servers/server_core.cc:573] (Re-)adding model: Toy # 2020-05-28 10:38:51.156301: I tensorflow_serving/core/loader_harness.cc:138] Quiescing servable version {name: Toy_double version: 1} # 2020-05-28 10:38:51.156406: I tensorflow_serving/core/loader_harness.cc:145] Done quiescing servable version {name: Toy_double version: 1} # 2020-05-28 10:38:51.156442: I tensorflow_serving/core/loader_harness.cc:120] Unloading servable version {name: Toy_double version: 1} # ... # 2020-05-28 10:38:52.083807: I tensorflow_serving/core/loader_harness.cc:128] Done unloading servable version {name: Toy version: 2}
RESTful API
run POSTreq.py python3 POSTreq.py # this request is based on isntances # True # { # "predictions": [[0.990161777], [0.99347043] # ] # } # time consumption: 47.346710999999985ms # this request is based on inputs # True # { # "outputs": [ # [ # 0.985201657 # ], # [ # 0.99923408 # ] # ] # } # time consumption: 6.932738000000049ms