This page was generated from examples/models/metrics/metrics.ipynb.

Basic Examples with Different Protocols Showing Metrics

Prerequisites

• A kubernetes cluster with kubectl configured

• curl

• grpcurl

• pygmentize

Setup Seldon Core

Install Seldon Core as described in docs

Then port-forward to that ingress on localhost:8003 in a separate terminal either with:

• Ambassador:

bash  kubectl port-forward $(kubectl get pods -n seldon -l app.kubernetes.io/name=ambassador -o jsonpath='{.items[0].metadata.name}') -n seldon 8003:8080 * Istio:

bash  kubectl port-forward $(kubectl get pods -l istio=ingressgateway -n istio-system -o jsonpath='{.items[0].metadata.name}') -n istio-system 8003:80

!kubectl create namespace seldon

!kubectl config set-context $(kubectl config current-context) --namespace=seldon

Install Seldon Analytics

!helm install seldon-core-analytics ../../../helm-charts/seldon-core-analytics  \
        --set grafana_prom_admin_password=password \
        --set persistence.enabled=false \
        --namespace seldon-system \
        --wait

Port forward to the Grafana dashboard

kubectl port-forward $(kubectl get pods -n seldon-system  -l app.kubernetes.io/name=grafana -o jsonpath='{.items[0].metadata.name}') 3000:3000 -n seldon-system

%env RESOURCES=../../../notebooks/resources

Seldon Protocol REST Model

Make sure your active namespace is seldon

!pygmentize ${RESOURCES}/model_seldon_rest.yaml

!kubectl apply -f ${RESOURCES}/model_seldon_rest.yaml -n seldon

!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=rest-seldon \
                                 -o jsonpath='{.items[0].metadata.name}')

!for i in `seq 1 60`; do \
   sleep 1 && curl -d '{"data": {"ndarray":[[1.0, 2.0, 5.0]]}}' \
   -X POST http://localhost:8003/seldon/seldon/rest-seldon/api/v1.0/predictions \
   -H "Content-Type: application/json"; \
done

import time
for i in range(3):
    metric=!curl -s http://localhost:8003/seldon/seldon/rest-seldon/prometheus | grep seldon_api_executor_server_requests_seconds_count
    if metric and len(metric)>0:
        print(metric[0])
        assert(not metric[0] == "")
        break
    else:
        print("Failed to get metrics for rest-seldon")
        time.sleep(2)

!kubectl delete -f ${RESOURCES}/model_seldon_rest.yaml -n seldon

Seldon Protocol GRPC Model

!pygmentize ${RESOURCES}/model_seldon_grpc.yaml

!kubectl apply -f ${RESOURCES}/model_seldon_grpc.yaml -n seldon

!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=grpc-seldon \
                                 -o jsonpath='{.items[0].metadata.name}')

!cd ../../../executor/proto && for i in `seq 1 60`; do \
 sleep 1 && grpcurl -d '{"data":{"ndarray":[[1.0,2.0]]}}' \
         -rpc-header seldon:grpc-seldon -rpc-header namespace:seldon \
         -plaintext \
         -proto ./prediction.proto  0.0.0.0:8003 seldon.protos.Seldon/Predict; \
done

for i in range(3):
    metric=!curl -s http://localhost:8003/seldon/seldon/grpc-seldon/prometheus | grep seldon_api_executor_server_requests_seconds_count
    if metric and len(metric)>0:
        print(metric[0])
        assert(not metric[0] == "")
        break
    else:
        print("Failed to get metrics for grpc-seldon")
        time.sleep(2)

!kubectl delete -f ${RESOURCES}/model_seldon_grpc.yaml -n seldon

Tensorflow Protocol REST Model

!pygmentize ${RESOURCES}/model_tfserving_rest.yaml

!kubectl apply -f ${RESOURCES}/model_tfserving_rest.yaml -n seldon

!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=rest-tfserving \
                                 -o jsonpath='{.items[0].metadata.name}')

!for i in `seq 1 60`; do \
   sleep 1 && curl -d '{"instances": [1.0, 2.0, 5.0]}' \
   -X POST http://localhost:8003/seldon/seldon/rest-tfserving/v1/models/halfplustwo/:predict \
   -H "Content-Type: application/json"; \
done

for i in range(3):
    metric=!curl -s http://localhost:8003/seldon/seldon/rest-tfserving/prometheus | grep seldon_api_executor_server_requests_seconds_count
    if metric and len(metric)>0:
        print(metric[0])
        assert(not metric[0] == "")
        break
    else:
        print("Failed to get metrics for rest-tfserving")
        time.sleep(2)

!kubectl delete -f ${RESOURCES}/model_tfserving_rest.yaml -n seldon

Tensorflow Protocol GRPC Model

!pygmentize ${RESOURCES}/model_tfserving_grpc.yaml

!kubectl apply -f ${RESOURCES}/model_tfserving_grpc.yaml -n seldon

!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=grpc-tfserving \
                                 -o jsonpath='{.items[0].metadata.name}')

!cd ../../../executor/proto && for i in `seq 1 60`; do \
 sleep 1 && grpcurl \
   -d '{"model_spec":{"name":"halfplustwo"},"inputs":{"x":{"dtype": 1, "tensor_shape": {"dim":[{"size": 3}]}, "floatVal" : [1.0, 2.0, 3.0]}}}' \
   -rpc-header seldon:grpc-tfserving -rpc-header namespace:seldon \
   -plaintext -proto ./prediction_service.proto \
   0.0.0.0:8003 tensorflow.serving.PredictionService/Predict; \
done

for i in range(3):
    metric=!curl -s http://localhost:8003/seldon/seldon/grpc-tfserving/prometheus | grep seldon_api_executor_server_requests_seconds_count
    if metric and len(metric)>0:
        print(metric[0])
        assert(not metric[0] == "")
        break
    else:
        print("Failed to get metrics for grpc-tfserving")
        time.sleep(2)

!kubectl delete -f ${RESOURCES}/model_tfserving_grpc.yaml -n seldon

!kubectl config set-context $(kubectl config current-context) --namespace=seldon-system

!helm delete seldon-core-analytics -n seldon-system

!kubectl config set-context $(kubectl config current-context) --namespace=seldon