This page was generated from examples/models/tracing/tracing.ipynb.

Distributed Tracing Template

Illustrate the configuration for allowing distributed tracing using Jaeger.

Setup Seldon Core

Install Seldon Core as described in docs

Then port-forward to that ingress on localhost:8003 in a separate terminal either with:

  • Ambassador:

kubectl port-forward $(kubectl get pods -n seldon -l app.kubernetes.io/name=ambassador -o jsonpath='{.items[0].metadata.name}') -n seldon 8003:8080

  • Istio:

kubectl port-forward $(kubectl get pods -l istio=ingressgateway -n istio-system -o jsonpath='{.items[0].metadata.name}') -n istio-system 8003:80

[ ]:
!kubectl create namespace seldon
[ ]:
!kubectl config set-context $(kubectl config current-context) --namespace=seldon

Install Jaeger

Follow the Jaeger docs to install on Kubernetes.

[ ]:
!kubectl create namespace observability
!kubectl create -f https://raw.githubusercontent.com/jaegertracing/jaeger-operator/master/deploy/crds/jaegertracing.io_jaegers_crd.yaml
!kubectl create -f https://raw.githubusercontent.com/jaegertracing/jaeger-operator/master/deploy/service_account.yaml
!kubectl create -f https://raw.githubusercontent.com/jaegertracing/jaeger-operator/master/deploy/role.yaml
!kubectl create -f https://raw.githubusercontent.com/jaegertracing/jaeger-operator/master/deploy/role_binding.yaml
!kubectl create -f https://raw.githubusercontent.com/jaegertracing/jaeger-operator/master/deploy/operator.yaml
[ ]:
!pygmentize simplest.yaml
[ ]:
!kubectl apply -f simplest.yaml

Port forward to Jaeger UI

kubectl port-forward $(kubectl get pods -l app.kubernetes.io/name=simplest -n seldon -o jsonpath='{.items[0].metadata.name}') 16686:16686 -n seldon

Run Example REST Deployment

[ ]:
!pygmentize deployment_rest.yaml
[ ]:
!kubectl create -f deployment_rest.yaml
[ ]:
!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=tracing-example -o jsonpath='{.items[0].metadata.name}')
[ ]:
!curl -d '{"data": {"ndarray":[[1.0, 2.0, 5.0]]}}' \
   -X POST http://localhost:8003/seldon/seldon/tracing-example/api/v1.0/predictions \
   -H "Content-Type: application/json"

Check the Jaeger UI. You should be able to find traces like below:

rest

[ ]:
!kubectl delete -f deployment_rest.yaml

Run Example GRPC Deployment

[ ]:
!pygmentize deployment_grpc.yaml
[ ]:
!kubectl create -f deployment_grpc.yaml
[ ]:
!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=tracing-example -o jsonpath='{.items[0].metadata.name}')
[ ]:
!cd ../../../executor/proto && grpcurl -d '{"data":{"ndarray":[[1.0,2.0]]}}' \
         -rpc-header seldon:tracing-example -rpc-header namespace:seldon \
         -plaintext \
         -proto ./prediction.proto  0.0.0.0:8003 seldon.protos.Seldon/Predict

Check the Jaeger UI. You should be able to find traces like below:

grpc

[ ]:
!kubectl delete -f deployment_grpc.yaml
[ ]:
!kubectl delete -f https://raw.githubusercontent.com/jaegertracing/jaeger-operator/master/deploy/crds/jaegertracing.io_jaegers_crd.yaml
!kubectl delete -f https://raw.githubusercontent.com/jaegertracing/jaeger-operator/master/deploy/service_account.yaml
!kubectl delete -f https://raw.githubusercontent.com/jaegertracing/jaeger-operator/master/deploy/role.yaml
!kubectl delete -f https://raw.githubusercontent.com/jaegertracing/jaeger-operator/master/deploy/role_binding.yaml
!kubectl delete -f https://raw.githubusercontent.com/jaegertracing/jaeger-operator/master/deploy/operator.yaml
!kubectl delete namespace observability
[ ]: