This page was generated from examples/models/tracing/tracing.ipynb.
Distributed Tracing Template¶
Illustrate the configuration for allowing distributed tracing using Jaeger.
Setup Seldon Core¶
Install Seldon Core as described in docs
Then port-forward to that ingress on localhost:8003 in a separate terminal either with:
Ambassador:
kubectl port-forward $(kubectl get pods -n seldon -l app.kubernetes.io/name=ambassador -o jsonpath='{.items[0].metadata.name}') -n seldon 8003:8080
Istio:
kubectl port-forward $(kubectl get pods -l istio=ingressgateway -n istio-system -o jsonpath='{.items[0].metadata.name}') -n istio-system 8003:80
[1]:
!kubectl create namespace seldon
Error from server (AlreadyExists): namespaces "seldon" already exists
[2]:
!kubectl config set-context $(kubectl config current-context) --namespace=seldon
Context "kind-ansible" modified.
Install Jaeger¶
Follow the Jaeger docs to install on Kubernetes.
[3]:
!kubectl create namespace observability
!kubectl create -f https://github.com/jaegertracing/jaeger-operator/releases/download/v1.38.0/jaeger-operator.yaml -n observability
namespace/observability created
customresourcedefinition.apiextensions.k8s.io/jaegers.jaegertracing.io created
serviceaccount/jaeger-operator created
role.rbac.authorization.k8s.io/jaeger-operator created
rolebinding.rbac.authorization.k8s.io/jaeger-operator created
deployment.apps/jaeger-operator created
[4]:
!pygmentize simplest.yaml
apiVersion: jaegertracing.io/v1
kind: Jaeger
metadata:
name: simplest
spec:
agent:
strategy: DaemonSet
[5]:
!kubectl apply -f simplest.yaml
jaeger.jaegertracing.io/simplest created
Port forward to Jaeger UI
kubectl port-forward $(kubectl get pods -l app.kubernetes.io/name=simplest -n seldon -o jsonpath='{.items[0].metadata.name}') 16686:16686 -n seldon
Run Example REST Deployment¶
[12]:
!pygmentize deployment_rest.yaml
apiVersion: machinelearning.seldon.io/v1
kind: SeldonDeployment
metadata:
name: tracing-example
namespace: seldon
spec:
name: tracing-example
predictors:
- componentSpecs:
- spec:
containers:
- env:
- name: TRACING
value: '0'
- name: JAEGER_AGENT_HOST
valueFrom:
fieldRef:
fieldPath: status.hostIP
- name: JAEGER_AGENT_PORT
value: '5775'
- name: JAEGER_SAMPLER_TYPE
value: const
- name: JAEGER_SAMPLER_PARAM
value: '1'
image: seldonio/mock_classifier:1.9.0-dev
name: model1
terminationGracePeriodSeconds: 1
graph:
children: []
endpoint:
type: REST
name: model1
type: MODEL
name: tracing
replicas: 1
svcOrchSpec:
env:
- name: TRACING
value: '1'
- name: JAEGER_AGENT_HOST
valueFrom:
fieldRef:
fieldPath: status.hostIP
- name: JAEGER_AGENT_PORT
value: '5775'
- name: JAEGER_SAMPLER_TYPE
value: const
- name: JAEGER_SAMPLER_PARAM
value: '1'
[13]:
!kubectl create -f deployment_rest.yaml
seldondeployment.machinelearning.seldon.io/tracing-example created
[14]:
!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=tracing-example -o jsonpath='{.items[0].metadata.name}')
deployment "tracing-example-tracing-0-model1" successfully rolled out
[15]:
!curl -d '{"data": {"ndarray":[[1.0, 2.0, 5.0]]}}' \
-X POST http://localhost:8003/seldon/seldon/tracing-example/api/v1.0/predictions \
-H "Content-Type: application/json"
{"data":{"names":["proba"],"ndarray":[[0.43782349911420193]]},"meta":{"requestPath":{"model1":"seldonio/mock_classifier:1.9.0-dev"}}}
Check the Jaeger UI. You should be able to find traces like below:
[11]:
!kubectl delete -f deployment_rest.yaml
seldondeployment.machinelearning.seldon.io "tracing-example" deleted
Run Example GRPC Deployment¶
[ ]:
!pygmentize deployment_grpc.yaml
[ ]:
!kubectl create -f deployment_grpc.yaml
[ ]:
!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=tracing-example -o jsonpath='{.items[0].metadata.name}')
[ ]:
!cd ../../../executor/proto && grpcurl -d '{"data":{"ndarray":[[1.0,2.0]]}}' \
-rpc-header seldon:tracing-example -rpc-header namespace:seldon \
-plaintext \
-proto ./prediction.proto 0.0.0.0:8003 seldon.protos.Seldon/Predict
Check the Jaeger UI. You should be able to find traces like below:
[ ]:
!kubectl delete -f deployment_grpc.yaml
[ ]:
!kubectl delete -f https://raw.githubusercontent.com/jaegertracing/jaeger-operator/master/deploy/crds/jaegertracing.io_jaegers_crd.yaml
!kubectl delete -f https://raw.githubusercontent.com/jaegertracing/jaeger-operator/master/deploy/service_account.yaml
!kubectl delete -f https://raw.githubusercontent.com/jaegertracing/jaeger-operator/master/deploy/role.yaml
!kubectl delete -f https://raw.githubusercontent.com/jaegertracing/jaeger-operator/master/deploy/role_binding.yaml
!kubectl delete -f https://raw.githubusercontent.com/jaegertracing/jaeger-operator/master/deploy/operator.yaml
!kubectl delete namespace observability
[ ]: