Source code for alibi_detect.utils.prediction
import numpy as np
from typing import Callable, Union
[docs]def tokenize_transformer(x: Union[list, np.ndarray], tokenizer: Callable, max_len: int, backend: str) -> dict:
"""
Batch tokenizer for transformer models.
Parameters
----------
x
Batch of instances.
tokenizer
Tokenizer for model.
max_len
Max token length.
backend
PyTorch ('pt') or TensorFlow ('tf') backend.
Returns
-------
Tokenized instances.
"""
return tokenizer(list(x), padding=True, truncation=True, max_length=max_len, return_tensors=backend)