transformer_text_field

allennlp.data.fields.transformer_text_field

TransformerTextField¶

class TransformerTextField(Field[torch.Tensor]):
 | def __init__(
 |     self,
 |     input_ids: Union[torch.Tensor, List[int]],
 |     token_type_ids: Optional[Union[torch.Tensor, List[int]]] = None,
 |     attention_mask: Optional[Union[torch.Tensor, List[int]]] = None,
 |     special_tokens_mask: Optional[Union[torch.Tensor, List[int]]] = None,
 |     offsets_mapping: Optional[Union[torch.Tensor, List[int]]] = None,
 |     padding_token_id: int = 0
 | ) -> None

A TransformerTextField is a collection of several tensors that are are a representation of text, tokenized and ready to become input to a transformer.

The naming pattern of the tensors follows the pattern that's produced by the huggingface tokenizers, and expected by the huggingface transformers.

get_padding_lengths¶

class TransformerTextField(Field[torch.Tensor]):
 | ...
 | def get_padding_lengths(self) -> Dict[str, int]

as_tensor¶

class TransformerTextField(Field[torch.Tensor]):
 | ...
 | def as_tensor(
 |     self,
 |     padding_lengths: Dict[str, int]
 | ) -> Dict[str, torch.Tensor]

empty_field¶

class TransformerTextField(Field[torch.Tensor]):
 | ...
 | def empty_field(self)

batch_tensors¶

class TransformerTextField(Field[torch.Tensor]):
 | ...
 | def batch_tensors(
 |     self,
 |     tensor_list: List[Dict[str, torch.Tensor]]
 | ) -> Dict[str, torch.Tensor]

human_readable_repr¶

class TransformerTextField(Field[torch.Tensor]):
 | ...
 | def human_readable_repr(self) -> Dict[str, Any]