allennlp.data.tokenizers
================================

.. automodule:: allennlp.data.tokenizers.token
   :members:
   :undoc-members:
   :show-inheritance:

.. automodule:: allennlp.data.tokenizers
   :members:
   :undoc-members:
   :show-inheritance:

* :ref:`Tokenizer<tokenizer>`
* :ref:`WordTokenizer<word-tokenizer>`
* :ref:`CharacterTokenizer<character-tokenizer>`
* :ref:`PretrainedTransformerTokenizer<pretrained-transformer-tokenizer>`
* :ref:`WordFilter<word-filter>`
* :ref:`WordSplitter<word-splitter>`
* :ref:`WordStemmer<word-stemmer>`

.. _tokenizer:
.. automodule:: allennlp.data.tokenizers.tokenizer
   :members:
   :undoc-members:
   :show-inheritance:

.. _word-tokenizer:
.. automodule:: allennlp.data.tokenizers.word_tokenizer
   :members:
   :undoc-members:
   :show-inheritance:

.. _character-tokenizer:
.. automodule:: allennlp.data.tokenizers.character_tokenizer
   :members:
   :undoc-members:
   :show-inheritance:

.. _pretrained-transformer-tokenizer:
.. automodule:: allennlp.data.tokenizers.pretrained_transformer_tokenizer
   :members:
   :undoc-members:
   :show-inheritance:

.. _word-filter:
.. automodule:: allennlp.data.tokenizers.word_filter
   :members:
   :undoc-members:
   :show-inheritance:

.. _word-splitter:
.. automodule:: allennlp.data.tokenizers.word_splitter
   :members:
   :undoc-members:
   :show-inheritance:

.. _word-stemmer:
.. automodule:: allennlp.data.tokenizers.word_stemmer
   :members:
   :undoc-members:
   :show-inheritance:

.. _sentence-splitter:
.. automodule:: allennlp.data.tokenizers.sentence_splitter
   :members:
   :undoc-members:
   :show-inheritance: