evaluator

allennlp.evaluation.evaluator

[SOURCE]

---

Evaluator class for evaluating a model with a given dataset

Evaluator

class Evaluator(Registrable):
 | def __init__(
 |     self,
 |     batch_serializer: Optional[Serializer] = None,
 |     cuda_device: Union[int, torch.device] = -1,
 |     postprocessor_fn_name: str = "make_output_human_readable"
 | )

Evaluation Base class

Parameters

• batch_postprocessor : Postprocessor, optional (default = SimplePostprocessor)
  The postprocessor to use for turning both the batches and the outputs of the model into human readable data.

• cuda_device : Union[int, torch.device], optional (default = -1)
  The cuda device to use for this evaluation. The model is assumed to already be using this device; this parameter is only used for moving the input data to the correct device.

• postprocessor_fn_name : str, optional (default = "make_output_human_readable")
  Function name of the model's postprocessing function.

default_implementation

class Evaluator(Registrable):
 | ...
 | default_implementation = "simple"

__call__

class Evaluator(Registrable):
 | ...
 | def __call__(
 |     self,
 |     model: Model,
 |     data_loader: DataLoader,
 |     batch_weight_key: str = None,
 |     metrics_output_file: Union[str, PathLike] = None,
 |     predictions_output_file: Union[str, PathLike] = None
 | ) -> Dict[str, Any]

Evaluate a single data source.

Parameters

• model : Model
  The model to evaluate
• data_loader : DataLoader
  The DataLoader that will iterate over the evaluation data (data loaders already contain their data).
• batch_weight_key : str, optional (default = None)
  If given, this is a key in the output dictionary for each batch that specifies how to weight the loss for that batch. If this is not given, we use a weight of 1 for every batch.

• metrics_output_file : Union[str, PathLike], optional (default = None)
  Optional path to write the final metrics to.

• predictions_output_file : Union[str, PathLike], optional (default = None)
  Optional path to write the predictions to. If passed the postprocessor will be called and its output will be written as lines.

Returns

• metrics : Dict[str, Any]
  The metrics from evaluating the file.

SimpleEvaluator

@Evaluator.register("simple")
class SimpleEvaluator(Evaluator):
 | def __init__(
 |     self,
 |     batch_serializer: Optional[Serializer] = None,
 |     cuda_device: Union[int, torch.device] = -1,
 |     postprocessor_fn_name: str = "make_output_human_readable"
 | )

Simple evaluator implementation. Uses the vanilla evaluation code.

Parameters

• batch_postprocessor : Postprocessor, optional (default = SimplePostprocessor)
  The postprocessor to use for turning both the batches and the outputs of the model into human readable data.

• cuda_device : Union[int, torch.device], optional (default = -1)
  The cuda device to use for this evaluation. The model is assumed to already be using this device; this parameter is only used for moving the input data to the correct device.

• postprocessor_fn_name : str, optional (default = "make_output_human_readable")
  Function name of the model's postprocessing function.

__call__

class SimpleEvaluator(Evaluator):
 | ...
 | def __call__(
 |     self,
 |     model: Model,
 |     data_loader: DataLoader,
 |     batch_weight_key: str = None,
 |     metrics_output_file: Union[str, PathLike] = None,
 |     predictions_output_file: Union[str, PathLike] = None
 | )

Evaluate a single data source.

Parameters

• model : Model
  The model to evaluate
• data_loader : DataLoader
  The DataLoader that will iterate over the evaluation data (data loaders already contain their data).
• batch_weight_key : str, optional (default = None)
  If given, this is a key in the output dictionary for each batch that specifies how to weight the loss for that batch. If this is not given, we use a weight of 1 for every batch.
• metrics_output_file : Union[str, PathLike], optional (default = None)
  Optional path to write the final metrics to.
• predictions_output_file : Union[str, PathLike], optional (default = None)
  Optional path to write the predictions to.

Returns

• metrics : Dict[str, Any]
  The metrics from evaluating the file.