Skip to content

normalization_bias_verification

allennlp.sanity_checks.normalization_bias_verification

[SOURCE]


Code based almost entirely from the pytorch-lightning-snippets repository.

NormalizationBiasVerification

class NormalizationBiasVerification(VerificationBase):
 | def __init__(self, model: nn.Module)

Network layers with biases should not be combined with normalization layers, as the bias makes normalization ineffective and can lead to unstable training. This verification detects such combinations.

normalization_layers

class NormalizationBiasVerification(VerificationBase):
 | ...
 | normalization_layers = (
        nn.BatchNorm1d,
        nn.BatchNorm2d,
        nn.BatchNorm3d,
        nn.SyncBatchNorm,
 ...

detected_pairs

class NormalizationBiasVerification(VerificationBase):
 | ...
 | @property
 | def detected_pairs(self) -> List[Tuple]

check

class NormalizationBiasVerification(VerificationBase):
 | ...
 | def check(self, inputs) -> bool

collect_detections

class NormalizationBiasVerification(VerificationBase):
 | ...
 | def collect_detections(self)

register_hooks

class NormalizationBiasVerification(VerificationBase):
 | ...
 | def register_hooks(self)

destroy_hooks

class NormalizationBiasVerification(VerificationBase):
 | ...
 | def destroy_hooks(self)