hebrewprober.py

Go to the documentation of this file.

 
 
from typing import Optional, Union
 
from .charsetprober import CharSetProber
from .enums import ProbingState
from .sbcharsetprober import SingleByteCharSetProber
 
# This prober doesn't actually recognize a language or a charset.
# It is a helper prober for the use of the Hebrew model probers
 
 
 
 
class HebrewProber(CharSetProber):
    SPACE = 0x20
    # windows-1255 / ISO-8859-8 code points of interest
    FINAL_KAF = 0xEA
    NORMAL_KAF = 0xEB
    FINAL_MEM = 0xED
    NORMAL_MEM = 0xEE
    FINAL_NUN = 0xEF
    NORMAL_NUN = 0xF0
    FINAL_PE = 0xF3
    NORMAL_PE = 0xF4
    FINAL_TSADI = 0xF5
    NORMAL_TSADI = 0xF6
 
    # Minimum Visual vs Logical final letter score difference.
    # If the difference is below this, don't rely solely on the final letter score
    # distance.
    MIN_FINAL_CHAR_DISTANCE = 5
 
    # Minimum Visual vs Logical model score difference.
    # If the difference is below this, don't rely at all on the model score
    # distance.
    MIN_MODEL_DISTANCE = 0.01
 
    VISUAL_HEBREW_NAME = "ISO-8859-8"
    LOGICAL_HEBREW_NAME = "windows-1255"
 
    def __init__(self) -> None:
        super().__init__()
        self._final_char_logical_score = 0
        self._final_char_visual_score = 0
        self._prev = self.SPACE
        self._before_prev = self.SPACE
        self._logical_prober: Optional[SingleByteCharSetProber] = None
        self._visual_prober: Optional[SingleByteCharSetProber] = None
        self.resetreset()
 
    def reset(self) -> None:
        self._final_char_logical_score = 0
        self._final_char_visual_score = 0
        # The two last characters seen in the previous buffer,
        # mPrev and mBeforePrev are initialized to space in order to simulate
        # a word delimiter at the beginning of the data
        self._prev = self.SPACE
        self._before_prev = self.SPACE
        # These probers are owned by the group prober.
 
    def set_model_probers(
        self,
        logical_prober: SingleByteCharSetProber,
        visual_prober: SingleByteCharSetProber,
    ) -> None:
        self._logical_prober = logical_prober
        self._visual_prober = visual_prober
 
    def is_final(self, c: int) -> bool:
        return c in [
            self.FINAL_KAFFINAL_KAF,
            self.FINAL_MEMFINAL_MEM,
            self.FINAL_NUNFINAL_NUN,
            self.FINAL_PEFINAL_PE,
            self.FINAL_TSADIFINAL_TSADI,
        ]
 
    def is_non_final(self, c: int) -> bool:
        # The normal Tsadi is not a good Non-Final letter due to words like
        # 'lechotet' (to chat) containing an apostrophe after the tsadi. This
        # apostrophe is converted to a space in FilterWithoutEnglishLetters
        # causing the Non-Final tsadi to appear at an end of a word even
        # though this is not the case in the original text.
        # The letters Pe and Kaf rarely display a related behavior of not being
        # a good Non-Final letter. Words like 'Pop', 'Winamp' and 'Mubarak'
        # for example legally end with a Non-Final Pe or Kaf. However, the
        # benefit of these letters as Non-Final letters outweighs the damage
        # since these words are quite rare.
        return c in [self.NORMAL_KAFNORMAL_KAF, self.NORMAL_MEMNORMAL_MEM, self.NORMAL_NUNNORMAL_NUN, self.NORMAL_PE]
 
    def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
        # Final letter analysis for logical-visual decision.
        # Look for evidence that the received buffer is either logical Hebrew
        # or visual Hebrew.
        # The following cases are checked:
        # 1) A word longer than 1 letter, ending with a final letter. This is
        #    an indication that the text is laid out "naturally" since the
        #    final letter really appears at the end. +1 for logical score.
        # 2) A word longer than 1 letter, ending with a Non-Final letter. In
        #    normal Hebrew, words ending with Kaf, Mem, Nun, Pe or Tsadi,
        #    should not end with the Non-Final form of that letter. Exceptions
        #    to this rule are mentioned above in isNonFinal(). This is an
        #    indication that the text is laid out backwards. +1 for visual
        #    score
        # 3) A word longer than 1 letter, starting with a final letter. Final
        #    letters should not appear at the beginning of a word. This is an
        #    indication that the text is laid out backwards. +1 for visual
        #    score.
        #
        # The visual score and logical score are accumulated throughout the
        # text and are finally checked against each other in GetCharSetName().
        # No checking for final letters in the middle of words is done since
        # that case is not an indication for either Logical or Visual text.
        #
        # We automatically filter out all 7-bit characters (replace them with
        # spaces) so the word boundary detection works properly. [MAP]
 
        if self.statestatestate == ProbingState.NOT_ME:
            # Both model probers say it's not them. No reason to continue.
            return ProbingState.NOT_ME
 
        byte_str = self.filter_high_byte_only(byte_str)
 
        for cur in byte_str:
            if cur == self.SPACE:
                # We stand on a space - a word just ended
                if self._before_prev != self.SPACE:
                    # next-to-last char was not a space so self._prev is not a
                    # 1 letter word
                    if self.is_final(self._prev):
                        # case (1) [-2:not space][-1:final letter][cur:space]
                        self._final_char_logical_score += 1
                    elif self.is_non_final(self._prev):
                        # case (2) [-2:not space][-1:Non-Final letter][
                        #  cur:space]
                        self._final_char_visual_score += 1
            else:
                # Not standing on a space
                if (
                    (self._before_prev == self.SPACE)
                    and (self.is_final(self._prev))
                    and (cur != self.SPACE)
                ):
                    # case (3) [-2:space][-1:final letter][cur:not space]
                    self._final_char_visual_score += 1
            self._before_prev = self._prev
            self._prev = cur
 
        # Forever detecting, till the end or until both model probers return
        # ProbingState.NOT_ME (handled above)
        return ProbingState.DETECTING
 
    @property
    def charset_name(self) -> str:
        assert self._logical_prober is not None
        assert self._visual_prober is not None
 
        # Make the decision: is it Logical or Visual?
        # If the final letter score distance is dominant enough, rely on it.
        finalsub = self._final_char_logical_score - self._final_char_visual_score
        if finalsub >= self.MIN_FINAL_CHAR_DISTANCE:
            return self.LOGICAL_HEBREW_NAME
        if finalsub <= -self.MIN_FINAL_CHAR_DISTANCE:
            return self.VISUAL_HEBREW_NAME
 
        # It's not dominant enough, try to rely on the model scores instead.
        modelsub = (
            self._logical_prober.get_confidence() - self._visual_prober.get_confidence()
        )
        if modelsub > self.MIN_MODEL_DISTANCE:
            return self.LOGICAL_HEBREW_NAME
        if modelsub < -self.MIN_MODEL_DISTANCE:
            return self.VISUAL_HEBREW_NAME
 
        # Still no good, back to final letter distance, maybe it'll save the
        # day.
        if finalsub < 0.0:
            return self.VISUAL_HEBREW_NAME
 
        # (finalsub > 0 - Logical) or (don't know what to do) default to
        # Logical.
        return self.LOGICAL_HEBREW_NAME
 
    @property
    def language(self) -> str:
        return "Hebrew"
 
    @property
    def state(self) -> ProbingState:
        assert self._logical_prober is not None
        assert self._visual_prober is not None
 
        # Remain active as long as any of the model probers are active.
        if (self._logical_prober.state == ProbingState.NOT_ME) and (
            self._visual_prober.state == ProbingState.NOT_ME
        ):
            return ProbingState.NOT_ME
        return ProbingState.DETECTING