ja_sentence_segmenter.split.simple_splitter

  1"""Simple sentence splitter for japanese text."""
  2
  3import re
  4from collections.abc import Generator, Iterator
  5from re import Match
  6from typing import overload
  7
  8BETWEEN_QUOTE_JA_REGEX = r"「[^「」]*」"
  9"""japanese quotation, whose punctuation is protected from splitting."""
 10
 11BETWEEN_PARENS_JA_REGEX = r"\([^()]*\)"
 12"""parentheses, whose punctuation is protected from splitting."""
 13
 14ESCAPE_CHAR = "∯"
 15"""sentinel wrapped around punctuation that must not split a sentence.
 16
 17it is stripped again once splitting is done. the splitter cannot tell a
 18sentinel apart from the same character in the input, so a U+222F already
 19present in the text interferes: punctuation directly preceded or followed by
 20it is not split, and a ``∯<punctuation>∯`` sequence loses both U+222F
 21(``"あ∯。∯い。"`` yields ``"あ。い。"``). such sequences are matched without
 22overlap, so in ``"あ∯。∯。∯い。"`` only the first one is unwrapped and the
 23result is ``"あ。。∯い。"``. other occurrences are kept. remove it
 24beforehand if your text may contain it.
 25"""
 26
 27DEFAULT_PUNCTUATION_REGEX = r"。!?"
 28"""default punctuation characters for splitting."""
 29
 30
 31def __split_newline_iter(texts: Iterator[str]) -> Generator[str, None, None]:
 32    for text in texts:
 33        yield from text.splitlines()
 34
 35
 36@overload
 37def split_newline(arg: str) -> Generator[str, None, None]: ...
 38
 39
 40@overload
 41def split_newline(arg: list[str]) -> Generator[str, None, None]: ...
 42
 43
 44@overload
 45def split_newline(arg: Iterator[str]) -> Generator[str, None, None]: ...
 46
 47
 48def split_newline(arg: str | list[str] | Iterator[str]) -> Generator[str, None, None]:
 49    """Split text with line boundaries.
 50
 51    Parameters
 52    ----------
 53    arg : Union[str, List[str], Iterator[str]]
 54        texts you want to split.
 55
 56    Yields
 57    ------
 58    Generator[str, None, None]
 59        texts splitted with line boundaries.
 60    """
 61    if isinstance(arg, str):
 62        yield from __split_newline_iter(iter([arg]))
 63    elif isinstance(arg, list):
 64        yield from __split_newline_iter(iter(arg))
 65    elif isinstance(arg, Iterator):
 66        yield from __split_newline_iter(arg)
 67
 68
 69def __split_punctuation_iter(texts: Iterator[str], punctuations: str, split_between_quote: bool, split_between_parens: bool) -> Generator[str, None, None]:
 70    def escape_between_punctuation(match: Match[str]) -> str:
 71        text = match.group()
 72        escape_regex = rf"(?<!{ESCAPE_CHAR})([{punctuations}])(?!{ESCAPE_CHAR})"
 73        result = re.sub(escape_regex, rf"{ESCAPE_CHAR}\1{ESCAPE_CHAR}", text)
 74        return result
 75
 76    def escape_between_quote(text: str) -> str:
 77        result = re.sub(BETWEEN_QUOTE_JA_REGEX, escape_between_punctuation, text)
 78        return result
 79
 80    def escape_between_parens(text: str) -> str:
 81        result = re.sub(BETWEEN_PARENS_JA_REGEX, escape_between_punctuation, text)
 82        return result
 83
 84    def sub_split_punctuation(text: str) -> list[str]:
 85        split_regex = rf"(?<!{ESCAPE_CHAR})([{punctuations}])(?!{ESCAPE_CHAR})"
 86        result = re.sub(split_regex, "\\1\n", text)
 87        unescape_regex = rf"({ESCAPE_CHAR})([{punctuations}])({ESCAPE_CHAR})"
 88        result = re.sub(unescape_regex, "\\2", result)
 89        return result.splitlines()
 90
 91    for text in texts:
 92        temp = text
 93        if not split_between_quote:
 94            temp = escape_between_quote(temp)
 95        if not split_between_parens:
 96            temp = escape_between_parens(temp)
 97        sentences = sub_split_punctuation(temp)
 98        yield from sentences
 99
100
101@overload
102def split_punctuation(
103    arg: str, punctuations: str = DEFAULT_PUNCTUATION_REGEX, split_between_quote: bool = False, split_between_parens: bool = False
104) -> Generator[str, None, None]: ...
105
106
107@overload
108def split_punctuation(
109    arg: list[str], punctuations: str = DEFAULT_PUNCTUATION_REGEX, split_between_quote: bool = False, split_between_parens: bool = False
110) -> Generator[str, None, None]: ...
111
112
113@overload
114def split_punctuation(
115    arg: Iterator[str], punctuations: str = DEFAULT_PUNCTUATION_REGEX, split_between_quote: bool = False, split_between_parens: bool = False
116) -> Generator[str, None, None]: ...
117
118
119def split_punctuation(
120    arg: str | list[str] | Iterator[str],
121    punctuations: str = DEFAULT_PUNCTUATION_REGEX,
122    split_between_quote: bool = False,
123    split_between_parens: bool = False,
124) -> Generator[str, None, None]:
125    """Split text with puctuations.
126
127    Parameters
128    ----------
129    arg : Union[str, List[str], Iterator[str]]
130        texts you want to split
131    punctuations : str, optional
132        regular expression for puctuations, by default DEFAULT_PUNCTUATION_REGEX
133    split_between_quote : bool, optional
134        split if punctuation between quotes, by default False
135    split_between_parens : bool, optional
136        split if punctuation between parentheses, by default False
137
138    Yields
139    ------
140    Generator[str, None, None]
141        texts splitted with puctuations.
142
143    Notes
144    -----
145    protecting punctuation inside quotes and parentheses is implemented by
146    wrapping it in ESCAPE_CHAR (U+222F) and stripping that again afterwards.
147    a U+222F already present in the input is mistaken for a sentinel:
148    punctuation directly next to it is not split, and a non-overlapping
149    ``∯<punctuation>∯`` sequence loses both U+222F. see ESCAPE_CHAR.
150    """
151    if isinstance(arg, str):
152        yield from __split_punctuation_iter(iter([arg]), punctuations, split_between_quote, split_between_parens)
153    elif isinstance(arg, list):
154        yield from __split_punctuation_iter(iter(arg), punctuations, split_between_quote, split_between_parens)
155    elif isinstance(arg, Iterator):
156        yield from __split_punctuation_iter(arg, punctuations, split_between_quote, split_between_parens)