ja_sentence_segmenter.split.simple_splitter

  1"""Simple sentence splitter for japanese text."""
  2
  3import re
  4from collections.abc import Generator, Iterator
  5from re import Match
  6from typing import Union, overload
  7
  8BETWEEN_QUOTE_JA_REGEX = r"「[^「」]*」"
  9"""japanese quotation, whose punctuation is protected from splitting."""
 10
 11BETWEEN_PARENS_JA_REGEX = r"\([^()]*\)"
 12"""parentheses, whose punctuation is protected from splitting."""
 13
 14ESCAPE_CHAR = "∯"
 15"""sentinel wrapped around punctuation that must not split a sentence.
 16
 17it is stripped again once splitting is done. that final strip cannot tell a
 18sentinel apart from the same character in the input, so any U+222F already
 19present in the text is silently removed. remove it beforehand if your text
 20may contain it.
 21"""
 22
 23DEFAULT_PUNCTUATION_REGEX = r"。!?"
 24"""default punctuation characters for splitting."""
 25
 26
 27def __split_newline_iter(texts: Iterator[str]) -> Generator[str, None, None]:
 28    for text in texts:
 29        yield from text.splitlines()
 30
 31
 32@overload
 33def split_newline(arg: str) -> Generator[str, None, None]: ...
 34
 35
 36@overload
 37def split_newline(arg: list[str]) -> Generator[str, None, None]: ...
 38
 39
 40@overload
 41def split_newline(arg: Iterator[str]) -> Generator[str, None, None]: ...
 42
 43
 44def split_newline(arg: Union[str, list[str], Iterator[str]]) -> Generator[str, None, None]:
 45    """Split text with line boundaries.
 46
 47    Parameters
 48    ----------
 49    arg : Union[str, List[str], Iterator[str]]
 50        texts you want to split.
 51
 52    Yields
 53    ------
 54    Generator[str, None, None]
 55        texts splitted with line boundaries.
 56    """
 57    if isinstance(arg, str):
 58        yield from __split_newline_iter(iter([arg]))
 59    elif isinstance(arg, list):
 60        yield from __split_newline_iter(iter(arg))
 61    elif isinstance(arg, Iterator):
 62        yield from __split_newline_iter(arg)
 63
 64
 65def __split_punctuation_iter(texts: Iterator[str], punctuations: str, split_between_quote: bool, split_between_parens: bool) -> Generator[str, None, None]:
 66    def escape_between_punctuation(match: Match[str]) -> str:
 67        text = match.group()
 68        escape_regex = rf"(?<!{ESCAPE_CHAR})([{punctuations}])(?!{ESCAPE_CHAR})"
 69        result = re.sub(escape_regex, rf"{ESCAPE_CHAR}\1{ESCAPE_CHAR}", text)
 70        return result
 71
 72    def escape_between_quote(text: str) -> str:
 73        result = re.sub(BETWEEN_QUOTE_JA_REGEX, escape_between_punctuation, text)
 74        return result
 75
 76    def escape_between_parens(text: str) -> str:
 77        result = re.sub(BETWEEN_PARENS_JA_REGEX, escape_between_punctuation, text)
 78        return result
 79
 80    def sub_split_punctuation(text: str) -> list[str]:
 81        split_regex = rf"(?<!{ESCAPE_CHAR})([{punctuations}])(?!{ESCAPE_CHAR})"
 82        result = re.sub(split_regex, "\\1\n", text)
 83        unescape_regex = rf"({ESCAPE_CHAR})([{punctuations}])({ESCAPE_CHAR})"
 84        result = re.sub(unescape_regex, "\\2", result)
 85        return result.splitlines()
 86
 87    for text in texts:
 88        temp = text
 89        if not split_between_quote:
 90            temp = escape_between_quote(temp)
 91        if not split_between_parens:
 92            temp = escape_between_parens(temp)
 93        sentences = sub_split_punctuation(temp)
 94        yield from sentences
 95
 96
 97@overload
 98def split_punctuation(
 99    arg: str, punctuations: str = DEFAULT_PUNCTUATION_REGEX, split_between_quote: bool = False, split_between_parens: bool = False
100) -> Generator[str, None, None]: ...
101
102
103@overload
104def split_punctuation(
105    arg: list[str], punctuations: str = DEFAULT_PUNCTUATION_REGEX, split_between_quote: bool = False, split_between_parens: bool = False
106) -> Generator[str, None, None]: ...
107
108
109@overload
110def split_punctuation(
111    arg: Iterator[str], punctuations: str = DEFAULT_PUNCTUATION_REGEX, split_between_quote: bool = False, split_between_parens: bool = False
112) -> Generator[str, None, None]: ...
113
114
115def split_punctuation(
116    arg: Union[str, list[str], Iterator[str]],
117    punctuations: str = DEFAULT_PUNCTUATION_REGEX,
118    split_between_quote: bool = False,
119    split_between_parens: bool = False,
120) -> Generator[str, None, None]:
121    """Split text with puctuations.
122
123    Parameters
124    ----------
125    arg : Union[str, List[str], Iterator[str]]
126        texts you want to split
127    punctuations : str, optional
128        regular expression for puctuations, by default DEFAULT_PUNCTUATION_REGEX
129    split_between_quote : bool, optional
130        split if punctuation between quotes, by default False
131    split_between_parens : bool, optional
132        split if punctuation between parentheses, by default False
133
134    Yields
135    ------
136    Generator[str, None, None]
137        texts splitted with puctuations.
138
139    Notes
140    -----
141    protecting punctuation inside quotes and parentheses is implemented by
142    wrapping it in ESCAPE_CHAR (U+222F) and stripping that again afterwards.
143    the strip cannot tell a sentinel apart from the same character in the
144    input, so any U+222F already present in the text is silently removed.
145    """
146    if isinstance(arg, str):
147        yield from __split_punctuation_iter(iter([arg]), punctuations, split_between_quote, split_between_parens)
148    elif isinstance(arg, list):
149        yield from __split_punctuation_iter(iter(arg), punctuations, split_between_quote, split_between_parens)
150    elif isinstance(arg, Iterator):
151        yield from __split_punctuation_iter(arg, punctuations, split_between_quote, split_between_parens)