ja_sentence_segmenter.split.simple_splitter

  1"""Simple sentence splitter for japanese text."""
  2
  3import re
  4from collections.abc import Generator, Iterator
  5from re import Match
  6from typing import Union, overload
  7
  8BETWEEN_QUOTE_JA_REGEX = r"「[^「」]*」"
  9BETWEEN_PARENS_JA_REGEX = r"\([^()]*\)"
 10ESCAPE_CHAR = "∯"
 11DEFAULT_PUNCTUATION_REGEX = r"。!?"
 12"""default punctuation characters for splitting."""
 13
 14
 15def __split_newline_iter(texts: Iterator[str]) -> Generator[str, None, None]:
 16    for text in texts:
 17        yield from text.splitlines()
 18
 19
 20@overload
 21def split_newline(arg: str) -> Generator[str, None, None]: ...
 22
 23
 24@overload
 25def split_newline(arg: list[str]) -> Generator[str, None, None]: ...
 26
 27
 28@overload
 29def split_newline(arg: Iterator[str]) -> Generator[str, None, None]: ...
 30
 31
 32def split_newline(arg: Union[str, list[str], Iterator[str]]) -> Generator[str, None, None]:
 33    """Split text with line boundaries.
 34
 35    Parameters
 36    ----------
 37    arg : Union[str, List[str], Iterator[str]]
 38        texts you want to split.
 39
 40    Yields
 41    ------
 42    Generator[str, None, None]
 43        texts splitted with line boundaries.
 44    """
 45    if isinstance(arg, str):
 46        yield from __split_newline_iter(iter([arg]))
 47    elif isinstance(arg, list):
 48        yield from __split_newline_iter(iter(arg))
 49    elif isinstance(arg, Iterator):
 50        yield from __split_newline_iter(arg)
 51
 52
 53def __split_punctuation_iter(texts: Iterator[str], punctuations: str, split_between_quote: bool, split_between_parens: bool) -> Generator[str, None, None]:
 54    def escape_between_punctuation(match: Match[str]) -> str:
 55        text = match.group()
 56        escape_regex = rf"(?<!{ESCAPE_CHAR})([{punctuations}])(?!{ESCAPE_CHAR})"
 57        result = re.sub(escape_regex, rf"{ESCAPE_CHAR}\1{ESCAPE_CHAR}", text)
 58        return result
 59
 60    def escape_between_quote(text: str) -> str:
 61        result = re.sub(BETWEEN_QUOTE_JA_REGEX, escape_between_punctuation, text)
 62        return result
 63
 64    def escape_between_parens(text: str) -> str:
 65        result = re.sub(BETWEEN_PARENS_JA_REGEX, escape_between_punctuation, text)
 66        return result
 67
 68    def sub_split_punctuation(text: str) -> list[str]:
 69        split_regex = rf"(?<!{ESCAPE_CHAR})([{punctuations}])(?!{ESCAPE_CHAR})"
 70        result = re.sub(split_regex, "\\1\n", text)
 71        unescape_regex = rf"({ESCAPE_CHAR})([{punctuations}])({ESCAPE_CHAR})"
 72        result = re.sub(unescape_regex, "\\2", result)
 73        return result.splitlines()
 74
 75    for text in texts:
 76        temp = text
 77        if not split_between_quote:
 78            temp = escape_between_quote(temp)
 79        if not split_between_parens:
 80            temp = escape_between_parens(temp)
 81        sentences = sub_split_punctuation(temp)
 82        yield from sentences
 83
 84
 85@overload
 86def split_punctuation(
 87    arg: str, punctuations: str = DEFAULT_PUNCTUATION_REGEX, split_between_quote: bool = False, split_between_parens: bool = False
 88) -> Generator[str, None, None]: ...
 89
 90
 91@overload
 92def split_punctuation(
 93    arg: list[str], punctuations: str = DEFAULT_PUNCTUATION_REGEX, split_between_quote: bool = False, split_between_parens: bool = False
 94) -> Generator[str, None, None]: ...
 95
 96
 97@overload
 98def split_punctuation(
 99    arg: Iterator[str], punctuations: str = DEFAULT_PUNCTUATION_REGEX, split_between_quote: bool = False, split_between_parens: bool = False
100) -> Generator[str, None, None]: ...
101
102
103def split_punctuation(
104    arg: Union[str, list[str], Iterator[str]],
105    punctuations: str = DEFAULT_PUNCTUATION_REGEX,
106    split_between_quote: bool = False,
107    split_between_parens: bool = False,
108) -> Generator[str, None, None]:
109    """Split text with puctuations.
110
111    Parameters
112    ----------
113    arg : Union[str, List[str], Iterator[str]]
114        texts you want to split
115    punctuations : str, optional
116        regular expression for puctuations, by default DEFAULT_PUNCTUATION_REGEX
117    split_between_quote : bool, optional
118        split if punctuation between quotes, by default False
119    split_between_parens : bool, optional
120        split if punctuation between parentheses, by default False
121
122    Yields
123    ------
124    Generator[str, None, None]
125        texts splitted with puctuations.
126    """
127    if isinstance(arg, str):
128        yield from __split_punctuation_iter(iter([arg]), punctuations, split_between_quote, split_between_parens)
129    elif isinstance(arg, list):
130        yield from __split_punctuation_iter(iter(arg), punctuations, split_between_quote, split_between_parens)
131    elif isinstance(arg, Iterator):
132        yield from __split_punctuation_iter(arg, punctuations, split_between_quote, split_between_parens)