ja_sentence_segmenter.concatenate.simple_concatenator

 1"""Simple sentence concatenator for japanese text."""
 2
 3import re
 4from collections.abc import Generator, Iterator
 5from typing import Optional, Union, overload
 6
 7
 8def __concatenate_matching_iter(
 9    texts: Iterator[str], former_matching_rule: Optional[str], latter_matching_rule: Optional[str], remove_former_matched: bool, remove_latter_matched: bool
10) -> Generator[str, None, None]:
11    try:
12        former = next(texts)
13
14        for latter in texts:
15            former_match_obj = re.match(former_matching_rule, former) if former_matching_rule else None
16            latter_match_obj = re.match(latter_matching_rule, latter) if latter_matching_rule else None
17
18            if former_matching_rule and latter_matching_rule and former_match_obj and latter_match_obj:
19                tmp_former = former_match_obj.group("result") if remove_former_matched else former
20                tmp_latter = latter_match_obj.group("result") if remove_latter_matched else latter
21                former = tmp_former + tmp_latter
22            elif former_matching_rule and not latter_matching_rule and former_match_obj:
23                tmp_former = former_match_obj.group("result") if remove_former_matched else former
24                former = tmp_former + latter
25            elif not former_matching_rule and latter_matching_rule and latter_match_obj:
26                tmp_latter = latter_match_obj.group("result") if remove_latter_matched else latter
27                former += tmp_latter
28            else:
29                yield former
30                former = latter
31
32        yield former
33    except StopIteration:
34        pass
35
36
37@overload
38def concatenate_matching(
39    arg: list[str],
40    former_matching_rule: Optional[str] = None,
41    latter_matching_rule: Optional[str] = None,
42    remove_former_matched: bool = True,
43    remove_latter_matched: bool = True,
44) -> Generator[str, None, None]: ...
45
46
47@overload
48def concatenate_matching(
49    arg: Iterator[str],
50    former_matching_rule: Optional[str] = None,
51    latter_matching_rule: Optional[str] = None,
52    remove_former_matched: bool = True,
53    remove_latter_matched: bool = True,
54) -> Generator[str, None, None]: ...
55
56
57def concatenate_matching(
58    arg: Union[str, list[str], Iterator[str]],
59    former_matching_rule: Optional[str] = None,
60    latter_matching_rule: Optional[str] = None,
61    remove_former_matched: bool = True,
62    remove_latter_matched: bool = True,
63) -> Generator[str, None, None]:
64    r"""Concatenate two lines with regular expression rule.
65
66    Parameters
67    ----------
68    arg : Union[str, List[str], Iterator[str]]
69        texts you want to concatenate.
70    former_matching_rule : Optional[str], optional
71        regular expression for former line, by default None
72    latter_matching_rule : Optional[str], optional
73        regular expression for latter line, by default None
74    remove_former_matched : bool, optional
75        whether to remove matched place of former line, by default True.
76        if this is True, former_matching_rule must contain named group 'result',
77        only that group remains.
78        e.g. r"^(\s*[>]+\s*)(?P<result>.+)$"
79    remove_latter_matched : bool, optional
80        whether to remove matched place of latter line, by default True.
81        if this is True, latter_matching_rule must contain named group 'result',
82        only that group remains.
83        e.g. r"^(\s*[>]+\s*)(?P<result>.+)$"
84
85    Yields
86    ------
87    Generator[str, None, None]
88        concatenated texts.
89    """
90    if isinstance(arg, list):
91        yield from __concatenate_matching_iter(iter(arg), former_matching_rule, latter_matching_rule, remove_former_matched, remove_latter_matched)
92    elif isinstance(arg, Iterator):
93        yield from __concatenate_matching_iter(arg, former_matching_rule, latter_matching_rule, remove_former_matched, remove_latter_matched)