From 5d4149be63b97c5f2ddb7729ad88e559eecd88ce Mon Sep 17 00:00:00 2001 From: kuwoyuki Date: Tue, 18 Feb 2025 03:39:44 +0600 Subject: [PATCH] feat: add chunking --- .gitignore | 4 + bertalign/aligner.py | 6 +- bertalign/chunk.py | 209 +++++++++++++++++++++++++++++++++++ chunk_test.py | 257 +++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 4 +- tests/chunk_test.py | 51 +++++++++ tests/data/ri_4.en | 93 ++++++++++++++++ tests/data/ri_4.zh | 151 +++++++++++++++++++++++++ 8 files changed, 771 insertions(+), 4 deletions(-) create mode 100644 .gitignore create mode 100644 bertalign/chunk.py create mode 100644 chunk_test.py create mode 100644 tests/chunk_test.py create mode 100644 tests/data/ri_4.en create mode 100644 tests/data/ri_4.zh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fe89e8d --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +venv +.vscode +.pytest_cache/ +__pycache__/ diff --git a/bertalign/aligner.py b/bertalign/aligner.py index 6c9f916..a8ffed9 100644 --- a/bertalign/aligner.py +++ b/bertalign/aligner.py @@ -15,6 +15,8 @@ class Bertalign: margin=True, len_penalty=True, is_split=False, + src_lang=None, + tgt_lang=None ): self.max_align = max_align @@ -26,8 +28,8 @@ class Bertalign: src = clean_text(src) tgt = clean_text(tgt) - src_lang = detect_lang(src) - tgt_lang = detect_lang(tgt) + src_lang = src_lang if src_lang is not None else detect_lang(src) + tgt_lang = tgt_lang if tgt_lang is not None else detect_lang(tgt) if is_split: src_sents = src.splitlines() diff --git a/bertalign/chunk.py b/bertalign/chunk.py new file mode 100644 index 0000000..d5fe854 --- /dev/null +++ b/bertalign/chunk.py @@ -0,0 +1,209 @@ +from bertalign.aligner import Bertalign +from dataclasses import dataclass +from typing import TypeAlias + +AlignmentResult: TypeAlias = list[tuple[list[int], list[int]]] + + +@dataclass +class TextChunk: + source_paragraphs: list[str] + target_paragraphs: list[str] + source_text: str = "" + target_text: str = "" + + +class AlignmentError(Exception): + pass + + +def split_into_paragraphs(text: str) -> list[str]: + return [p.strip() for p in text.split("\n\n") if p.strip()] + + +def get_paragraph_indices(text_paragraphs: list[str]) -> tuple[dict[int, int], int]: + paragraph_map: dict[int, int] = {} + total_sentences = 0 + + for i, paragraph in enumerate(text_paragraphs): + num_sentences = len(paragraph.split("\n")) + paragraph_map.update({total_sentences + j: i for j in range(num_sentences)}) + total_sentences += num_sentences + + return paragraph_map, total_sentences + + +def validate_alignment( + alignments: list[tuple[list[int], list[int]]], source_count: int, target_count: int +) -> None: + if not alignments: + raise AlignmentError("Aligner produced no results") + + max_source = max(idx for alignment in alignments for idx in alignment[0]) + max_target = max(idx for alignment in alignments for idx in alignment[1]) + + if max_source >= source_count or max_target >= target_count: + raise AlignmentError("Aligner produced invalid sentence indices") + + +def validate_chunks( + chunks: list[TextChunk], source_pars: list[str], target_pars: list[str] +) -> None: + chunk_source_pars = [par for chunk in chunks for par in chunk.source_paragraphs] + chunk_target_pars = [par for chunk in chunks for par in chunk.target_paragraphs] + + def validate_pars(orig: list[str], chunk: list[str], name: str) -> None: + if len(orig) != len(chunk): + raise AlignmentError(f"{name} paragraph count mismatch") + if not all(o == c for o, c in zip(orig, chunk)): + raise AlignmentError(f"{name} paragraph content mismatch") + + validate_pars(source_pars, chunk_source_pars, "Source") + validate_pars(target_pars, chunk_target_pars, "Target") + + +def process_alignments( + alignments: list[tuple[list[int], list[int]]], + source_paragraphs: list[str], + target_paragraphs: list[str], + source_par_map: dict[int, int], + target_par_map: dict[int, int], + max_chars: int, +) -> list[TextChunk]: + """Process alignments into chunks.""" + chunks: list[TextChunk] = [] + current = TextChunk([], []) + processed_source: set[int] = set() + processed_target: set[int] = set() + + for source_indices, target_indices in alignments: + # add para to current chunk + if source_indices: + source_range = get_paragraph_range(source_indices, source_par_map) + add_paragraphs( + current.source_paragraphs, + source_paragraphs, + source_range, + processed_source, + ) + + if target_indices: + target_range = get_paragraph_range(target_indices, target_par_map) + add_paragraphs( + current.target_paragraphs, + target_paragraphs, + target_range, + processed_target, + ) + + # upd chunk texts + current.source_text = "\n\n".join(current.source_paragraphs) + current.target_text = "\n\n".join(current.target_paragraphs) + + if should_create_new_chunk( + current, + max_chars, + source_indices, + target_indices, + source_par_map, + target_par_map, + ): + chunks.append(current) + current = TextChunk([], []) + processed_source.clear() + processed_target.clear() + + if current.source_paragraphs: + chunks.append(current) + + validate_chunks(chunks, source_paragraphs, target_paragraphs) + return chunks + + +def get_paragraph_range(indices: list[int], par_map: dict[int, int]) -> range: + return range(par_map[indices[0]], par_map[indices[-1]] + 1) + + +def add_paragraphs( + chunk_paragraphs: list[str], + paragraphs: list[str], + par_range: range, + processed: set[int], +) -> None: + for idx in par_range: + if idx not in processed: + chunk_paragraphs.append(paragraphs[idx]) + processed.add(idx) + + +def should_create_new_chunk( + chunk: TextChunk, + max_chars: int, + source_indices: list[int], + target_indices: list[int], + source_par_map: dict[int, int], + target_par_map: dict[int, int], +) -> bool: + if len(chunk.source_text) <= max_chars and len(chunk.target_text) <= max_chars: + return False + + def at_paragraph_boundary(indices: list[int], par_map: dict[int, int]) -> bool: + if not indices: + return True + return all( + idx == max(i for i, par_idx in par_map.items() if par_idx == par_map[idx]) + for idx in indices + ) + + return at_paragraph_boundary( + source_indices, source_par_map + ) and at_paragraph_boundary(target_indices, target_par_map) + + +def create_aligned_chunks( + source_text: str, target_text: str, max_chars: int = 500 +) -> list[TextChunk]: + """Create aligned chunks of text respecting paragraph and sentence alignment. + + Args: + source_text: The source text to be chunked + target_text: The target text to be chunked + max_chars: Maximum characters per chunk (default: 500) + + Returns: + List of TextChunk objects containing aligned paragraphs + """ + source_paragraphs = split_into_paragraphs(source_text) + target_paragraphs = split_into_paragraphs(target_text) + + # TODO: just accept Bertalign instance as arg + aligner = Bertalign( + "\n".join(source_paragraphs), + "\n".join(target_paragraphs), + is_split=True, + src_lang="zh", + tgt_lang="en", + ) + aligner.align_sents() + + if not isinstance(aligner.result, list): + raise AlignmentError("Invalid alignment result type") + + alignments: AlignmentResult = aligner.result + if not alignments: + raise AlignmentError("Aligner produced no results") + + # get para mappings and validate + source_par_map, source_sent_count = get_paragraph_indices(source_paragraphs) + target_par_map, target_sent_count = get_paragraph_indices(target_paragraphs) + + validate_alignment(aligner.result, source_sent_count, target_sent_count) + + return process_alignments( + aligner.result, + source_paragraphs, + target_paragraphs, + source_par_map, + target_par_map, + max_chars, + ) diff --git a/chunk_test.py b/chunk_test.py new file mode 100644 index 0000000..bfbdd37 --- /dev/null +++ b/chunk_test.py @@ -0,0 +1,257 @@ +from bertalign.chunk import create_aligned_chunks + +target_text = """The sun rises in the sky, the sun ray brilliant. The mountain fog is not very thick; the sharp rays easily pass through. + +Over a hundred 15 year-old youths gathered in front of the clan pavilion. The clan pavilion was in the middle of the village, reaching 5 storeys and having sharp tilted roofs; it was heavily guarded. Before the pavilion was the square, and in the pavilion was the shrine of the Gu Yue ancestor memorial tablets. Every generation of clan head had lived in the pavilion. With every major ceremony or big incident, the clan elders would gather and discuss meetings here as well. This was the entire village’s authority central. + +“Good, all of you are punctual. Today is the Awakening Ceremony; it is your life’s great turning point. I won’t say much, just come with me.” The one responsible at the moment was the elder of the academy. His beard and hair were white and he was in high spirits as he led the young teenagers into the pavilion. However they did not go up, but were led downstairs after going through the entrance of a great hall. Following down a constructed stone ladder, they went into an underground cave. + +The group of youths made surprised and amazed noises. The underground cave was beautiful, stalactites sparkling with the colors of the rainbow. This light shone on the youth’s faces, the neon hues gorgeous. + +Fang Yuan was mixed into the crowd, quietly observing everything that was happening. In his heart, he thought: Hundreds of years ago, the Gu Yue clan came to Qing Mao Mountain and settled down after migrating from the central lands to the South Border. It was when they found a spirit spring in this underground cave. This spirit spring produces a large number of primeval stones – It could be said that this was the foundation of the Gu Yue village. + +They walked several hundred steps. It got darker and the sounds of water were faintly heard. After turning around a corner, a 3 zhang – wide underground river greeted them. By now the colorful lights of the stalactites had disappeared completely, yet in the darkness the river emitted faint blue light. It was like a star river of the night sky. + +The river flowed from the dark depths of the cave. Inside the crystal clear waters, one could see fish, aquatic plants and even the sand beneath the river. Opposite the river was a sea of flowers. + +This was the Gu Yue Clan’s closely cultivated moon orchids. The beautiful blue and pink colored petals were like shaped like a crescent moon; the flower stems were like jade, the center of the flower shining like the sort of warm brilliance that radiates from pearls under the light. At first glance, in the dark background the flower sea looks like a huge piece of land covered in bluish green carpet dotted with countless pearls. + +The moon orchid is food for a lot of Gu. This flower sea could be said as the clan’s biggest cultivation medium, Fang Yuan thought knowingly to himself. + +“Wow, so pretty!” + +“It really is beautiful!” + +The new sight opened the young teenagers’ eyes. Each one of them had a light radiating from their gaze with excited and anxious feelings. + +“Alright, listen as I call your names. Those who are called must walk through this river to the opposite bank. Walk as far as you can, of course the further you go the better it is. Are you all clear?” The elder said. + +“All clear,” the youths replied. Actually before they came here, they had all heard their family members or seniors talk about it. It is known that the further you can walk, the better your talent is. Your future will also become brighter. + +“Gu Yue Chen Bo.” The elder held the name list and called out the first person. + +The river was wide but not deep – it covered up to a youth’s kneecaps. Chen Bo’s face was full of seriousness as he stepped into the flower sea ashore. As he did so, he could feel an invisible pressure as if there was a wall in front of him that he could not see, blocking him from walking forward. During this moment, the flowers at his feet suddenly gave off a weak white light. The light gathered around Chen Bo and entered his body. For a moment Chen Bo felt the pressure drop; the invisible wall blocking him suddenly felt softer. With this, Chen Bo gritted his teeth and mustered his strength, walking forward. He tried to force his way in stiffly, yet after three steps the wall in front of him hardened again back to the state before. Thus he could not walk any further. + +As he watched this the elder sighed. While recording what happened, he said, “Gu Yue Chen Bo, 3 steps, no talent to become a Gu Master. Next, Gu Yue Zao Xie!” + +Chen Bo was deathly pale as he walked pass the river back to the youths, clenching his teeth. Without the endowed talent he could live as a normal human, holding the lowest position in the clan. + +His stature was shaky; it was a huge blow to him, as if reality had killed all his hopes. Many people threw him pitiful gazes, while even more had fixated stares at the second person crossing the river. + +It was a pity that this youth could only walk four steps forward – he did not have talent either. + +Not everyone has the natural talent to be a Gu Master. Generally speaking, it is not bad if five out of ten people have talent. In the Gu Clan, this ratio is higher, reaching six people. This is because the Gu Yue clan’s ancestor – The first generation clan leader was a famous, legendary and powerful man. Due to cultivation reasons his bloodline carried powerful genes, thus the average quality of talent in the Gu Yue clan was generally higher as they carried his blood in their veins. + +With two consecutive failures, the other elders observing the scene in the dark started making ugly expressions. Even the clan head was frowning slightly. The next moment, the academy elder called out the third name: Gu Yue Mo Bei. + +“Here!” A horse-faced youth dressed in linen robes lightly called as he came forth. He was tall in build, looking much sturdier than his peers. There was a brave aura about him. He crossed the river in a few steps and reached the opposite bank. 10 steps, 20 steps, 30 steps; one after another small lights entered his body. He walked until he reached 36 steps before he could finally go no further. + +The youths at the riverbank watched with wide opened eyes, shocked. The academy elder happily exclaimed, “Good, Gu Yue Mo Bei, B grade talent! Come here, let me see your Primeval sea.” + +Gu Yue Mo Bei walked back to the academy elder’s side. The latter stretched out his hand and put it on the juvenile’s shoulder, closing his eyes as he checked with focus. Then he retracted his hand and nodded, recording down on the paper: Gu Yue Mo Bei, primeval sea measuring six by six, can be vigorously trained. + +This special talent can be measured by four grades – A grade to D grade. A D grade talent youth who is raised for 3 years would be able to become a rank one senior Gu Master, become the foundation of the family. A C grade talent youth after two years of cultivation will usually be able to become a rank two senior Gu Master, becoming the clan’s backbone. A B grade talent must be cared for. Often becoming a future clan elder, with 6-7 years of training they will become rank three Gu Masters. + +And when it comes to A grade, even if it was just one, would bring great luck to the entire clan. Great care must be given; with this talent in about 10 years they can become a rank four Gu Master. At that moment they would be able to compete for the position of the head of the clan! + +In other words, as long as this Gu Yue Mo Bei grows up, eventually he will become one of the elders of the Gu Yue clan. That is why the academy elder laughed happily; the elders watching in the darkness also sighed in relief, then they all turned to look at one of the elders amongst them with jealousy. + +This elder was also horse-faced, known as Gu Yue Mo Bei’s grandfather, Gu Yue Mo Chen. His face was already smiling. He provokingly looked at his old nemesis and said, “What do you think? My grandson isn’t bad huh, Gu Yue Chi Lian.” + +Gu Yue Chi Lian had a head full of red hair. He made an annoyed ‘hmph’, not replying to other. It was apparent that his face expression was really dark. + +One hour later, half of the youths had already walked through the flower sea. There were quite a number of C and D grade talents among them, while half of those youths had no talent at all. + +“Sigh, the bloodline is getting thinner. These few years the clan hasn’t had any rank four masters to strengthen the bloodline. The fourth generation clan head was the only rank five master, but in the end he perished together with the Flower Wine monk and did not leave behind any descendants. The Gu Yue clan’s later generation talents are getting weaker and weaker,” the clan head said with a deep sigh. + +At this moment, the academy elder shouted, “Gu Yue Chi Chen!” + +On hearing this name all the elders looked at Gu Yue Chi Lian; this was Gu Yue Chi Lian’s grandson. + +Gu Yue Chi Lian had a small and short build with a face full of pockmarks. He was clenching his fist, his entire face sweating. It was evident that he was incredibly nervous. + +As he walked onto the opposite bank, the little lights entered his body; after walking straight for 36 steps he stopped. + +“Another B grade!” The academy elder yelled. + +The youths started a commotion, sending Gu Yue Chi Chen envious stares. + +“Hahaha, 36 steps, 36 steps!” Gu Yue Chi Lian shouted, proudly staring at Gu Yue Mo Bei. This time it was Gu Yue Mo Chen’s turn to have a sour face. + +“Gu Yue Chi Chen, huh…” In the midst of the crowd, Fang Yuan stroked his chin thoughtfully. In his memories, the clan heavily punished Gu Yue Chi Chen because he cheated during the Awakening Ceremony. In reality Chi Chen only had a C grade talent, but because his grandfather Gu Yue Chi Lian helped him fake the results, that’s why he appeared to have B grade talent. + +To be honest if he wanted to cheat, Fang Yuan had a countless number of ways to do so, some ways even more perfect than Gu Yue Chi Chen’s method. If a B grade or A grade talent appeared, they would receive the clan’s huge care. + +But firstly, Fang Yuan had only just been reborn. It was hard to prepare the cheating method by this condition. Secondly, even if he managed to cheat, he would not be able to fake his cultivation speed. He would be exposed by then. However Gu Yue Chi Chen was different; his grandfather was Gu Yue Chi Lian – One of the two elders with the most authority within the clan. With this Chi Lian would be able to cover up for his grandson. + +“Gu Yue Chi Lian was always hostile towards Gu Yue Mo Chen, these two elders are the clan’s two biggest influential authorities. To suppress his opponent he would need his own grandson to have an outstanding talent. It is also because he was helping from behind, Gu Yue Chi Chen was able to conceal the truth for a time. In my memories, if it were not for that incident, the truth would never have been exposed.” + +Fang Yuan’s eyes shone with light, his mind thinking up ways to use this knowledge to his advantage. + +If he exposed the matter on the spot, he would receive a bit of reward from the clan, but then he would offend the highly powerful Gu Yue Chi Lian. This was not advisable. + +Within such a short time he also could not blackmail them. Due to having low status, it would just backfire on him. + +As he pondered, he suddenly heard the academy elder call out his own name: “Gu Yue Fang Yuan!”""" +source_text = """朝阳升起来，霞光烂漫。 + +山雾不是很浓，被利剑般的阳光轻易洞穿。 + +一百多位十五岁的少年，此刻汇集在家主阁前。 + +家主阁就处在山寨的正中央，高达五层，飞檐翘角，重兵把守。阁前就是广场，阁内供奉着古月先人的牌位。每代族长也都生活起居在这里面，每逢重大典礼，或者有突发大事，也会在这里召集家老们商讨议论。这是整个山寨的权利中枢。 + +“很好，都准时来了。今天是开窍大典，是你们人生的重大转折点。闲话不多说了，随我来吧。”负责此行的，是学堂家老。他须发皆白，精神矍铄地领着少年们进入家主阁。 + +不过却没有上楼，而是通过一层大堂的入口，往下走。 + +顺着打造好的石梯，就进入地下溶洞。 + +少年们纷纷发出惊叹之声。地下溶洞美轮美奂，钟乳石散发着赤橙黄绿青蓝紫七色光华，这光彩映照在少年们的脸上，霓虹般绚烂。 + +方源混杂在人群中央，静静地审视这一切，心中暗暗思量：“数百年前，古月一族从中土迁徙到南疆，在这青茅山驻扎下来。就是看中了这里地下溶洞的一口灵泉。这灵泉产出大量元石，可以说是古月山寨的根基。” + +行了数百步，却是越来越暗，并且依稀听到了水声。 + +转过转角，一条宽三丈有余的地下河，就展现在众人眼前。 + +此地钟乳石的彩光，已经彻底消失了。 + +但是黑暗中，河水却散发着淡淡的幽蓝之光，好像是夜空中的星河。 + +河水从溶洞的黑暗深处流淌过来，清澈无比，甚至可以看到里面的游鱼，水草，以及河底的沙石。 + +在河的对岸，是一片花海。 + +这是古月一族有意栽培的月兰花，花瓣如月牙，呈现出清丽淡雅的蓝粉色。花茎如玉，花心闪耀着，好像是珍珠在光下的折射出来的温润光华。 + +乍一眼看上去，在黑暗的背景中，河畔花海就好像是一大片的蓝绿地毯，点缀着数不清的珍珠。 + +“月兰花，是很多蛊虫的食材。这片花海，可以说是家族最大的培养基地了。”方源对此心知肚明。 + +“好美。” + +“真是漂亮呀。” + +少年们算是开了眼界，一个个双眼放光，既兴奋又紧张。 + +“好了，下面听我报名，叫到的人穿过这河，到对岸去。能走多远，就走多远，当然越远越好。都听清楚了吗？”家老此刻说着。 + +“清楚了。”少年们纷纷应是。其实来之前，都听家人或者前辈们讲过，知道走的越远，代表资质越好，日后的成就也就越大。 + +“古月陈博。”家老拿着名单点出第一人。 + +河水虽宽却并不深，只及少年膝盖。陈博一脸的严肃，踏上河畔花海。 + +顿时他就感觉到一股隐形的压力，好像面前有一面看不见的墙，在阻挡他前进。 + +正举步维艰之时，脚畔的花海中忽然浮起一蓬光点，光点很稀薄，呈现素白之色。 + +光辉向陈博汇集过去，并投入到他的体内。 + +陈博瞬间感觉到压力剧减。那堵无形的墙壁，忽然变得柔软起来。 + +他咬牙用力向前走，硬生生的挤进去。走了三步之后，前方的压力又大增，一如之前如墙壁一样，不能再进分毫。 + +见到此景，家老一叹，当场一边记录，一边道：“古月陈博，三步，没有蛊师资质。下一个，古月藻榭。” + +陈博脸色顿时苍白，咬着牙，穿过河水，回到原处。没有资质，今后就只能作为一个凡人活着，在家族中也只能是最底层的地位。 + +他身躯摇摇欲坠，打击太大了，等于是扼杀了一生的希望。 + +很多人都向他投来怜悯的目光，更多的人则关注着第二位登上彼岸的少年。 + +可惜这个少年，也只能前进四步，同样没有资质。 + +并非所有人都有修行的资质，一般而言，十个人中有五人能修行，就已经不错了。在古月家族里，这个比例还要高一些，达到六人的程度。 + +这是因为古月先祖，也就是一代族长，是一位大名鼎鼎的传奇强者，因为修行的缘故导致他的血脉中隐藏着承载力量的基因。古月族人因为有着他的血脉，因此资质普遍较高。 + +连续两个没有资质的情况，让暗中关注的其他家老们都脸色难看起来，就是老成持重的古月族长，也微微蹙眉。 + +就在这时，学堂家老喊出第三个名字：“古月漠北。” + +“在！”一个身穿麻布衣衫的马脸少年，轻喝一声，越众而出。 + +他身材高大，比同龄人要粗壮得多，透着一丝彪悍气息。 + +三两步过了河，踏上对岸。 + +十步，二十步，三十步，陆续有微光投入到他的体内。 + +一直走到三十六步，终于走不动了。 + +少年们隔岸看得目瞪口呆，学堂家老欢喜得大叫起来：“好，古月漠北，乙等资质，来这里，让我看看你的元海。” + +古月漠北便又回到学堂家老的身边，后者伸出手，搭在少年的肩膀，闭目凝神探查了一番，便收回手，点点头，在纸上记录起来：“古月漠北，元海六成六，可大力栽培。” + +这资质从上到下，分甲乙丙丁四等。 + +一位丁等资质的少年，培养个三年，就能晋升成一转的资深蛊师，成为家族的基石。 + +一位丙等资质的少年，培养两年，大多都能成为二转的蛊师，成为家族的中坚存在。 + +一位乙等资质，就要呵护了。往往要作为未来的家老培养，六七年的功夫，能成为三转蛊师。 + +至于甲等资质，哪怕出现一位，都是整个家族的幸运。要细心关照，倾斜资源，十年左右就能成为四转蛊师，到那时便能竞争族长之位！ + +也就是说，这古月漠北只要成长起来，就是今后古月一族的家老。难怪学堂长老喜得哈哈一笑，而暗中关注的众家老们都统统舒了一口气，而后又纷纷向其中一位家老投去羡慕的目光。 + +这家老也是一副马脸，正是古月漠北的爷爷古月漠尘。他脸上早已经荡漾起笑意，又挑衅地看了一眼自己的老对头：“怎么样，我的孙儿不差吧，古月赤练。” + +家老古月赤练一头红发，此时冷哼一声，并未答话，脸色阴沉得很是难看。 + +半个时辰之后，已有一半少年踏足过花海，涌现了不少丙等、丁等的资质，当然毫无资质的占了几乎一半。 + +“唉，血脉越来越稀薄，加上这些年来，家族也没有出现几位四转强者，来增强血脉。四代族长是唯一的五转强者，结果却和花酒行者同归于尽，没有留下血脉后裔。古月一族后辈的资质是越来越弱了。”族长深深的叹息着。 + +就在这时，学堂家老喊道：“古月赤城。” + +听到这个名字，家老们纷纷看向古月赤练，这是古月赤练的孙子。 + +古月赤城身材矮小，满脸麻子，捏着拳头，满脸出汗，显得特别的紧张。 + +他踏上对岸，光点纷纷投入他的体内，一连走到三十六步，这才停步。 + +“又一个乙等！”学堂家老叫喊着。 + +少年们骚动起来，纷纷向古月赤城投来羡慕的目光。 + +“哈哈哈，三十六步，三十六步！”古月赤练大叫着，示威地瞪着古月漠尘。 + +这次轮到古月漠尘脸色铁青了。 + +“古月赤城么……”人群中，方源若有所思的摩挲着下巴。 + +记忆中，他因为在开窍大典中作弊，而受到了族中的严厉惩罚。 + +事实上，他的资质只有丙等，但是他的爷爷古月赤练为他作假，因此有了乙等资质的假象。 + +其实要作弊，方源心中有数十种方案，有些方案比古月赤城的更加完美。若是表现出乙等，或者甲等的资质，必然受到家族的大力栽培。 + +但是一来，方源重生的时间太短，以他的境况难以准备作弊手段。 + +二来，就算是作弊成功，日后修行时的速度是掩盖不住的，照样会露相。 + +而这个古月赤城却不一样，他的爷爷就是古月赤练，是家族中权势最重的两个家老之一，能够为他遮掩。 + +“古月赤练一直和古月漠尘敌对，这两个家老是家族中最大的两派势力。为了打压对手，他需要自己的孙子资质出众。也正是因为他在背后掩护，古月赤城才能够隐瞒一时。记忆中要不是那场意外，也不会暴露。” + +方源眼中闪烁着精芒，思量着该怎么利用这点来谋夺最大的利益呢？ + +当场揭露，虽然会得到家族的一点奖赏，但是却会得罪位高权重的古月赤练，绝不可取。 + +短时间之内，也不能敲诈勒索。因为实力太低，反而会自取其祸。 + +正思量着，忽然听到学堂家老叫出自己的名字：“古月方源！”""" + +chunks = create_aligned_chunks(source_text, target_text, max_chars=500) + +# Print chunks +for i, chunk in enumerate(chunks, 1): + print(f"\nChunk {i}:") + print("\nSource:") + print(chunk.source_text) + print("\nTarget:") + print(chunk.target_text) + print("-" * 80) diff --git a/requirements.txt b/requirements.txt index 2a3f060..a021400 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ numba==0.60.0 -#faiss-cpu==1.7.2 -faiss-gpu==1.7.2 +faiss-cpu==1.7.2 +#faiss-gpu==1.7.2 googletrans==4.0.0rc1 sentence-splitter==1.4 sentence-transformers==3.2.1 diff --git a/tests/chunk_test.py b/tests/chunk_test.py new file mode 100644 index 0000000..45eb9ed --- /dev/null +++ b/tests/chunk_test.py @@ -0,0 +1,51 @@ +from typing_extensions import override +import unittest +import os +from bertalign.chunk import create_aligned_chunks, TextChunk, AlignmentError + + +class TestChunk(unittest.TestCase): + data_dir: str + source_file: str + target_file: str + source_text: str = "" + target_text: str = "" + + def __init__(self, methodName: str = "runTest") -> None: + super().__init__(methodName) + + self.data_dir = os.path.join(os.path.dirname(__file__), "data") + self.source_file = os.path.join(self.data_dir, "ri_4.zh") # Source text file + self.target_file = os.path.join(self.data_dir, "ri_4.en") # Target text file + + @override + def setUp(self): + # Load source and target text from files + with open(self.source_file, "r", encoding="utf-8") as f: + self.source_text = f.read() + + with open(self.target_file, "r", encoding="utf-8") as f: + self.target_text = f.read() + + def test_create_aligned_chunks(self): + chunks = create_aligned_chunks( + self.source_text, self.target_text, max_chars=500 + ) + + self.assertIsInstance(chunks, list) + self.assertTrue(all(isinstance(chunk, TextChunk) for chunk in chunks)) + + for chunk in chunks: + self.assertIsInstance(chunk.source_text, str) + self.assertIsInstance(chunk.target_text, str) + + self.assertGreater(len(chunks), 0) + + @unittest.skip("no") + def test_create_aligned_chunks_empty_input(self): + c = create_aligned_chunks("", self.target_text) + self.assertIsNone(c) + + +if __name__ == "__main__": + _ = unittest.main() diff --git a/tests/data/ri_4.en b/tests/data/ri_4.en new file mode 100644 index 0000000..ee13887 --- /dev/null +++ b/tests/data/ri_4.en @@ -0,0 +1,93 @@ +The sun rises in the sky, the sun ray brilliant. The mountain fog is not very thick; the sharp rays easily pass through. + +Over a hundred 15 year-old youths gathered in front of the clan pavilion. The clan pavilion was in the middle of the village, reaching 5 storeys and having sharp tilted roofs; it was heavily guarded. Before the pavilion was the square, and in the pavilion was the shrine of the Gu Yue ancestor memorial tablets. Every generation of clan head had lived in the pavilion. With every major ceremony or big incident, the clan elders would gather and discuss meetings here as well. This was the entire village’s authority central. + +“Good, all of you are punctual. Today is the Awakening Ceremony; it is your life’s great turning point. I won’t say much, just come with me.” The one responsible at the moment was the elder of the academy. His beard and hair were white and he was in high spirits as he led the young teenagers into the pavilion. However they did not go up, but were led downstairs after going through the entrance of a great hall. Following down a constructed stone ladder, they went into an underground cave. + +The group of youths made surprised and amazed noises. The underground cave was beautiful, stalactites sparkling with the colors of the rainbow. This light shone on the youth’s faces, the neon hues gorgeous. + +Fang Yuan was mixed into the crowd, quietly observing everything that was happening. In his heart, he thought: Hundreds of years ago, the Gu Yue clan came to Qing Mao Mountain and settled down after migrating from the central lands to the South Border. It was when they found a spirit spring in this underground cave. This spirit spring produces a large number of primeval stones – It could be said that this was the foundation of the Gu Yue village. + +They walked several hundred steps. It got darker and the sounds of water were faintly heard. After turning around a corner, a 3 zhang – wide underground river greeted them. By now the colorful lights of the stalactites had disappeared completely, yet in the darkness the river emitted faint blue light. It was like a star river of the night sky. + +The river flowed from the dark depths of the cave. Inside the crystal clear waters, one could see fish, aquatic plants and even the sand beneath the river. Opposite the river was a sea of flowers. + +This was the Gu Yue Clan’s closely cultivated moon orchids. The beautiful blue and pink colored petals were like shaped like a crescent moon; the flower stems were like jade, the center of the flower shining like the sort of warm brilliance that radiates from pearls under the light. At first glance, in the dark background the flower sea looks like a huge piece of land covered in bluish green carpet dotted with countless pearls. + +The moon orchid is food for a lot of Gu. This flower sea could be said as the clan’s biggest cultivation medium, Fang Yuan thought knowingly to himself. + +“Wow, so pretty!” + +“It really is beautiful!” + +The new sight opened the young teenagers’ eyes. Each one of them had a light radiating from their gaze with excited and anxious feelings. + +“Alright, listen as I call your names. Those who are called must walk through this river to the opposite bank. Walk as far as you can, of course the further you go the better it is. Are you all clear?” The elder said. + +“All clear,” the youths replied. Actually before they came here, they had all heard their family members or seniors talk about it. It is known that the further you can walk, the better your talent is. Your future will also become brighter. + +“Gu Yue Chen Bo.” The elder held the name list and called out the first person. + +The river was wide but not deep – it covered up to a youth’s kneecaps. Chen Bo’s face was full of seriousness as he stepped into the flower sea ashore. As he did so, he could feel an invisible pressure as if there was a wall in front of him that he could not see, blocking him from walking forward. During this moment, the flowers at his feet suddenly gave off a weak white light. The light gathered around Chen Bo and entered his body. For a moment Chen Bo felt the pressure drop; the invisible wall blocking him suddenly felt softer. With this, Chen Bo gritted his teeth and mustered his strength, walking forward. He tried to force his way in stiffly, yet after three steps the wall in front of him hardened again back to the state before. Thus he could not walk any further. + +As he watched this the elder sighed. While recording what happened, he said, “Gu Yue Chen Bo, 3 steps, no talent to become a Gu Master. Next, Gu Yue Zao Xie!” + +Chen Bo was deathly pale as he walked pass the river back to the youths, clenching his teeth. Without the endowed talent he could live as a normal human, holding the lowest position in the clan. + +His stature was shaky; it was a huge blow to him, as if reality had killed all his hopes. Many people threw him pitiful gazes, while even more had fixated stares at the second person crossing the river. + +It was a pity that this youth could only walk four steps forward – he did not have talent either. + +Not everyone has the natural talent to be a Gu Master. Generally speaking, it is not bad if five out of ten people have talent. In the Gu Clan, this ratio is higher, reaching six people. This is because the Gu Yue clan’s ancestor – The first generation clan leader was a famous, legendary and powerful man. Due to cultivation reasons his bloodline carried powerful genes, thus the average quality of talent in the Gu Yue clan was generally higher as they carried his blood in their veins. + +With two consecutive failures, the other elders observing the scene in the dark started making ugly expressions. Even the clan head was frowning slightly. The next moment, the academy elder called out the third name: Gu Yue Mo Bei. + +“Here!” A horse-faced youth dressed in linen robes lightly called as he came forth. He was tall in build, looking much sturdier than his peers. There was a brave aura about him. He crossed the river in a few steps and reached the opposite bank. 10 steps, 20 steps, 30 steps; one after another small lights entered his body. He walked until he reached 36 steps before he could finally go no further. + +The youths at the riverbank watched with wide opened eyes, shocked. The academy elder happily exclaimed, “Good, Gu Yue Mo Bei, B grade talent! Come here, let me see your Primeval sea.” + +Gu Yue Mo Bei walked back to the academy elder’s side. The latter stretched out his hand and put it on the juvenile’s shoulder, closing his eyes as he checked with focus. Then he retracted his hand and nodded, recording down on the paper: Gu Yue Mo Bei, primeval sea measuring six by six, can be vigorously trained. + +This special talent can be measured by four grades – A grade to D grade. A D grade talent youth who is raised for 3 years would be able to become a rank one senior Gu Master, become the foundation of the family. A C grade talent youth after two years of cultivation will usually be able to become a rank two senior Gu Master, becoming the clan’s backbone. A B grade talent must be cared for. Often becoming a future clan elder, with 6-7 years of training they will become rank three Gu Masters. + +And when it comes to A grade, even if it was just one, would bring great luck to the entire clan. Great care must be given; with this talent in about 10 years they can become a rank four Gu Master. At that moment they would be able to compete for the position of the head of the clan! + +In other words, as long as this Gu Yue Mo Bei grows up, eventually he will become one of the elders of the Gu Yue clan. That is why the academy elder laughed happily; the elders watching in the darkness also sighed in relief, then they all turned to look at one of the elders amongst them with jealousy. + +This elder was also horse-faced, known as Gu Yue Mo Bei’s grandfather, Gu Yue Mo Chen. His face was already smiling. He provokingly looked at his old nemesis and said, “What do you think? My grandson isn’t bad huh, Gu Yue Chi Lian.” + +Gu Yue Chi Lian had a head full of red hair. He made an annoyed ‘hmph’, not replying to other. It was apparent that his face expression was really dark. + +One hour later, half of the youths had already walked through the flower sea. There were quite a number of C and D grade talents among them, while half of those youths had no talent at all. + +“Sigh, the bloodline is getting thinner. These few years the clan hasn’t had any rank four masters to strengthen the bloodline. The fourth generation clan head was the only rank five master, but in the end he perished together with the Flower Wine monk and did not leave behind any descendants. The Gu Yue clan’s later generation talents are getting weaker and weaker,” the clan head said with a deep sigh. + +At this moment, the academy elder shouted, “Gu Yue Chi Chen!” + +On hearing this name all the elders looked at Gu Yue Chi Lian; this was Gu Yue Chi Lian’s grandson. + +Gu Yue Chi Lian had a small and short build with a face full of pockmarks. He was clenching his fist, his entire face sweating. It was evident that he was incredibly nervous. + +As he walked onto the opposite bank, the little lights entered his body; after walking straight for 36 steps he stopped. + +“Another B grade!” The academy elder yelled. + +The youths started a commotion, sending Gu Yue Chi Chen envious stares. + +“Hahaha, 36 steps, 36 steps!” Gu Yue Chi Lian shouted, proudly staring at Gu Yue Mo Bei. This time it was Gu Yue Mo Chen’s turn to have a sour face. + +“Gu Yue Chi Chen, huh…” In the midst of the crowd, Fang Yuan stroked his chin thoughtfully. In his memories, the clan heavily punished Gu Yue Chi Chen because he cheated during the Awakening Ceremony. In reality Chi Chen only had a C grade talent, but because his grandfather Gu Yue Chi Lian helped him fake the results, that’s why he appeared to have B grade talent. + +To be honest if he wanted to cheat, Fang Yuan had a countless number of ways to do so, some ways even more perfect than Gu Yue Chi Chen’s method. If a B grade or A grade talent appeared, they would receive the clan’s huge care. + +But firstly, Fang Yuan had only just been reborn. It was hard to prepare the cheating method by this condition. Secondly, even if he managed to cheat, he would not be able to fake his cultivation speed. He would be exposed by then. However Gu Yue Chi Chen was different; his grandfather was Gu Yue Chi Lian – One of the two elders with the most authority within the clan. With this Chi Lian would be able to cover up for his grandson. + +“Gu Yue Chi Lian was always hostile towards Gu Yue Mo Chen, these two elders are the clan’s two biggest influential authorities. To suppress his opponent he would need his own grandson to have an outstanding talent. It is also because he was helping from behind, Gu Yue Chi Chen was able to conceal the truth for a time. In my memories, if it were not for that incident, the truth would never have been exposed.” + +Fang Yuan’s eyes shone with light, his mind thinking up ways to use this knowledge to his advantage. + +If he exposed the matter on the spot, he would receive a bit of reward from the clan, but then he would offend the highly powerful Gu Yue Chi Lian. This was not advisable. + +Within such a short time he also could not blackmail them. Due to having low status, it would just backfire on him. + +As he pondered, he suddenly heard the academy elder call out his own name: “Gu Yue Fang Yuan!” \ No newline at end of file diff --git a/tests/data/ri_4.zh b/tests/data/ri_4.zh new file mode 100644 index 0000000..9070f7d --- /dev/null +++ b/tests/data/ri_4.zh @@ -0,0 +1,151 @@ +朝阳升起来，霞光烂漫。 + +山雾不是很浓，被利剑般的阳光轻易洞穿。 + +一百多位十五岁的少年，此刻汇集在家主阁前。 + +家主阁就处在山寨的正中央，高达五层，飞檐翘角，重兵把守。阁前就是广场，阁内供奉着古月先人的牌位。每代族长也都生活起居在这里面，每逢重大典礼，或者有突发大事，也会在这里召集家老们商讨议论。这是整个山寨的权利中枢。 + +“很好，都准时来了。今天是开窍大典，是你们人生的重大转折点。闲话不多说了，随我来吧。”负责此行的，是学堂家老。他须发皆白，精神矍铄地领着少年们进入家主阁。 + +不过却没有上楼，而是通过一层大堂的入口，往下走。 + +顺着打造好的石梯，就进入地下溶洞。 + +少年们纷纷发出惊叹之声。地下溶洞美轮美奂，钟乳石散发着赤橙黄绿青蓝紫七色光华，这光彩映照在少年们的脸上，霓虹般绚烂。 + +方源混杂在人群中央，静静地审视这一切，心中暗暗思量：“数百年前，古月一族从中土迁徙到南疆，在这青茅山驻扎下来。就是看中了这里地下溶洞的一口灵泉。这灵泉产出大量元石，可以说是古月山寨的根基。” + +行了数百步，却是越来越暗，并且依稀听到了水声。 + +转过转角，一条宽三丈有余的地下河，就展现在众人眼前。 + +此地钟乳石的彩光，已经彻底消失了。 + +但是黑暗中，河水却散发着淡淡的幽蓝之光，好像是夜空中的星河。 + +河水从溶洞的黑暗深处流淌过来，清澈无比，甚至可以看到里面的游鱼，水草，以及河底的沙石。 + +在河的对岸，是一片花海。 + +这是古月一族有意栽培的月兰花，花瓣如月牙，呈现出清丽淡雅的蓝粉色。花茎如玉，花心闪耀着，好像是珍珠在光下的折射出来的温润光华。 + +乍一眼看上去，在黑暗的背景中，河畔花海就好像是一大片的蓝绿地毯，点缀着数不清的珍珠。 + +“月兰花，是很多蛊虫的食材。这片花海，可以说是家族最大的培养基地了。”方源对此心知肚明。 + +“好美。” + +“真是漂亮呀。” + +少年们算是开了眼界，一个个双眼放光，既兴奋又紧张。 + +“好了，下面听我报名，叫到的人穿过这河，到对岸去。能走多远，就走多远，当然越远越好。都听清楚了吗？”家老此刻说着。 + +“清楚了。”少年们纷纷应是。其实来之前，都听家人或者前辈们讲过，知道走的越远，代表资质越好，日后的成就也就越大。 + +“古月陈博。”家老拿着名单点出第一人。 + +河水虽宽却并不深，只及少年膝盖。陈博一脸的严肃，踏上河畔花海。 + +顿时他就感觉到一股隐形的压力，好像面前有一面看不见的墙，在阻挡他前进。 + +正举步维艰之时，脚畔的花海中忽然浮起一蓬光点，光点很稀薄，呈现素白之色。 + +光辉向陈博汇集过去，并投入到他的体内。 + +陈博瞬间感觉到压力剧减。那堵无形的墙壁，忽然变得柔软起来。 + +他咬牙用力向前走，硬生生的挤进去。走了三步之后，前方的压力又大增，一如之前如墙壁一样，不能再进分毫。 + +见到此景，家老一叹，当场一边记录，一边道：“古月陈博，三步，没有蛊师资质。下一个，古月藻榭。” + +陈博脸色顿时苍白，咬着牙，穿过河水，回到原处。没有资质，今后就只能作为一个凡人活着，在家族中也只能是最底层的地位。 + +他身躯摇摇欲坠，打击太大了，等于是扼杀了一生的希望。 + +很多人都向他投来怜悯的目光，更多的人则关注着第二位登上彼岸的少年。 + +可惜这个少年，也只能前进四步，同样没有资质。 + +并非所有人都有修行的资质，一般而言，十个人中有五人能修行，就已经不错了。在古月家族里，这个比例还要高一些，达到六人的程度。 + +这是因为古月先祖，也就是一代族长，是一位大名鼎鼎的传奇强者，因为修行的缘故导致他的血脉中隐藏着承载力量的基因。古月族人因为有着他的血脉，因此资质普遍较高。 + +连续两个没有资质的情况，让暗中关注的其他家老们都脸色难看起来，就是老成持重的古月族长，也微微蹙眉。 + +就在这时，学堂家老喊出第三个名字：“古月漠北。” + +“在！”一个身穿麻布衣衫的马脸少年，轻喝一声，越众而出。 + +他身材高大，比同龄人要粗壮得多，透着一丝彪悍气息。 + +三两步过了河，踏上对岸。 + +十步，二十步，三十步，陆续有微光投入到他的体内。 + +一直走到三十六步，终于走不动了。 + +少年们隔岸看得目瞪口呆，学堂家老欢喜得大叫起来：“好，古月漠北，乙等资质，来这里，让我看看你的元海。” + +古月漠北便又回到学堂家老的身边，后者伸出手，搭在少年的肩膀，闭目凝神探查了一番，便收回手，点点头，在纸上记录起来：“古月漠北，元海六成六，可大力栽培。” + +这资质从上到下，分甲乙丙丁四等。 + +一位丁等资质的少年，培养个三年，就能晋升成一转的资深蛊师，成为家族的基石。 + +一位丙等资质的少年，培养两年，大多都能成为二转的蛊师，成为家族的中坚存在。 + +一位乙等资质，就要呵护了。往往要作为未来的家老培养，六七年的功夫，能成为三转蛊师。 + +至于甲等资质，哪怕出现一位，都是整个家族的幸运。要细心关照，倾斜资源，十年左右就能成为四转蛊师，到那时便能竞争族长之位！ + +也就是说，这古月漠北只要成长起来，就是今后古月一族的家老。难怪学堂长老喜得哈哈一笑，而暗中关注的众家老们都统统舒了一口气，而后又纷纷向其中一位家老投去羡慕的目光。 + +这家老也是一副马脸，正是古月漠北的爷爷古月漠尘。他脸上早已经荡漾起笑意，又挑衅地看了一眼自己的老对头：“怎么样，我的孙儿不差吧，古月赤练。” + +家老古月赤练一头红发，此时冷哼一声，并未答话，脸色阴沉得很是难看。 + +半个时辰之后，已有一半少年踏足过花海，涌现了不少丙等、丁等的资质，当然毫无资质的占了几乎一半。 + +“唉，血脉越来越稀薄，加上这些年来，家族也没有出现几位四转强者，来增强血脉。四代族长是唯一的五转强者，结果却和花酒行者同归于尽，没有留下血脉后裔。古月一族后辈的资质是越来越弱了。”族长深深的叹息着。 + +就在这时，学堂家老喊道：“古月赤城。” + +听到这个名字，家老们纷纷看向古月赤练，这是古月赤练的孙子。 + +古月赤城身材矮小，满脸麻子，捏着拳头，满脸出汗，显得特别的紧张。 + +他踏上对岸，光点纷纷投入他的体内，一连走到三十六步，这才停步。 + +“又一个乙等！”学堂家老叫喊着。 + +少年们骚动起来，纷纷向古月赤城投来羡慕的目光。 + +“哈哈哈，三十六步，三十六步！”古月赤练大叫着，示威地瞪着古月漠尘。 + +这次轮到古月漠尘脸色铁青了。 + +“古月赤城么……”人群中，方源若有所思的摩挲着下巴。 + +记忆中，他因为在开窍大典中作弊，而受到了族中的严厉惩罚。 + +事实上，他的资质只有丙等，但是他的爷爷古月赤练为他作假，因此有了乙等资质的假象。 + +其实要作弊，方源心中有数十种方案，有些方案比古月赤城的更加完美。若是表现出乙等，或者甲等的资质，必然受到家族的大力栽培。 + +但是一来，方源重生的时间太短，以他的境况难以准备作弊手段。 + +二来，就算是作弊成功，日后修行时的速度是掩盖不住的，照样会露相。 + +而这个古月赤城却不一样，他的爷爷就是古月赤练，是家族中权势最重的两个家老之一，能够为他遮掩。 + +“古月赤练一直和古月漠尘敌对，这两个家老是家族中最大的两派势力。为了打压对手，他需要自己的孙子资质出众。也正是因为他在背后掩护，古月赤城才能够隐瞒一时。记忆中要不是那场意外，也不会暴露。” + +方源眼中闪烁着精芒，思量着该怎么利用这点来谋夺最大的利益呢？ + +当场揭露，虽然会得到家族的一点奖赏，但是却会得罪位高权重的古月赤练，绝不可取。 + +短时间之内，也不能敲诈勒索。因为实力太低，反而会自取其祸。 + +正思量着，忽然听到学堂家老叫出自己的名字：“古月方源！” \ No newline at end of file