first commit

bin/vecalign/overlap.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+
+"""
+Copyright 2019 Brian Thompson
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+
+import argparse
+
+from dp_utils import yield_overlaps
+
+
+def go(output_file, input_files, num_overlaps):
+    output = set()
+    for fin in input_files:
+        lines = open(fin, 'rt', encoding="utf-8").readlines()
+        for out_line in yield_overlaps(lines, num_overlaps):
+            output.add(out_line)
+
+    # for reproducibility
+    output = list(output)
+    output.sort()
+
+    with open(output_file, 'wt', encoding="utf-8") as fout:
+        for line in output:
+            fout.write(line + '\n')
+
+
+def _main():
+    parser = argparse.ArgumentParser('Create text file containing overlapping sentences.',
+                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+
+    parser.add_argument('-i', '--inputs', type=str, nargs='+',
+                        help='input text file(s).')
+
+    parser.add_argument('-o', '--output', type=str,
+                        help='output text file containing overlapping sentneces')
+
+    parser.add_argument('-n', '--num_overlaps', type=int, default=4,
+                        help='Maximum number of allowed overlaps.')
+
+    args = parser.parse_args()
+    go(output_file=args.output,
+       num_overlaps=args.num_overlaps,
+       input_files=args.inputs)
+
+
+if __name__ == '__main__':
+    _main()