chore: stuff

scripts/avg_length.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+if [ $# -ne 1 ]; then
+    echo "Usage: $0 <filename>"
+    exit 1
+fi
+
+file="$1"
+
+# Check if file exists
+if [ ! -f "$file" ]; then
+    echo "Error: File '$file' does not exist"
+    exit 1
+fi
+
+# Calculate average line length and store line lengths with their content
+avg_length=$(awk '{ 
+    total_length += length($0)
+    count++
+} 
+END { 
+    if (count > 0) printf "%.0f", total_length/count 
+}' "$file")
+
+# Find the line closest to average length
+awk -v target=$avg_length '
+{
+    curr_length = length($0)
+    diff = sqrt((curr_length - target)^2)
+    if (NR == 1 || diff < min_diff) {
+        min_diff = diff
+        closest_line = $0
+        line_num = NR
+        actual_length = curr_length
+    }
+}
+END {
+    print "Average line length: " target
+    print "Closest matching line (#" line_num "):"
+    print closest_line
+    print "Length: " actual_length
+}' "$file"

scripts/jsonl_to_json.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+[ "$#" -eq 2 ] || {
+    echo "Usage: $0 input.jsonl output.json" >&2
+    exit 1
+}
+jq -s '.' <"$1" >"$2"

scripts/shit_names.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+# For dumb fucking Calibre splits
+# Usage: ./find_b_chapters.sh [path_to_epub_root]
+
+EPUB_ROOT="${1:-custom/ast/en/OEBPS/Text}"
+
+if [ ! -d "$EPUB_ROOT" ]; then
+    echo "Error: Directory '$EPUB_ROOT' not found" >&2
+    exit 1
+fi
+
+find "$EPUB_ROOT" -type f -name "Section*B.xhtml" | while read -r file; do
+    basename "$file"
+    echo "$file"
+done | sort -t'n' -k1.8,1.11n
+
+exit 0

scripts/truncate.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+
+if [ $# -ne 3 ]; then
+    printf "Usage: %s input_file max_chars output_file\n" "$0"
+    printf "Example: %s data.jsonl 10000 cleaned.jsonl\n" "$0"
+    exit 1
+fi
+
+input_file=$1
+max_chars=$2
+output_file=$3
+
+# Check if input file exists
+if [ ! -f "$input_file" ]; then
+    printf "Error: Input file '%s' not found\n" "$input_file"
+    exit 1
+fi
+
+# Process the file: keep lines shorter than max_chars
+awk -v max="$max_chars" 'length($0) <= max' "$input_file" >"$output_file"
+
+printf "Processing complete. Lines longer than %s characters have been removed.\n" "$max_chars"
+printf "Result saved to: %s\n" "$output_file"