24 lines
626 B
Bash
Executable File
24 lines
626 B
Bash
Executable File
#!/bin/sh
|
|
|
|
if [ $# -ne 3 ]; then
|
|
printf "Usage: %s input_file max_chars output_file\n" "$0"
|
|
printf "Example: %s data.jsonl 10000 cleaned.jsonl\n" "$0"
|
|
exit 1
|
|
fi
|
|
|
|
input_file=$1
|
|
max_chars=$2
|
|
output_file=$3
|
|
|
|
# Check if input file exists
|
|
if [ ! -f "$input_file" ]; then
|
|
printf "Error: Input file '%s' not found\n" "$input_file"
|
|
exit 1
|
|
fi
|
|
|
|
# Process the file: keep lines shorter than max_chars
|
|
awk -v max="$max_chars" 'length($0) <= max' "$input_file" >"$output_file"
|
|
|
|
printf "Processing complete. Lines longer than %s characters have been removed.\n" "$max_chars"
|
|
printf "Result saved to: %s\n" "$output_file"
|