don't treat 's as strictly punctuation

This commit is contained in:
2024-05-31 16:08:43 -06:00
parent e7cf386c76
commit 85cc3ee2ae

View File

@@ -16,9 +16,12 @@
"-"
"!"
"?"
"'s" // possessive
])
(var punctuationPlusSome (concat punctuation [
"'s" // possessive
]))
(var delimiters [
" "
"\n"
@@ -44,7 +47,7 @@
((apply or (for quote quotesAndThings (token.endsWith quote)))
(normalize (substr token 0 -1) toLower))
// Remove punctuation after
((apply or (for punct punctuation (token.endsWith punct)))
((apply or (for punct punctuationPlusSome (token.endsWith punct)))
(normalize (substr token 0 -1) toLower))
// Lower-case
(toLower (token.toLowerCase))