Remove tokens from the prompt that the Java side can't handle

This commit is contained in:
tiferet
2022-12-06 16:37:11 -08:00
parent 12def779e6
commit 4ed57e71db

View File

@@ -35,11 +35,18 @@ module ModelPrompt {
strictconcat(Token token |
location.containsLoosely(token.getLocation())
|
token.getValue(),
// Use space as the separator, since that is most likely.
// May not be an exact reconstruction, e.g. if the code
// had newlines between successive tokens.
" "
token
.getValue()
.replaceAll("\"", " ")
.replaceAll("\\", " ")
.replaceAll("\n", " ")
.replaceAll("\r", " ")
.replaceAll("|", " ")
.replaceAll("`", " ") +
// Use space as the separator, since that is most likely.
// May not be an exact reconstruction, e.g. if the code
// had newlines between successive tokens.
" "
order by
token.getLocation().getStartLine(), token.getLocation().getStartColumn()
)