JonKazama-Hellion/HellionChat
1
0
Fork 0
Code Issues Pull Requests Actions Releases 28 Activity

- More ImRaii

Browse Source 
- Cleanup
- Spanish, French, Dutch, Chinese loc update
This commit is contained in:
Infi
2024-11-21 11:58:22 +01:00
parent 1a1995759a
commit 3cfe65d2d4
23 changed files with 283 additions and 332 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ChatTwo/Util/Tokenizer.cs
+15 -15
View File
@@ -36,21 +36,21 @@ public static class Tokenizer
static PrecedenceBasedRegexTokenizer()
{
TokenDefinitions = new List<TokenDefinition>
{
new(TokenType.CloseParenthesis, "\\)", 1),
new(TokenType.Comma, ",", 1),
new(TokenType.Dot, "\\.", 1),
new(TokenType.QuestionMark, "\\?", 1),
new(TokenType.ExclamationMark, "!", 1),
new(TokenType.Semicolon, ";", 1),
new(TokenType.Whitespace, "\\s", 1),
new(TokenType.Equals, "=", 1),
new(TokenType.OpenParenthesis, "\\(", 1),
new(TokenType.UrlString, URLRegex, 1),
new(TokenType.StringValue, "\\p{IsBasicLatin}", 2),
new(TokenType.Leftover, ".", 3)
};
TokenDefinitions =
[
new TokenDefinition(TokenType.CloseParenthesis, "\\)", 1),
new TokenDefinition(TokenType.Comma, ",", 1),
new TokenDefinition(TokenType.Dot, "\\.", 1),
new TokenDefinition(TokenType.QuestionMark, "\\?", 1),
new TokenDefinition(TokenType.ExclamationMark, "!", 1),
new TokenDefinition(TokenType.Semicolon, ";", 1),
new TokenDefinition(TokenType.Whitespace, "\\s", 1),
new TokenDefinition(TokenType.Equals, "=", 1),
new TokenDefinition(TokenType.OpenParenthesis, "\\(", 1),
new TokenDefinition(TokenType.UrlString, URLRegex, 1),
new TokenDefinition(TokenType.StringValue, "\\p{IsBasicLatin}", 2),
new TokenDefinition(TokenType.Leftover, ".", 3)
];
}
public static IEnumerable<Token> Tokenize(string lqlText)