68 lines
1.3 KiB
AutoHotkey
68 lines
1.3 KiB
AutoHotkey
tokenize(string, lexer)
|
|
{
|
|
stringo := string ; store original string
|
|
locationInString := 1
|
|
size := strlen(string)
|
|
tokens := object()
|
|
|
|
start:
|
|
Enum := Lexer._NewEnum()
|
|
While Enum[type, value] ; loop through regular expression lexing rules
|
|
{
|
|
if (1 == regexmatch(string, value, tokenValue))
|
|
{
|
|
token := object()
|
|
token.pos := locationInString
|
|
token.value := tokenValue
|
|
token.length := strlen(tokenValue)
|
|
token.type := type
|
|
tokens._Insert(token)
|
|
locationInString += token.length
|
|
string := substr(string, token.length + 1)
|
|
goto start
|
|
}
|
|
continue
|
|
}
|
|
if (locationInString < size)
|
|
msgbox % "unrecognized token at " substr(stringo, locationInstring)
|
|
return tokens
|
|
}
|
|
|
|
makeCalcLexer()
|
|
{
|
|
calcLexer := object()
|
|
PLUS := "\+"
|
|
MINUS := "-"
|
|
MULT := "\*"
|
|
DIV := "/"
|
|
OPEN := "\("
|
|
CLOSE := "\)"
|
|
NUMBER := "\d+"
|
|
WS := "[ \t\n]+"
|
|
END := "\."
|
|
RULES := "PLUS,MINUS,MULT,DIV,OPEN,CLOSE,NUMBER,WS,END"
|
|
loop, parse, rules, `,
|
|
{
|
|
type := A_LoopField
|
|
value := %A_LoopField%
|
|
calcLexer._Insert(type, value)
|
|
}
|
|
return calcLexer
|
|
}
|
|
|
|
printTokens(tokens)
|
|
{
|
|
loop % tokens._MaxIndex()
|
|
{
|
|
tokenString .= printToken(tokens[A_Index]) "`n`n"
|
|
}
|
|
return tokenString
|
|
}
|
|
|
|
|
|
printToken(token)
|
|
{
|
|
string := "pos= " token.pos "`nvalue= " token.value "`ntype= " token.type
|
|
return string
|
|
}
|