mirror of https://github.com/egormanga/Slang.git
36 lines
666 B
Plaintext
36 lines
666 B
Plaintext
# Slang tokens
|
|
|
|
const str whitespace = " \t\r\v\f"
|
|
|
|
tuple lstripcount(str s, str chars) {
|
|
int ii = -1
|
|
char i
|
|
|
|
for i in s {
|
|
ii += 1
|
|
if i not in chars: break
|
|
} else: ii = 0
|
|
|
|
return (ii, s[ii:])
|
|
}
|
|
|
|
class Token {
|
|
const tuple types = ('SPECIAL', 'OPERATOR', 'LITERAL', 'KEYWORD', 'IDENTIFIER') # order is also resolution order
|
|
|
|
int type, lineno, offset
|
|
str token
|
|
|
|
constr (int .type, str .token, int .lineno, int .offset);
|
|
|
|
repr = f"<Token {.typename} «{repr(.token)[1:-1]}» at line {.lineno}, offset {.offset}>"
|
|
|
|
eq = (super == x or .token == x)
|
|
|
|
property typename = .types[.type]
|
|
|
|
property length = .token.length
|
|
}
|
|
|
|
# by Sdore, 2021
|
|
# slang.sdore.me
|