2020-03-18 07:00:12 +03:00
|
|
|
# Slang tokens
|
|
|
|
|
2024-01-11 02:55:07 +03:00
|
|
|
const str whitespace = " \t\r\v\f"
|
2020-03-18 07:00:12 +03:00
|
|
|
|
|
|
|
tuple lstripcount(str s, str chars) {
|
|
|
|
int ii = -1
|
|
|
|
char i
|
2024-01-11 02:55:07 +03:00
|
|
|
|
2020-03-18 07:00:12 +03:00
|
|
|
for i in s {
|
|
|
|
ii += 1
|
2024-01-11 02:55:07 +03:00
|
|
|
if i not in chars: break
|
|
|
|
} else: ii = 0
|
|
|
|
|
|
|
|
return (ii, s[ii:])
|
2020-03-18 07:00:12 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
class Token {
|
2024-01-11 02:55:07 +03:00
|
|
|
const tuple types = ('SPECIAL', 'OPERATOR', 'LITERAL', 'KEYWORD', 'IDENTIFIER') # order is also resolution order
|
2020-03-18 07:00:12 +03:00
|
|
|
|
2024-01-11 02:55:07 +03:00
|
|
|
int type, lineno, offset
|
|
|
|
str token
|
2020-03-18 07:00:12 +03:00
|
|
|
|
2024-01-11 02:55:07 +03:00
|
|
|
constr (int .type, str .token, int .lineno, int .offset);
|
2020-03-18 07:00:12 +03:00
|
|
|
|
2024-01-11 02:55:07 +03:00
|
|
|
repr = f"<Token {.typename} «{repr(.token)[1:-1]}» at line {.lineno}, offset {.offset}>"
|
2020-03-18 07:00:12 +03:00
|
|
|
|
2024-01-11 02:55:07 +03:00
|
|
|
eq = (super == x or .token == x)
|
2020-03-18 07:00:12 +03:00
|
|
|
|
2024-01-11 02:55:07 +03:00
|
|
|
property typename = .types[.type]
|
2020-03-18 07:00:12 +03:00
|
|
|
|
2024-01-11 02:55:07 +03:00
|
|
|
property length = .token.length
|
2020-03-18 07:00:12 +03:00
|
|
|
}
|
|
|
|
|
2024-01-11 02:55:07 +03:00
|
|
|
# by Sdore, 2021
|
|
|
|
# slang.sdore.me
|