Huge work; i'm too tired to write commit message for all of it. Pleez let me sleep acouple of hours. Made most tests work. Started SBC.

This commit is contained in:
egormanga
2020-03-18 07:00:12 +03:00
parent 7badd31e42
commit 27d951227d
51 changed files with 3004 additions and 717 deletions

17
sl/Slang/Slang.sl Normal file
View File

@@ -0,0 +1,17 @@
# Slang
import lexer:*
str src = "main {print('hello');}"
main {
stdio.println("Source: {"+src+"}\n")
list tl = parse_string(src)
stdio.println(tl)
stdio.println("Tokens:")
stdio.println(tl)
stdio.println("\n")
}
# by Sdore, 2020

74
sl/Slang/lexer.sl Normal file
View File

@@ -0,0 +1,74 @@
# Slang lexer
import tokens:*
tuple read_token(str src, int lineno, int offset, int lineoff) {
tuple c = lstripcount(src#|[offset:]|#, whitespace)
int l = c[0]
src = c[1]
str line = src
offset += l
if not src or src[0] in '\n;' {
return (offset, None)
}
int length = 0
int ii = -1
for i in Token.types {
ii += 1
auto r = globals['find_'+i.casefold()](src) or 0
if r isof int and r <= 0 {
length = max(length, -r)
continue
}
tuple c
if r isof tuple {
c = r
}
else {
c = (r, src[:r])
}
int n = c[0]
str s = c[1]
return (offset+n, Token(ii, s, lineno=lineno, offset=offset+lineoff))
} #else raise SlSyntaxError("Invalid token", line, lineno=lineno, offset=offset+lineoff, length=length)
return (0, Token())
}
tuple parse_expr(str src, int lineno = 1, int lineoff = 0) {
list r = [Token]
int offset
while 1 {
offset, tok = read_token(src, lineno=lineno, offset=offset, lineoff=lineoff)
if tok is None {
break
}
r.append(tok)
}
return (offset, r)
}
list parse_string(str src) {
src = src.rstrip()
list tl = [Token]
int lines = src.count('\n')
int lineoff = 0
while src {
tuple c = parse_expr(src, lineno=lines-src.count('\n')+1, lineoff=lineoff)
int offset = c[0]
list r = c[1]
lineoff += offset
if offset < src.len {
if src[offset] == '\n' {
lineoff = 0
}
else {
lineoff += 1
}
}
src = src[offset+1:]
tl.append(r)
}
return tl
}
# by Sdore, 2020

49
sl/Slang/tokens.sl Normal file
View File

@@ -0,0 +1,49 @@
# Slang tokens
str whitespace = ' \t\r\v\f'
tuple lstripcount(str s, str chars) {
int ii = -1
char i
for i in s {
ii += 1
if i not in chars {
break
}
}
else {
ii = 0
}
return (ii, s#|[ii:]|#)
}
class Token {
int type
str token
int lineno
int offset
tuple types# = ('SPECIAL', 'OPERATOR', 'LITERAL', 'KEYWORD', 'IDENTIFIER') # order is also resolution order
constr(int type, str token, int lineno, int offset) {
#.type, .token, .lineno, .offset = type, token, lineno, offset
}
#|repr {
return "<Token {.types[.type]} «{repr(.token)[1:-1]}» at line {.lineno}, offset {.offset}>"
}|#
#|eq {
#return super() == x or .token == x
}|#
#|property typename {
#return .types[.type]
}
property length {
#return .token.length
}|#
}
# by Sdore, 2020