2019-08-28 21:24:54 +03:00
#!/usr/bin/python3
# Slang lexer
from . tokens import *
def read_token ( src , * , lineno , offset , lineoff ) :
( l , src ) , line = lstripcount ( src [ offset : ] , whitespace ) , src
offset + = l
if ( src [ : 1 ] in ' \n ; ' ) : return ( offset , None )
2024-01-11 02:55:07 +03:00
err = ( 0 , 0 )
2019-08-28 21:24:54 +03:00
for ii , i in enumerate ( Token . types ) :
2024-01-11 02:55:07 +03:00
r = globals ( ) [ ' find_ ' + i . casefold ( ) ] ( src )
n , s = r if ( isinstance ( r , tuple ) ) else ( r , src [ : r ] ) if ( isinstance ( r , int ) and r > 0 ) else ( 0 , None )
if ( isinstance ( n , int ) and n < = 0 ) : err = max ( err , ( - n , s if ( isinstance ( s , int ) ) else 0 ) ) ; continue
2019-08-28 21:24:54 +03:00
return ( offset + n , Token ( ii , s , lineno = lineno , offset = offset + lineoff ) )
2024-01-11 02:55:07 +03:00
else : raise SlSyntaxError ( " Invalid token " , [ None ] * ( lineno - 1 ) + line . split ( ' \n ' ) , lineno = lineno , offset = offset + lineoff , length = err [ 0 ] + l , char = err [ 1 ] )
2019-08-28 21:24:54 +03:00
def parse_expr ( src , * , lineno = 1 , lineoff = 0 ) :
r = list ( )
2020-03-18 07:00:12 +03:00
lines = src . count ( ' \n ' )
2019-08-28 21:24:54 +03:00
offset = int ( )
2020-03-18 07:00:12 +03:00
continueln = False
2019-08-28 21:24:54 +03:00
while ( True ) :
2020-03-18 07:00:12 +03:00
offset , tok = read_token ( src , lineno = lines - src [ offset : ] . count ( ' \n ' ) + lineno , offset = offset , lineoff = lineoff )
if ( tok is None ) :
if ( not continueln ) : break
continueln = False
offset + = 1
lineoff = - offset
continue
elif ( continueln and tok . token [ 0 ] != ' # ' ) : raise SlSyntaxError ( " Expected newline or comment after line continuation " , src , lineno = lines - src [ offset : ] . count ( ' \n ' ) + lineno , offset = tok . offset , length = tok . length )
2019-08-28 21:24:54 +03:00
r . append ( tok )
2020-03-18 07:00:12 +03:00
if ( tok . token [ 0 ] != ' # ' ) : continueln = ( tok . token == ' \\ ' and tok . offset )
2019-08-28 21:24:54 +03:00
return offset , r
2020-03-18 07:00:12 +03:00
def parse_string ( src , lnooff = 0 ) :
2019-08-28 21:24:54 +03:00
src = src . rstrip ( )
tl = list ( )
2020-03-18 07:00:12 +03:00
lines = src . count ( ' \n ' ) + lnooff
2019-08-28 21:24:54 +03:00
lineoff = int ( )
while ( src ) :
offset , r = parse_expr ( src , lineno = lines - src . count ( ' \n ' ) + 1 , lineoff = lineoff )
lineoff + = offset
if ( offset < len ( src ) ) :
if ( src [ offset ] == ' \n ' ) : lineoff = int ( )
else : lineoff + = 1
src = src [ offset + 1 : ]
tl . append ( r )
return tl
2020-03-18 07:00:12 +03:00
# by Sdore, 2020