Latest changes back from 2021, before starting from scratch.

This commit is contained in:
Egor Vorontsov 2024-01-11 02:55:07 +03:00
parent b8495b4cc3
commit 6bdc8cce09
Signed by: Egor Vorontsov
GPG Key ID: DDAF621F6C6FEB43
64 changed files with 2113 additions and 1577 deletions

2
.gitattributes vendored Normal file
View File

@ -0,0 +1,2 @@
*.sl linguist-detectable=false
*.sld linguist-detectable=false

11
.gitignore vendored
View File

@ -1,8 +1,3 @@
old.c
**.db
**.log
**.pyc
**.old
**.bak
**__pycache__
**_config.py
*.old
*.bak
__pycache__/

View File

@ -1 +0,0 @@
Slang.md

View File

@ -1,50 +0,0 @@
#include <stdarg.h>
#include <string.h>
typedef struct atom {
enum {b, i8, u8, i16, u16, i32, u32, i64, u64, i128, u128} type;
union {
_Bool* b;
int8_t* i8;
uint8_t* u8;
int16_t* i16;
uint16_t* u16;
int32_t* i32;
uint32_t* u32;
int64_t* i64;
uint64_t* u64;
void* data;
};
} atom_t;
typedef atom_t (*builtin_function)(int nargs, atom_t args[nargs]);
typedef struct builtin {
const char* name;
builtin_function fp;
} builtin_t;
/// XXX ///
atom_t _builtin_println(int nargs, atom_t args[nargs]) {
static int res;
res = puts(args[0].data);
return (atom_t){i32, .i32 = &res};
}
/// XXX ///
builtin_t builtins[] = {
{"println", _builtin_println},
{NULL, NULL}};
builtin_function get_builtin(const char* name) {
for (builtin_t* i = builtins; i->name != NULL; i++)
if (strcmp(i->name, name) == 0)
return i->fp;
return NULL;
}

View File

@ -1,171 +0,0 @@
# Slang Bytecode
## Standalone
### NOP
0x00
> Does nothing.
### END
0x01
> Closes previously opened block.
### POP
0x02
> Drops `TOS`.
### RET
0x03
> Returns `TOS` to caller.
### BLTIN
0x04
> Reads string from the bytecode until null byte (max 255 bytes) and pushes builtin function with that name.
### CODE
0x05
> Reads code from the bytecode until the corresponding `END` instruction and pushes a reference to it and then its length.
> Opens a block.
## Unary
### POS
0x10
> Pushes `abs(TOS)`.
### NEG
0x11
> Pushes `-TOS`.
### NOT
0x12
> Pushes `!TOS`.
### INV
0x13
> Pushes `~TOS`.
### ATOI
0x14
> Pushes integer of smallest possible width parsed from string `TOS`.
### ITOA
0x15
> Pushes string representation of integer `TOS`.
### ITOF
0x16
> Pushes real of smallest possible width equal to integer `TOS`.
### CEIL
0x17
> Pushes smallest integer of smallest possible width greater or equal to real `TOS`.
### FLR
0x18
> Pushes largest integer of smallest possible width less or equal to real `TOS`.
### RND
0x19
> Pushes integer of smallest possible width equal to rounded `TOS`.
### CTOS
0x1A
> Pushes string consisting of char `TOS` and a null byte.
## Binary
### ADD
0x20
> Pushes `TOS1 + TOS`.
### SUB
0x21
> Pushes `TOS1 - TOS`.
### MUL
0x22
> Pushes `TOS1 * TOS`.
### DIV
0x23
> Pushes `TOS1 / TOS`.
### IDIV
0x24
> Pushes `TOS1 // TOS`.
### MOD
0x25
> Pushes `TOS1 % TOS`.
### POW
0x26
> Pushes `TOS1 ** TOS`.
### LSH
0x27
> Pushes `TOS1 << TOS`.
### RSH
0x28
> Pushes `TOS1 >> TOS`.
### AND
0x29
> Pushes `TOS1 & TOS`.
### OR
0x2A
> Pushes `TOS1 | TOS`.
### XOR
0x2B
> Pushes `TOS1 ^ TOS`.
## Comparisons
### EQ
0x30
> Pushes `TOS1 == TOS`.
### NE
0x31
> Pushes `TOS1 != TOS`.
### LT
0x32
> Pushes `TOS1 < TOS`.
### GT
0x33
> Pushes `TOS1 > TOS`.
### LE
0x34
> Pushes `TOS1 <= TOS`.
### GE
0x35
> Pushes `TOS1 >= TOS`.
### IS
0x36
> Pushes `TOS1 is TOS`.
### ISNOT
0x37
> Pushes `TOS1 is not TOS`.
## Flow control
### IF
0x40
> If `TOS` is false, skips bytecode until corresponding `ELSE` (if exists) or `END`.
> Opens a block.
### ELSE
0x41
> Pops last `IF` result from `IF`-stack, and if it is true, skips bytecode to corresponding `END`.
> Opens a block.
### EXEC
0x42
> Executes code block `TOS1` of length `TOS` and pushes the result.
## With argument
### ALLOC*(bytes)*
0xA0
> Pushes reference to `calloc(1, bytes)`.
### EXTEND*(bytes)*
0xA1
> Extends integer `TOS` width to `bytes` bytes if narrower.
### CONST*(bytes)*
0xA2
> Reads next `bytes` bytes of bytecode and pushes a reference to a copy of them.
### JUMPF*(offset)*
0xA3
> Jumps `offset` bytes of bytecode forward.
### JUMPB*(offset)*
0xA4
> Jumps `offset` bytes of bytecode backward.
### SCPGET*(cell)*
0xA5
> Pushes the value of cell `cell` of local scope variables array.
### SCPSET*(cell)*
0xA6
> Sets the value of cell `cell` of local scope variables array to `TOS`.
### CALL*(nargs)*
0xA7
> Calls `TOS` with `nargs` arguments popped from stack (below the callable).

View File

@ -1,50 +0,0 @@
#!/usr/bin/python3
# SBC Disassembler
from utils.nolog import *
opcodes = {int(m[2], 16): m[1] for m in re.finditer(r'### (\w+).*?\n(\w+)', open(os.path.join(os.path.dirname(sys.argv[0]), 'bytecode.md')).read())}
HASARG = 0xa0
def dissbc(code, no_opnames=False):
cp = 0
codesize = len(code)
blocklvl = 0
while (cp < codesize):
opcode = code[cp]; cp += 1
opname = opcodes.get(opcode)
if (opname == 'END'): blocklvl -= 1
print('\t'*blocklvl, end='')
if (opname is None): print("\033[2mUNKNOWN: %02x\033[0m" % opcode); continue
print(f"\033[1m0x{opcode:02x}{' '+opname if (not no_opnames) else ''}\033[0m", end='')
if (opcode > HASARG):
arg = code[cp]; cp += 1
print("(%d|0x%02x)" % (arg, arg), end='')
if (opname == 'CONST'):
const = code[cp:cp+arg]
cp += arg
print(':', '0x'+const.hex(), end='')
print(' |', str().join(chr(i) if (32 <= i < 128) else '.' for i in const), end='')
elif (opname == 'BLTIN'):
name = bytearray()
while (code[cp] != 0):
name.append(code[cp]); cp += 1
else: cp += 1
print(':', name.decode('ascii'), end='')
if (opname in ('IF', 'ELSE', 'CODE')):
print(':', end='')
blocklvl += 1
print()
@apmain
@aparg('file', metavar='<file.sbc>')
@aparg('--no-opnames', action='store_true')
def main(cargs):
dissbc(open(cargs.file, 'rb').read(), no_opnames=cargs.no_opnames)
if (__name__ == '__main__'): exit(main(nolog=True), nolog=True)
# by Sdore, 2020

BIN
SBC/sbc

Binary file not shown.

217
SBC/sbc.c
View File

@ -1,217 +0,0 @@
// SBC
#define _GNU_SOURCE
#include <ctype.h>
#include <stdio.h>
#include <assert.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include "builtins.c"
#include "stack.c"
typedef uint8_t code_t;
enum {
// Standalone
NOP = 0x00,
END = 0x01,
POP = 0x02,
RET = 0x03,
BLTIN = 0x04,
CODE = 0x05,
// Unary
POS = 0x10,
NEG = 0x11,
NOT = 0x12,
INV = 0x13,
ATOI = 0x14,
ITOA = 0x15,
ITOF = 0x16,
CEIL = 0x17,
FLR = 0x18,
RND = 0x19,
CTOS = 0x1A,
// Binary
ADD = 0x20,
SUB = 0x21,
MUL = 0x22,
DIV = 0x23,
IDIV = 0x24,
MOD = 0x25,
POW = 0x26,
SHL = 0x27,
SHR = 0x28,
AND = 0x29,
OR = 0x2A,
XOR = 0x2B,
// Comparisons
EQ = 0x30,
NE = 0x31,
LT = 0x32,
GT = 0x33,
LE = 0x34,
GE = 0x35,
IS = 0x36,
ISNOT = 0x37,
// Flow control
IF = 0x40,
ELSE = 0x41,
EXEC = 0x42,
// With argument
ALLOC = 0xA0,
EXTEND = 0xA1,
CONST = 0xA2,
JUMPF = 0xA3,
JUMPB = 0xA4,
SCPGET = 0xA5,
SCPSET = 0xA6,
CALL = 0xA7,
HASARG = 0xA0,
};
atom_t exec(code_t* code, uint32_t codesize) { // TODO: freeing
stack_t* st = stack();
atom_t scp[255]; // TODO
code_t* cb[255]; // TODO
uint32_t cbi = 0;
uint32_t cp = 0;
while (cp < codesize) {
code_t opcode = code[cp++];
uint32_t ocp = cp;
switch (opcode) {
// Standalone
case NOP: break;
case END: break;
case POP: stack_pop(st); break;
case RET: return st->top->data;
case BLTIN: {
char name[256];
strncpy(name, (char*)code+cp, 255);
while (code[cp++] != '\0');
stack_push(st, (atom_t){.data = get_builtin(name)});
}; break;
case CODE: {
code_t* code_block = malloc(codesize);
static uint32_t cbp = 0;
uint32_t blocklvl = 1;
while (cp < codesize) {
code_t c = code[cp++];
if (c == CODE ||
c == IF ||
c == ELSE) blocklvl++;
else if (c == END) blocklvl--;
if (blocklvl <= 0) break;
code_block[cbp++] = c;
if (c > HASARG) code_block[cbp++] = code[cp++];
if (c == CONST)
for (uint8_t i = code_block[cbp-1]; i > 0; i--)
code_block[cbp++] = code[cp++];
else if (c == BLTIN)
do code_block[cbp++] = code[cp++];
while (code[cp-1] != '\0');
}
cb[cbi++] = realloc(code_block, cbp);
free(code_block);
stack_push(st, (atom_t){u32, .u32 = &cbp});
stack_push(st, (atom_t){.data = &cb[cbi-1]});
}; break;
// Unary
case POS: *st->top->data.i32 = abs(*st->top->data.i32); break;
case ITOA: {
char s[12];
fprintf(stderr, "-- %x\n", *st->top->data.i32);
snprintf(s, sizeof(s)/sizeof(*s), "%d", *st->top->data.i32);
st->top->data.data = strdup(s);
}; break;
// Binary (TODO)
case ADD: *st->top->data.i32 += *stack_pop(st).i32; break;
case SUB: *st->top->data.i32 -= *stack_pop(st).i32; break;
// Comparisons
case LT: *st->top->data.b = *stack_pop(st).i32 > *st->top->data.i32; break;
// Flow control
case EXEC: {
uint32_t exec_codesize = *stack_pop(st).u32;
code_t* exec_code = stack_pop(st).data;
stack_push(st, exec(exec_code, exec_codesize));
}; break;
// With argument
case CONST: {
uint8_t len = code[cp++];
stack_push(st, (atom_t){.data = memcpy(malloc(len), code+cp, len)});
fprintf(stderr, "-- l=%02x: %x\n", len, *st->top->data.i32);
cp += len;
}; break;
case SCPGET: {
uint8_t cell = code[cp++];
stack_push(st, scp[cell]);
}; break;
case SCPSET: {
uint8_t cell = code[cp++];
scp[cell] = stack_pop(st);
fprintf(stderr, "-- c%d = %d\n", cell, *scp[cell].i32);
}; break;
case CALL: {
uint8_t nargs = code[cp++];
fprintf(stderr, "-- nargs=%d\n", nargs);
builtin_function func = stack_pop(st).data;
atom_t args[nargs];
for (uint8_t i = 0; i < nargs; i++)
args[i] = stack_pop(st);
stack_push(st, func(nargs, args));
}; break;
default: fprintf(stderr, "Not Implemented opcode: 0x%02x\n", opcode); exit(3);
}
fprintf(stderr, "[%02x", opcode);
if (opcode > HASARG) fprintf(stderr, "(%d|0x%02x)", code[ocp], code[ocp++]);
fputc(':', stderr);
do fprintf(stderr, " %02x", code[ocp++]);
while (ocp < cp && ocp < codesize);
if (st->top != NULL) {
fprintf(stderr, ", TOS = %u(0x%02x)", *st->top->data.i32, *st->top->data.i32);
if (isprint(*(char*)st->top->data.data)) {
fprintf(stderr, " | ");
for (char* p = st->top->data.data; *p != '\0'; p++)
fputc(isprint(*p)?*p:'.', stderr);
}
}
fprintf(stderr, "]\n");
}
return (atom_t){.data = NULL};
}
int main(int argc, char* argv[]) {
if (argc != 2) {
fprintf(stderr, "Usage: %s <file.sbc>\n", basename(argv[0]));
exit(1);
}
FILE* fd = fopen(argv[1], "rb");
fseek(fd, 0, SEEK_END);
long fsize = ftell(fd);
fseek(fd, 0, SEEK_SET);
code_t code[fsize];
fread(code, sizeof(*code), fsize, fd);
fclose(fd);
exec(code, fsize);
}
// by Sdore, 2020

View File

@ -1,53 +0,0 @@
/*
#define CAT(a, b) a##_##b
#define TEMPLATE(name, type) CAT(name, type)
#define stack TEMPLATE(stack, T)
#define stack_t TEMPLATE(stack_t, T)
#define stack_item TEMPLATE(stack_item, T)
#define stack_push TEMPLATE(stack_push, T)
#define stack_pop TEMPLATE(stack_pop, T)
*/
#define T atom_t
typedef struct stack {
struct stack_item* top;
} stack_t;
struct stack_item {
T data;
struct stack_item* below;
};
stack_t* stack() {
stack_t* st = malloc(sizeof(*st));
st->top = NULL;
return st;
}
void stack_push(stack_t* st, T data) {
struct stack_item* new = malloc(sizeof(*new));
new->data = data;
if (st->top != NULL) new->below = st->top;
st->top = new;
}
T stack_pop(stack_t* st) {
assert (st->top != NULL);
struct stack_item* item = st->top;
st->top = item->below;
T data = item->data;
free(item);
return data;
}
/*
#undef stack
#undef stack_t
#undef stack_item
#undef stack_push
#undef stack_pop
*/
#undef T

Binary file not shown.

View File

@ -1,4 +1,4 @@
# Slang
# Slang
<!-- <font size=0>(June, 30 12:04 AM draft)</font> -->
### Example code
@ -8,26 +8,26 @@
#| and that is a
multiline one. |#
const u32 n = 123123 # n is of type const u32
const i64 m = 10**18 # m is of type const i64
const int z = 2**128 # z is of type const int (unsized)
const auto q = 2**256 # q is of type const int
const u32 n = 123123 # n: const u32 (unsigned 32 bit)
const i64 m = 10**18 # m: const i64 (signed 64 bit)
const int z = 2**128 # z: const int (signed unsized)
const auto q = 2**256 # q: const int (signed unsized)
char f(str x) { # f() is of type char, x is of type str
auto c = x[1] # c is of type char
return c
char f(str x) { # f(): char, x: str
auto c = x[1] # c: char
return c # char
}
auto g(str x) { # g() is of type char, x is of type str
return x[0] # retval is of type char
auto g(str x) { # g(): char, x: str
return x[0] # char
}
int h(int x) = x+1 # h() is of type int, x is of type int
int h(int x) = x+1 # h(): int, x: int
main {
stdio.println(h(n), \ # comment here too
f('123asd') + g('32') + 1) #--> «123124 f»
stdio.println(q/z/2**96) #--> «4294967296.0»
stdio.println(h(n), \ # comments allowed here too
f('123asd') + g('32') + 1) #--> «123124 f»
stdio.println(q/z/2**96) #--> «4294967296.0»
}
```
@ -58,8 +58,7 @@ _Note: `*` after syntax unit means any number of them._
### Primitive
* `<<literal> | <funccall> | <attrget> | <itemget> | <identifier> | <lambda>>``value`
* `<value>\[<value>\]``itemget`
* `<<funccall> | <itemget> | <attrget> | <identifier> | <lambda> | <literal>>``value`
* `<(<expr>) | <value> | <operator> <expr> | <expr> <operator> <expr>>``expr` (`f(x+3)` is an instance of `expr`, also `f`, `x+3` and `x` are `expr`s too)
### Non-final
@ -73,15 +72,17 @@ _Note: `*` after syntax unit means any number of them._
> `=` — default value if argument not specified.
* `([<argdef>[, <argdef>]*]) -> <typedef> = <expr>``lambda` (lambda function)
* `<<expr>[, <expr>]*[, *<expr>] | *<expr>>``callargs`
* `<<identifier>=<expr>[, <identifier>=<expr>]*[, **<expr>] | **<expr>>``callkwargs`
* `<<identifier> =|: <expr>[, <identifier> =|: <expr>]*[, **<expr>] | **<expr>>``callkwargs`
* `<value>\[<expr>\]``itemget`
* `<value>.<identifier>``attrget`
### Final (ordered by resolution order)
* `<typedef> <identifier>([<argdef>[, <argdef>]*]) <<code> | = <expr>>``funcdef` (function definition)
* `<exprkeyword> [expr]``keywordexpr` (keyword expression)
* `<typedef> <identifier> [= <value>]` — `vardef` (variable definition)
* `<identifier> = <value>` — `assignment`
* `<identifier>[, <identifier>]* = <value>` — `unpackassignment`
* `<typedef> <identifier> [= <expr>]` — `vardef` (variable definition)
* `<identifier> = <expr>` — `assignment`
* `<identifier>[, <identifier>]* = <expr>` — `unpackassignment`
* `<value>([<callargs> | <callkwargs> | <callargs>, <callkwargs>])``funccall` (function call)
* `<expr>` — expr evaluation (only in REPL)
* `if (<expr>) <block>``conditional`
@ -100,12 +101,22 @@ _Note: `*` after syntax unit means any number of them._
### Reserved keywords
* `def`
* `try`
* `catch`
* `except`
* `finally`
* `raise`
* `with`
* `yield`
* `include`
* `using`
* `default`
## Identifiers
Non-empty sequence of alphanumeric characters plus underscore («_»), not starting with a digit character.
Regex: `[_\w][_\w\d]*`
Regex: `[^\W\d][\w]*`
### Data types
@ -113,10 +124,14 @@ Regex: `[_\w][_\w\d]*`
* `u8`, `u16`, `u32`, `u64`, `u128` — fixed size unsigned integer
* `f8`, `f16`, `f32`, `f64`, `f128` — fixed size IEEE-754 floating point number
* `uf8`, `uf16`, `uf32`, `uf64`, `uf128` — fixed size unsigned floating point number
* `c8`, `c16`, `c32`, `c64`, `c128` — fixed size complex number
* `uc8`, `uc16`, `uc32`, `uc64`, `uc128` — fixed size unsigned complex number
* `int` — unsized («big») integer
* `uint` — unsized unsigned integer
* `float` — unsized floating point number
* `ufloat` — unsized unsigned floating point
* `complex` — unsized complex number
* `ucomplex` — unsized unsigned complex number
* `bool` — logical (boolean) value
* `byte` — single byte
* `char` — UTF-8 character
@ -171,5 +186,9 @@ A set of pre-defined keyword operators:
All character class checks are performed in current locale.
<br>
---
_by Sdore, 2020_
_by Sdore, 2021-22_<br>
_slang.sdore.me_

View File

@ -5,41 +5,45 @@ from .ast import *
from .compilers import *
from utils.nolog import *; logstart('Slang')
def compile(src, filename='<string>', *, compiler):
def compile(src, filename='<string>', *, compiler, optimize=0):
try:
#print(f"Source: {{\n{S(src).indent()}\n}}\n")
tl = parse_string(src)
#print(f"Tokens:\n{pformat(tl)}\n")
ast = build_ast(tl, filename)
#print(f"Code: {ast.code}\n")
ast = build_ast(tl, filename.join('""'))
print(f"Code: {ast.code}\n")
#print(f"Nodes: {pformat(list(walk_ast_nodes(ast)))}\n")
optimize_ast(ast, validate_ast(ast))
#print(f"Optimized: {ast.code}\n")
if (optimize):
optimize_ast(ast, validate_ast(ast), optimize)
print(f"Optimized: {ast.code}\n")
#print(f"Optimized Nodes: {pformat(list(walk_ast_nodes(ast)))}\n")
code = compiler.compile_ast(ast, validate_ast(ast), filename=filename)
ns = validate_ast(ast)
code = compiler.compile_ast(ast, ns, filename=filename)
#print("Compiled.\n")
except (SlSyntaxError, SlValidationError, SlCompilationError) as ex:
if (not ex.line): ex.line = src.split('\n')[ex.lineno-1]
sys.exit(ex)
except (SlSyntaxException, SlNodeException) as ex:
if (not ex.srclines): ex.srclines = src.split('\n')
sys.exit(str(ex))
return code
@apmain
@aparg('file', metavar='<file>', type=argparse.FileType('r'))
@aparg('-o', dest='output')
@aparg('-f', dest='compiler', required=True)
@aparg('file', metavar='<file.sl>', type=argparse.FileType('r'))
@aparg('-o', metavar='output', dest='output')
@aparg('-f', metavar='compiler', dest='compiler', default='pyssembly')#required=True)
@aparg('-O', metavar='level', help='Code optimization level', type=int, default=DEFAULT_OLEVEL)
def main(cargs):
if (cargs.output is None and not cargs.file.name.rpartition('.')[0]):
argparser.add_argument('-o', dest='output', required=True)
cargs = argparser.parse_args()
src = cargs.file.read()
filename = cargs.file.name
_cns = importlib.import_module('.compilers.'+cargs.compiler, package=__package__).__dict__.values()
compiler = first(i for i in allsubclasses(Compiler) if i in _cns)
code = compile(src, filename=filename.join('""'), compiler=compiler)
compiler = importlib.import_module('.compilers.'+cargs.compiler, package=__package__).__dict__['compiler']
code = compile(src, filename=filename, compiler=compiler, optimize=cargs.O)
open(cargs.output or cargs.file.name.rpartition('.')[0]+compiler.ext, 'wb').write(code)
if (__name__ == '__main__'): exit(main())

14
TODO.md
View File

@ -3,12 +3,16 @@
- https://esolangs.org/wiki/Stlang
- https://esolangs.org/wiki/Object_disoriented
- https://esolangs.org/wiki/Funciton ?
- `O.each { code }` (i.e. `O.each({code})`)
- Proposal: `if`, `for` and `while` without brackets
- Proposal: `if`, `for` and `while` without `()`
- OOP
- Increments
- FIXME: whitespace in srclength
- renew all `__repr__`s *AND* `__str__`s
- Proposal: `f(kw: arg)` = `f(kw=arg)`
- Proposal: [https://docs.microsoft.com/ru-ru/dotnet/csharp/language-reference/operators/default]
- FIXME: `auto' rettype
- FIXME: https://docs.python.org/3/reference/expressions.html#the-power-operator
- Proposal: `lazy` modifier and `lazy: expr` -- expr that evaluates/runs on first request after declaration
- time literal
- Proposal: arbitrary size numeric data types
- Proposal: single argument lambda without `()`
- https://lucidchart.com/techblog/2015/08/31/the-worst-mistake-of-computer-science
- ${lang} as logo ?
- Inheritance as in Ruby: `class A < B { }`

BIN
asm/hw

Binary file not shown.

View File

@ -1,16 +0,0 @@
bits 64
;section .rodata
hw: db "hw", 0
.text
global _start
extern puts
_start:
push rbp
mov rbp, rsp
lea rdi, [hw+rip]
call puts
mov rax, 0
pop rbp
ret

View File

@ -1 +0,0 @@
main() {puts("hw");}

BIN
asm/hw.o

Binary file not shown.

1122
ast.py

File diff suppressed because it is too large Load Diff

View File

@ -1,40 +1,24 @@
#!/usr/bin/python3
# Slang compilers
from ..ast import SlSyntaxError, SlValidationError
from ..ast import Slots, lstripcount, SlSyntaxError, SlNodeException, SlValidationError
import abc, traceback
class Compiler(abc.ABC):
ext = ''
@abc.abstractclassmethod
def compile_ast(cls, ast):
pass
def lstripcount(s, chars): # TODO: commonize
for ii, i in enumerate(s):
if (i not in chars): break
else: ii = 0
return (ii, s[ii:])
class SlCompilationError(SlNodeException):
desc: ...
class SlCompilationError(Exception):
__slots__ = ('desc', 'node', 'line', 'scope')
def __init__(self, desc, *args, **kwargs):
super().__init__(*args, **kwargs)
self.desc = desc
def __init__(self, desc, node, line='', *, scope=None):
self.desc, self.node, self.line, self.scope = desc, node, line, scope
def __exline__(self):
return f"Compilation error: {self.desc}"
def __str__(self):
l, line = lstripcount(self.line.partition('\n')[0].replace('\t', ' '), ' \t')
offset = (self.node.offset-l) if (self.node.offset != -1) else len(line)
return (f'\033[2m(in {self.scope})\033[0m ' if (self.scope is not None) else '') + f"Compilation error: {self.desc}{self.at}"+(':\n' + \
' \033[1m'+line[:offset]+'\033[91m'+line[offset:]+'\033[0m\n' + \
' '+' '*offset+'\033[95m^'+'~'*(self.node.length-1)+'\033[0m' if (line) else '') + \
(f"\n\n\033[1;95mCaused by:\033[0m\n{self.__cause__ if (isinstance(self.__cause__, (SlSyntaxError, SlValidationError, SlCompilationError))) else ' '+str().join(traceback.format_exception(type(self.__cause__), self.__cause__, self.__cause__.__traceback__))}" if (self.__cause__ is not None) else '')
@property
def at(self):
return f" at line {self.node.lineno}, offset {self.node.offset}"
@property
def lineno(self):
return self.node.lineno
# by Sdore, 2019
# by Sdore, 2021

View File

@ -3,7 +3,7 @@
# https://esolangs.org/wiki/Gibberish_(programming_language)
from .. import *
from Slang.ast import *
from ...ast import *
from utils import *
class Instrs:
@ -132,4 +132,6 @@ class GibberishCompiler(Compiler):
dlog("Code:\n"+code.decode())
return code
# by Sdore, 2019
compiler = GibberishCompiler
# by Sdore, 2020

View File

@ -0,0 +1,13 @@
#!/usr/bin/python3
# Slang Native (ASM) compiler target
from .. import *
from ...ast import *
from utils import *
class NativeCompiler(Compiler): pass
try: compiler = importlib.import_module(f".{platform.machine()}", __package__).compiler
except ModuleNotFoundError: compiler = NativeCompiler
# by Sdore, 2021

235
compilers/native/x86_64.py Normal file
View File

@ -0,0 +1,235 @@
#!/usr/bin/python3
# Slang Native (ASM) x86_64 compiler target
from . import *
word_size = 8
registers = ('rax', 'rcx', 'rdx', 'rbx', 'rsi', 'rdi', 'r8', 'r9', 'r10', 'r11', 'r12', 'r13', 'r14', 'r15')
regs64 = {i: () for i in registers}
regs32 = {'e'+i: ('r'+i, *regs64['r'+i]) for i in ('ax', 'cx', 'dx', 'bx', 'si', 'di')}
regs16 = {i: ('e'+i, *regs32['e'+i]) for i in ('ax', 'cx', 'dx', 'bx', 'si', 'di')}
regs8 = {i+j: (i+'x', *regs16[i+'x']) for i in 'acdb' for j in 'lh'}
class Instrs:
binopmap = {
'+': 'add',
}
stacksize = 0x10 # TODO
@init_defaults
def __init__(self, *, name, ns, filename, nargs: int, freeregs: set):
self.name, self.ns, self.filename, self.nargs, self.freeregs = name, ns, filename, nargs, freeregs
self.instrs = list()
self.varoff = bidict.OrderedBidict()
self.varsize = dict()
self.regpool = list(registers)
self.clobbered = set()
def compile(self):
saveregs = tuple(sorted(self.clobbered - self.freeregs))
instrs = [
"push rbp",
"mov rbp, rsp",
*(f"push {i}" for i in saveregs),
f"sub rsp, {self.stacksize}",
'\n',
*self.instrs,
'\n',
*(f"pop {i}" for i in saveregs[::-1]),
"leave",
f"ret {self.nargs}" if (self.nargs > 0) else "ret",
]
return f"global {self.name}\n{self.name}:\n\t"+'\n\t'.join('\t'.join(i.split(maxsplit=1)) for i in instrs)+'\n'
def var(self, name):
if (self.varoff[name] == 0): return '[rbp]'
return "[rbp%+d]" % -self.varoff[name]
@contextlib.contextmanager
def vars(self, *names):
yield tuple(map(self.var, names))
@contextlib.contextmanager
def regs(self, n):
regs = {self.regpool.pop(0) for _ in range(n)}
self.clobbered |= regs
try: yield regs
finally: self.regpool = [i for i in registers if i in {*self.regpool, *regs}]
#def syscall(self, rax, rdi=None, rsi=None, rdx=None, r10=None, r8=None, r9=None):
# self.instrs.append("
@dispatch
def add(self, x: ASTRootNode):
self.add(x.code)
@dispatch
def add(self, x: ASTCodeNode):
for i in x.nodes:
self.add(i)
@dispatch
def add(self, x: ASTVardefNode):
name = x.name.identifier
try:
ln, lo = last(self.varoff.items())
self.varoff[name] = lo+self.varsize[ln]
except StopIteration: self.varoff[name] = word_size
self.varsize[name] = self.sizeof(x.type)
if (x.value is not None): self.set(x.name, x.value)
@dispatch
def add(self, x: ASTFunccallNode):
callarguments = CallArguments.build(x, self.ns)
fsig = Signature.build(x.callable, self.ns)
fcall = fsig.compatible_call(callarguments, self.ns)
if (fcall is None): raise TODO(fcall)
n = int()
if (fcall[0] is not None): fname = f"{fsig.name}({CallArguments(args=fcall[0], ns=self.ns)})"
else:
if (isinstance(x.callable.value, ASTAttrgetNode)):
ofsig = Signature.build(x.callable.value.value, self.ns)
if (isinstance(fsig, stdlib.Builtin)):
fname = x.callable
else: raise NotImplementedError(ofsig)
else: raise NotImplementedError(x.callable.value)
for i in x.callargs.callargs:
self.push(i)
n += 1
if (x.callargs.starargs or x.callkwargs.callkwargs or x.callkwargs.starkwargs): raise NotImplementedError()
self.instrs.append(f"call {fname}")
@dispatch
def load(self, x: ASTValueNode, *reg):
return self.load(x.value, *reg)
@dispatch
def load(self, x: ASTLiteralNode):
return x.literal
@dispatch
def load(self, x):
with self.regs(1) as (reg,):
self.load(x, reg)
return reg
#reg = self.regpool.pop(0)
#self.clobbered.add(reg)
#self.load(x, reg)
#return reg
@dispatch
def load(self, x: ASTIdentifierNode, reg):
self.instrs.append(f"mov {reg}, {self.var(x.identifier)}")
@dispatch
def load(self, x: ASTBinaryExprNode, reg):
self.load(x.lvalue, reg)
self.instrs.append(f"{self.binopmap[x.operator.operator]} {reg}, {self.load(x.rvalue)}")
@dispatch
def load(self, x: str, reg):
self.instrs.append(f"mov {reg}, {x}")
@dispatch
def set(self, name: ASTIdentifierNode, value):
self.set(name.identifier, value)
@dispatch
def set(self, name: str, value: ASTValueNode):
self.set(name, value.value)
@dispatch
def set(self, name: str, value: ASTLiteralNode):
self.set(name, self.load(value))
@dispatch
def set(self, name: str, value):
with self.vars(name) as (var,):
self.instrs.append(f"mov {var}, {self.load(value)}")
def push(self, x):
self.instrs.append(f"push {self.load(x)}")
@dispatch
def sizeof(self, x: ASTTypedefNode):
return self.sizeof(Signature.build(x, self.ns))
@dispatch
def sizeof(self, x: lambda x: hasattr(x, 'fmt')):
return struct.calcsize(x.fmt)
class BuiltinInstrs(Instrs):
@init_defaults
def __init__(self, *, freeregs: set):
self.freeregs = freeregs
@singleton
class builtin_stdio_println(BuiltinInstrs):
name = 'stdio.println'
instrs = [
"mov al, [rbp+0x10]",
"add al, '0'",
"mov [rbp-2], al",
"mov al, 10",
"mov [rbp-1], al",
"mov rax, 1 ; sys_write",
"mov rdi, 1 ; stdout",
"lea rsi, [rbp-2] ; buf",
"mov rdx, 2 ; count",
"syscall",
]
nargs = 1
stacksize = 2
clobbered = {'rax', 'rcx', 'rdx', 'rdi', 'rsi', 'r11'}
class x86_64Compiler(NativeCompiler):
ext = '.o'
header = """
section .text
global _start
_start:
call main
mov rax, 60 ; sys_exit
mov rdi, 0 ; error_code
syscall
hlt
"""
@classmethod
def compile_ast(cls, ast, ns, *, filename):
instrs = Instrs(name='main', ns=ns, filename=filename)
instrs.add(ast)
src = f"# {filename}\n{S(cls.header).unindent().strip()}\n"
src += '\n'+builtin_stdio_println.compile()
src += '\n'+instrs.compile()
sname = os.path.splitext(filename)[0]+'.s'
sfd = open(sname, 'w')
try:
sfd.write(src)
sfd.close()
log("Src:\n"+src)
ofd, oname = tempfile.mkstemp(prefix=f"slc-{os.path.splitext(filename)[0]}-", suffix='.o')
try:
subprocess.run(('yasm', '-felf64', '-o', oname, sname), stderr=subprocess.PIPE, text=True, check=True)
return os.fdopen(ofd, 'rb').read()
except subprocess.CalledProcessError as ex: raise SlCompilationError('\n'+S(ex.stderr).indent(), ast, scope=instrs.ns) from ex
finally:
try: os.remove(oname)
except OSError: pass
finally:
sfd.close()
try: os.remove(sname)
except OSError: pass
compiler = x86_64Compiler
# by Sdore, 2021

1
compilers/pyssembly/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
stdlib/

View File

@ -38,7 +38,7 @@ class Instrs:
def _class_init(self, *args, **kwargs):
getattr(self, '<init>')()
getattr(self, f"<constructor ({', '.join(type(i).__name__ for i in (*args, *kwargs.values()))})>")(*args, **kwargs)
_class_init = _class_init.__code__.replace(co_name='<new>')
_class_init = code_with(_class_init.__code__, co_name='<new>')
@init_defaults
def __init__(self, *, name, ns, filename, argdefs=(), lastnodens: lambda: [None, None], firstlineno=0):
@ -52,7 +52,7 @@ class Instrs:
self.cellvars = self.argnames.copy()
self.srclnotab = list()
self.lastln = self.firstlineno
self._class_init = self._class_init.replace(co_filename=self.filename, co_consts=tuple(i.replace(co_filename=self.filename) if (isinstance(i, CodeType)) else i for i in self._class_init.co_consts))
self._class_init = code_with(self._class_init, co_filename=self.filename, co_consts=tuple(code_with(i, co_filename=self.filename) if (isinstance(i, CodeType)) else i for i in self._class_init.co_consts))
def compile(self):
return pyssembly.Code('\n'.join(self.instrs), name=self.name, filename=self.filename.strip('"'), srclnotab=self.srclnotab, firstlineno=self.firstlineno, consts=self.consts, argnames=self.argnames)
@ -146,7 +146,7 @@ class Instrs:
@dispatch
def add(self, x: ASTFuncdefNode):
code_ns = self.ns.derive(x.name.identifier)
name = f"{x.name.identifier}({CallArguments(args=tuple(Signature.build(i, code_ns) for i in x.argdefs))})"
name = f"{x.name.identifier}({CallArguments(args=x.argdefs, ns=code_ns)})"
self.lastnodens[1] = code_ns
fname = f"{self.name}.<{x.__fsig__()}>"
f_instrs = Instrs(name=fname, ns=code_ns, filename=self.filename, argdefs=x.argdefs, lastnodens=self.lastnodens, firstlineno=x.lineno)
@ -214,8 +214,9 @@ class Instrs:
src = open(filename, 'r').read()
tl = parse_string(src)
ast = build_ast(tl, filename)
optimize_ast(ast, validate_ast(ast))
instrs = Instrs(name='<module>', ns=validate_ast(ast), filename=filename)
if (x.flags.optimized): optimize_ast(ast, validate_ast(ast))
ns = validate_ast(ast)
instrs = Instrs(name=filename, ns=ns, filename=filename)
instrs.add(ast)
code = instrs.compile().to_code()
# TODO
@ -359,36 +360,54 @@ class Instrs:
@dispatch
def load(self, x: ASTFunccallNode):
if (isinstance(x.callable, ASTValueNode) and isinstance(x.callable.value, ASTIdentifierNode) and x.callable.value.identifier in self.ns.signatures and not isinstance(self.ns.signatures[x.callable.value.identifier], Class)):
self.load(f"{x.callable.value.identifier}({CallArguments.build(x, self.ns)})")
else: self.load(x.callable)
#if (isinstance(x.callable, ASTValueNode) and isinstance(x.callable.value, ASTIdentifierNode) and x.callable.value.identifier in self.ns.signatures and not isinstance(self.ns.signatures[x.callable.value.identifier], Class)):
callarguments = CallArguments.build(x, self.ns)
fsig = Signature.build(x.callable, self.ns)
fcall = fsig.compatible_call(callarguments, self.ns)
if (fcall is None): raise TODO(fcall)
n = int()
if (fcall[0] is not None): self.load(f"{fsig.name}({CallArguments(args=fcall[0], ns=self.ns)})")
else:
if (isinstance(x.callable.value, ASTAttrgetNode)):
ofsig = Signature.build(x.callable.value.value, self.ns)
if (type(ofsig) is Function):
ofcall = ofsig.compatible_call(callarguments, self.ns)
assert (ofcall is not None)
self.load(fsig.name)
self.load(f"{ofsig.name}({CallArguments(args=ofcall[0], ns=self.ns)})")
n += 1
elif (type(ofsig) is stdlib.list):
f = ofsig.attrops[x.callable.value.optype.special, x.callable.value.attr.identifier]
self.load(f.name)
self.load(x.callable.value.value)
n += 1
elif (isinstance(fsig, stdlib.Builtin)):
self.load(x.callable)
else: raise NotImplementedError(ofsig)
else: raise NotImplementedError(x.callable.value)
for i in x.callargs.callargs:
self.load(i)
n += 1
if (x.callargs.starargs):
if (n):
self.instrs.append(f"BUILD_TUPLE {n}")
n = 1
if (n or len(x.callargs.starargs) > 1): self.instrs.append(f"BUILD_LIST {n}")
for i in x.callargs.starargs:
self.load(i)
n += 1
self.instrs.append(f"BUILD_TUPLE_UNPACK_WITH_CALL {n}")
if (n or len(x.callargs.starargs) > 1): self.instrs.append("LIST_EXTEND 1")
if (n or len(x.callargs.starargs) > 1): self.instrs.append("LIST_TO_TUPLE")
n = 0
elif (x.callkwargs.starkwargs):
self.instrs.append("BUILD_TUPLE {n}")
n = 0
for i in x.callkwargs.callkwargs:
self.load(f"'{i[0]}'")
self.load(i[1])
n += 1
if (n and (x.callargs.starargs or x.callkwargs.starkwargs)):
self.instrs.append(f"BUILD_MAP {n}")
n = 1
if (x.callkwargs.starkwargs):
self.instrs.append(f"BUILD_MAP {n}")
for i in x.callkwargs.starkwargs:
self.load(i)
n += 1
self.instrs.append(f"BUILD_MAP_UNPACK_WITH_CALL {n}")
self.instrs.append("DICT_MERGE 1")
n = 1
self.instrs.append(f"CALL{'EX' if (x.callargs.starargs or x.callkwargs.starkwargs) else 'KW' if (x.callkwargs.callkwargs) else ''} {n}")
@ -453,18 +472,18 @@ class PyssemblyCompiler(Compiler):
@classmethod
def compile_ast(cls, ast, ns, *, filename):
instrs = Instrs(name='<module>', ns=ns, filename=filename)
instrs = Instrs(name=filename, ns=ns, filename=filename)
instrs.consts.append(compile(open(std.__file__).read(), '<std>', 'exec'))
instrs.consts.append(code_with(compile(open(std.__file__).read(), '<stdlib>', 'exec'), name='stdlib'))
instrs.instrs += [
f"LOAD {len(instrs.consts)-1}",
"LOAD ('<std>')",
"LOAD ('<stdlib>')",
"MKFUNC 0", # TODO?
"CALL",
]
try: instrs.add(ast)
except Exception as ex: raise SlCompilationError('Compilation error', instrs.lastnodens[0], scope=instrs.lastnodens[1].scope) from ex
except Exception as ex: raise SlCompilationError('Compilation error', instrs.lastnodens[0], scope=instrs.lastnodens[1].scope if (instrs.lastnodens[1] is not None) else '<UNKNOWN>') from ex
#dlog("Instrs:\n"+'\n'.join(instrs.instrs)+'\n')
@ -483,4 +502,6 @@ class PyssemblyCompiler(Compiler):
return code
compiler = PyssemblyCompiler
# by Sdore, 2020

181
compilers/pyssembly/genstdlib.py Executable file
View File

@ -0,0 +1,181 @@
#!/usr/bin/python3
from utils.nolog import *
builtin_types = {i for i in itertools.chain(builtins.__dict__.values(), types.__dict__.values()) if isinstance(i, type)}
@dispatch
def gen(x, name: NoneType, *, scope: dict):
return gen(x, '', scope=scope).rstrip(' ')
@dispatch
def gen(x: lambda x: x is NoneType, name: str, *, scope: dict):
return f"void {name}"
@dispatch
def gen(x: lambda x: isinstance(x, type) and (getattr(x, '__module__', None) == '__main__' or x in builtin_types), name: str, *, scope: dict):
return f"{x.__name__} {name}"
@dispatch
def gen(x: lambda x: typing_inspect.get_origin(x) is not None, name: str, *, scope: dict):
o = typing_inspect.get_origin(x)
if (o is typing.Literal):
return gen(type(x.__args__[0]), name, scope=scope)
elif (o is typing.Union):
if (x.__args__[1] is NoneType): return f"{gen(x.__args__[0], name, scope=scope).rstrip()}?"
else: return f"{'|'.join(gen(i, None, scope=scope) for i in x.__args__)} {name}"
elif (o in (tuple, list, dict, set, frozenset)):
return f"{o.__name__}{', '.join(gen(i, None, scope=scope) for i in x.__args__ if i is not ...).join('[]') if (x.__args__ and x.__args__ != (typing.Any,) and x.__args__ != (typing.Any, ...)) else ''} {name}" # TODO args
# TODO FIXME:
elif (o is collections.abc.Iterable):
return f"iterable {name}"
elif (o is collections.abc.Iterator):
return f"iterator {name}"
elif (o is collections.abc.Sequence):
return f"sequence {name}"
elif (o is collections.abc.Mapping):
return f"mapping[{gen(x.__args__[0], None, scope=scope)}, {gen(x.__args__[1], None, scope=scope)}] {name}"
elif (o is collections.abc.ItemsView):
return f"items {name}"
elif (o is collections.abc.KeysView):
return f"keys {name}"
elif (o is collections.abc.ValuesView):
return f"values {name}"
elif (o is collections.abc.Set):
return f"set {name}"
elif (o is collections.abc.Callable):
return f"callable {name}"
elif (o is type):
return f"{x.__args__[0].__name__} {name}"
else: raise NotImplementedError(x, o)
@dispatch
def gen(x: lambda x: x is typing.Any, name: str, *, scope: dict): # TODO FIXME
return f"object {name}"
@dispatch
def gen(x: typing.TypeVar, name: str, *, scope: dict):
return f"{x.__name__} {name}"
@dispatch
def gen(x: lambda x: isinstance(x, type) and issubclass(x, typing.Protocol), name: str, *, scope: dict):
if (x is typing.SupportsInt):
t = int
elif (x is typing.SupportsFloat):
t = float
else: raise NotImplementedError(x)
return gen(t, name, scope=scope)
@dispatch
def gen(x: function, name: str, *, scope: dict):
fsig = inspect.signature(x)
return f"{gen(typing.get_type_hints(x, scope)['return'], None, scope=scope)} {name}({', '.join(gen(v, x, k, scope=scope) for ii, (k, v) in enumerate(fsig.parameters.items()) if not (ii == 0 and k == 'self'))})"
@dispatch
def gen(x: inspect.Parameter, f, name: str, *, scope: dict):
t = gen(typing.get_type_hints(f, scope)[x.name], name, scope=scope)
if (x.default is not inspect._empty):
if (x.default is ...):
if (t[-1:] != '?'): t += '?'
else: raise NotImplementedError(x.default)
return t
@dispatch
def gen(x: property, name: str, *, scope: dict):
return gen(typing.get_type_hints(x.fget, scope)['return'], name, scope=scope)
@dispatch
def gen(x: lambda x: isinstance(x, type) and x.__module__ == '__main__', name: str, *, scope: dict):
return f"{x.__name__} {name}"
#@dispatch
#def gen(x: type, name: str, *, scope: dict):
# return f"{type.__name__} {name}"
@dispatch
def gen(x: lambda x: isinstance(x, type) and x.__module__ == '__main__', *, scope: dict):
r = []
for k, v in typing.get_type_hints(x, scope).items():
r.append(f"{gen(v, k, scope=scope)};".lstrip(';'))
for k, v in x.__dict__.items():
if (k in ('__module__', '__doc__')): continue
if (not isinstance(v, property) and (not isinstance(v, function) or getattr(v, '__module__', None) != '__main__')): continue
r.append(f"{gen(v, k, scope=scope)};".lstrip(';'))
if (not r): return ''
return f"class {x.__name__} {{\n\t"+'\n\t'.join(r)+'\n}'
@dispatch
def gen(x: CodeType, *, package):
r = {'__name__': '__main__', '__package__': package}
exec(x, r)
return '\n\n'.join(Slist(gen(i, scope=r) for i in r.values() if isinstance(i, type) and i.__module__ == '__main__').strip(''))
class AnnotationsFileLoader(importlib.machinery.SourceFileLoader):
header = "from __future__ import annotations"
def get_data(self, path):
dlog(path)
data = super().get_data(path)
if (not path.endswith('.pyi')): return data
return (self.header+'\n\n').encode() + data
def path_hook_factory(f):
def path_hook(path):
finder = f(path)
finder._loaders.insert(0, ('.pyi', AnnotationsFileLoader))
return path_hook
@apmain
@aparg('typeshed', metavar='<typeshed repo path>')
@aparg('output', metavar='<stdlib .sld output dir>')
def main(cargs):
if (sys.version_info < (3,8)): raise NotImplementedError("Currently only Python 3.8+ is supported.")
dirs = ('stdlib', 'stubs')
import __future__
sys.dont_write_bytecode = True
sys.path = [__future__.__file__] + [os.path.join(cargs.typeshed, i) for i in dirs] + [os.path.join(cargs.typeshed, 'stubs', i, j) for i in os.listdir(os.path.join(cargs.typeshed, 'stubs')) for j in os.listdir(os.path.join(cargs.typeshed, 'stubs', i))]
#sys.path_hooks[-1] = path_hook_factory(sys.path_hooks[-1])
#sys.path_hooks[-1] = importlib.machinery.FileFinder.path_hook((AnnotationsFileLoader, ['.pyi']+[i for i in importlib.machinery.all_suffixes() if i != '.pyc']))
sys.path_hooks = [importlib.machinery.FileFinder.path_hook((AnnotationsFileLoader, ['.pyi']))]
sys.meta_path = [sys.meta_path[2]]
skipped = int()
for d in dirs:
for v in sorted(os.listdir(os.path.join(cargs.typeshed, d))):
if (v in ('@python2', '_typeshed')): continue
for p, _, n in os.walk(os.path.join(cargs.typeshed, d, v)):
if ('@python2' in p): continue
o = os.path.join(cargs.output, p.partition(os.path.join(cargs.typeshed, d, v, ''))[2])
os.makedirs(o, exist_ok=True)
for i in sorted(n, key=lambda x: 'builtins.pyi' not in x):
if (not i.endswith('.pyi')): continue
filename = os.path.join(p, i)
log(filename)
code = compile(b"from __future__ import annotations\n\n"+open(filename, 'rb').read(), filename, 'exec')
try: r = gen(code, package=os.path.splitext(filename)[0].replace('/', '.'))
except Exception as ex:
if ('builtins.pyi' in i): raise
logexception(ex)
skipped += 1
raise # XXX
else:
if (not r): continue
if ('builtins.pyi' in i): r = 'import sld:std:*;\n\n'+r
else: r = 'import py:builtins:*;\n\n'+r
open(os.path.join(o, os.path.splitext(i)[0]+'.sld'), 'w').write(r+'\n')
if (skipped): print(f"\033[93m{skipped} errors caught ({skipped} files skipped).\033[0m")
print("\033[1mSuccess!\033[0m")
if (__name__ == '__main__'): exit(main(nolog=True))
else: logimported()
# by Sdore, 2021

View File

@ -0,0 +1 @@
typing_inspect

View File

@ -1,2 +1,5 @@
class stdio:
println = print
def _map(f, l): return list(map(f, l))
def _each(l, f): return list(map(f, l))

@ -1 +1 @@
Subproject commit b44cd294c4f6cdb66fdd6c13aebabb10855e7dc2
Subproject commit e2fd852952ff807c60819f8840e8b23664cc7593

View File

@ -1,351 +0,0 @@
#!/usr/bin/python3
# Slang Bytecode (SBC) compiler target
from . import *
from Slang.ast import *
from utils import *
NOP = 0x00
END = 0x01
POP = 0x02
RET = 0x03
BLTIN = 0x04
CODE = 0x05
POS = 0x10
NEG = 0x11
NOT = 0x12
INV = 0x13
ATOI = 0x14
ITOA = 0x15
ITOF = 0x16
CEIL = 0x17
FLR = 0x18
RND = 0x19
CTOS = 0x1a
ADD = 0x20
SUB = 0x21
MUL = 0x22
DIV = 0x23
IDIV = 0x24
MOD = 0x25
POW = 0x26
LSH = 0x27
RSH = 0x28
AND = 0x29
OR = 0x2a
XOR = 0x2b
EQ = 0x30
NE = 0x31
LT = 0x32
GT = 0x33
LE = 0x34
GE = 0x35
IS = 0x36
ISNOT = 0x37
IF = 0x40
ELSE = 0x41
EXEC = 0x42
ALLOC = 0xa0
EXTEND = 0xa1
CONST = 0xa2
JUMPF = 0xa3
JUMPB = 0xa4
SCPGET = 0xa5
SCPSET = 0xa6
CALL = 0xa7
HASARG = 0xa0
def readVarInt(s):
r = int()
i = int()
while (True):
b = s.recv(1)[0]
r |= (b & (1 << 7)-1) << (7*i)
if (not b & (1 << 7)): break
i += 1
return r
def writeVarInt(v):
assert v >= 0
r = bytearray()
while (True):
c = v & (1 << 7)-1
v >>= 7
if (v): c |= (1 << 7)
r.append(c)
if (not v): break
return bytes(r)
class Instrs:
unops = '+-!~'
binops = (*'+-*/%', '**', '<<', '>>', '&', '|', '^')
unopmap = {
'+': POS,
'-': NEG,
'!': NOT,
'~': INV,
'not': NOT,
}
binopmap = {
'+': ADD,
'-': SUB,
'*': MUL,
'/': DIV,
'//': IDIV,
'%': MOD,
'**': POW,
'<<': LSH,
'>>': RSH,
'&': AND,
'|': OR,
'^': XOR,
'==': EQ,
'!=': NE,
'<': LT,
'>': GT,
'<=': LE,
'>=': GE,
'is': IS,
'is not': ISNOT,
}
@init_defaults
def __init__(self, *, name, ns, filename, scpcells: indexset):
self.name, self.ns, self.filename, self.scpcells = name, ns, filename, scpcells
self.instrs = bytearray()
def compile(self):
return bytes(self.instrs)
@dispatch
def add(self, opcode: lambda x: isinstance(x, int) and x < HASARG):
self.instrs.append(opcode)
@dispatch
def add(self, opcode: lambda x: isinstance(x, int) and x >= HASARG, oparg: int):
self.instrs.append(opcode)
self.instrs.append(oparg)
@dispatch
def add(self, x: ASTRootNode):
self.add(x.code)
@dispatch
def add(self, x: ASTCodeNode):
for i in x.nodes:
self.add(i)
@dispatch
def add(self, x: ASTValueNode):
self.add(x.value)
@dispatch
def add(self, x: ASTVardefNode):
if (x.value is not None):
self.load(x.value)
self.store(x.name)
@dispatch
def add(self, x: ASTAssignmentNode):
if (x.inplace_operator is not None): raise NotImplementedError()
self.load(x.value)
self.store(x.name)
@dispatch
def add(self, x: ASTFunccallNode):
self.load(x)
self.add(POP)
@dispatch
def add(self, x: ASTBlockNode):
self.add(x.code)
@dispatch
def add(self, x: ASTFuncdefNode):
code_ns = self.ns.derive(x.name.identifier)
name = f"{x.name.identifier}__{self.ns.signatures[x.name.identifier].call.index(CallArguments(args=tuple(Signature.build(i, code_ns) for i in x.argdefs)))}"
fname = f"{self.name}.<{x.__fsig__()}>"
self.scpcells[name]
f_instrs = Instrs(name=fname, ns=code_ns, filename=self.filename, scpcells=self.scpcells.copy())
for i in x.argdefs:
f_instrs.scpcells[i.name.identifier]
f_instrs.add(x.code)
self.add(CODE)
self.instrs += f_instrs.instrs
self.add(END)
self.store(name+'.len')
self.store(name)
@dispatch
def add(self, x: ASTKeywordExprNode):
if (x.keyword.keyword == 'import'):
ns, _, name = x.value.identifier.partition('::')
raise TODO
self.store(name)
elif (x.keyword.keyword == 'return'):
self.load(x.value)
self.add(RET)
else: raise NotImplementedError(x.keyword)
@dispatch
def add(self, x: ASTKeywordDefNode):
if (x.keyword.keyword == 'main'):
name = '<main>'
code_ns = self.ns.derive(name)
f_instrs = Instrs(name=name, ns=code_ns, filename=self.filename)
f_instrs.add(x.code)
self.add(CODE)
self.instrs += f_instrs.instrs
self.add(END)
self.add(EXEC)
self.add(POP)
else: raise NotImplementedError(x.keyword)
@dispatch
def add(self, x: ASTConditionalNode):
self.load(x.condition)
self.add(IF)
self.add(x.code)
self.add(END)
#@dispatch
def add_(self, x: ASTForLoopNode):
self.load(x.iterable)
#self.cellvars.append(x.name.identifier) # TODO FIXME
self.instrs += [
"SETUP_LOOP :end",
"ITER",
":for",
"FOR :else",
]
self.store(x.name)
self.add(x.code)
self.instrs += [
"JUMPA :for",
":else",
"POP_BLOCK",
":end",
]
#@dispatch
def add_(self, x: ASTWhileLoopNode):
self.instrs += [
"SETUP_LOOP :end",
":while",
]
self.load(x.condition)
self.instrs.append("JPOPF :else")
self.add(x.code)
self.instrs += [
"JUMPA :while",
":else",
"POP_BLOCK",
":end",
]
#@dispatch
def add_(self, x: ASTElseClauseNode):
assert (self.instrs[-1] == ":end")
popped = [self.instrs.pop()]
if (self.instrs[-1] == "POP_BLOCK"): popped.append(self.instrs.pop())
self.add(x.code)
self.instrs += reversed(popped)
@dispatch
def load(self, x: ASTLiteralNode):
sig = Signature.build(x, ns=self.ns)
if (hasattr(sig, 'fmt')):
v = struct.pack(sig.fmt, int(x.literal))
elif (isinstance(sig, stdlib.int)):
v = writeVarInt(int(x.literal))
elif (isinstance(sig, stdlib.str)):
v = x.literal.encode('utf8')
else: raise NotImplementedError(sig)
self.add(CONST, len(v))
self.instrs += v
@dispatch
def load(self, x: ASTIdentifierNode):
self.load(x.identifier)
@dispatch
def load(self, x: ASTValueNode):
self.load(x.value)
@dispatch
def load(self, x: ASTFunccallNode):
nargs = int()
for i in x.callargs.callargs:
self.load(i)
nargs += 1
if (x.callargs.starargs): raise NotImplementedError()
if (x.callkwargs.callkwargs): raise NotImplementedError()
if (x.callkwargs.starkwargs): raise NotImplementedError()
if (isinstance(x.callable, ASTValueNode) and isinstance(x.callable.value, ASTIdentifierNode) and x.callable.value.identifier in self.ns.signatures):
name = f"{x.callable.value.identifier}__{self.ns.signatures[x.callable.value.identifier].call.index(CallArguments.build(x, self.ns))}"
self.load(name)
self.load(name+'.len')
self.add(EXEC)
else: # builtin
#self.add(ITOA) # TODO FIXME cast
self.add(BLTIN)
self.instrs += x.callable.value.identifier.encode('ascii')+b'\0'
self.add(CALL, nargs)
#@dispatch
def load_(self, x: ASTAttrgetNode):
self.load(x.value)
assert x.optype.special == '.' # TODO
self.instrs.append(f"GETATTR ({x.attr})")
@dispatch
def load(self, x: ASTUnaryExprNode):
self.load(x.value)
self.add(self.unopmap[x.operator.operator])
@dispatch
def load(self, x: ASTBinaryExprNode):
self.load(x.lvalue)
self.load(x.rvalue)
if (x.operator.operator == 'to'): raise NotImplementedError()
else: self.add(self.binopmap[x.operator.operator])
#@dispatch
def load_(self, x: ASTItemgetNode):
self.load(x.value)
self.load(x.key)
self.instrs.append("SUBSCR")
@dispatch
def load(self, x: str):
self.add(SCPGET, self.scpcells[x])
@dispatch
def store(self, x: ASTIdentifierNode):
self.store(x.identifier)
@dispatch
def store(self, x: str):
self.add(SCPSET, self.scpcells[x])
class SBCCompiler(Compiler):
ext = '.sbc'
@classmethod
def compile_ast(cls, ast, ns, *, filename):
instrs = Instrs(name='<module>', ns=ns, filename=filename)
instrs.add(ast)
code = instrs.compile()
return code
# by Sdore, 2019

289
compilers/sbc/__init__.py Normal file
View File

@ -0,0 +1,289 @@
#!/usr/bin/python3
# Slang Bytecode (SBC) compiler target
from . import *
from .. import *
from ...ast import *
from utils import *
NOP = 0x00
POP = 0x01
DUP = 0x02
RET = 0x03
CODE = 0x04
IF = 0x05
LOOP = 0x06
ELSE = 0x07
END = 0x08
CALL = 0x09
ASGN = 0x0A
BLTIN = 0x0B
CONST = 0x0C
SGET = 0x0D
SSET = 0x0E
def readVarInt(s):
r = int()
i = int()
while (True):
b = s.recv(1)[0]
r |= (b & (1 << 7)-1) << (7*i)
if (not b & (1 << 7)): break
i += 1
return r
def writeVarInt(v):
assert v >= 0
r = bytearray()
while (True):
c = v & (1 << 7)-1
v >>= 7
if (v): c |= (1 << 7)
r.append(c)
if (not v): break
return bytes(r)
class Instrs:
unopmap = {
'!': 'not',
'+': 'abs',
'-': 'neg',
'~': 'inv',
'++': 'inc',
'--': 'dec',
'**': 'sqr',
'not': 'not',
}
binopmap = {
'+': 'add',
'-': 'sub',
'*': 'mul',
'/': 'div',
'//': 'idiv',
'%': 'mod',
'**': 'pow',
'<<': 'ls',
'>>': 'rs',
'&': 'and',
'^': 'xor',
'|': 'or',
'&&': 'and',
'^^': 'xor',
'||': 'or',
'==': 'eq',
'!=': 'ne',
'<': 'lt',
'>': 'gt',
'<=': 'le',
'>=': 'ge',
'is': 'is',
'is not': 'isnot',
'in': 'in',
'not in': 'notin',
'isof': 'isof',
'and': 'and',
'but': 'and',
'xor': 'xor',
'or': 'or',
'to': 'range',
}
@init_defaults
def __init__(self, *, name, ns, filename, scpcells: indexset):
self.name, self.ns, self.filename, self.scpcells = name, ns, filename, scpcells
self.instrs = bytearray()
self.opmap = dict()
def compile(self):
return bytes(self.instrs)
@dispatch
def add(self, opcode: int, *args: int):
self.instrs.append(opcode)
if (args): self.instrs += bytes(args)
@dispatch
def add(self, x: ASTRootNode):
self.add(x.code)
@dispatch
def add(self, x: ASTBlockNode):
self.add(x.code)
@dispatch
def add(self, x: ASTCodeNode):
for i in x.nodes:
self.add(i)
@dispatch
def add(self, x: ASTValueNode):
self.add(x.value)
@dispatch
def add(self, x: ASTVardefNode):
if (x.value is not None):
self.load(x.value, sig=Signature.build(x.type, ns=self.ns))
self.store(x.name)
@dispatch
def add(self, x: ASTFunccallNode):
self.load(x)
self.add(POP)
@dispatch
def add(self, x: ASTKeywordDefNode):
if (x.keyword.keyword == 'main'):
name = '<main>'
code_ns = self.ns.derive(name)
f_instrs = Instrs(name=name, ns=code_ns, filename=self.filename)
f_instrs.add(x.code)
self.add(CODE)
self.instrs += f_instrs.instrs
self.add(END)
self.add(CALL, 0)
self.add(POP)
else: raise NotImplementedError(x.keyword)
@dispatch
def add(self, x: ASTConditionalNode):
self.load(x.condition)
self.add(IF)
self.add(x.code)
self.add(END)
@dispatch
def add(self, x: ASTForLoopNode):
self.load(x.iterable)
self.builtin('iter', 1)
self.store(x.name)
self.add(DUP, 0)
self.add(LOOP)
self.add(x.code)
self.builtin('iter', 1)
self.store(x.name)
self.add(DUP, 0)
self.add(END)
@dispatch
def add(self, x: ASTWhileLoopNode):
self.load(x.condition)
self.add(LOOP)
self.add(x.code)
self.load(x.condition)
self.add(END)
@dispatch
def add(self, x: ASTElseClauseNode):
assert (self.instrs[-1] == END)
end = self.instrs.pop()
self.add(ELSE)
self.add(x.code)
self.add(end)
@dispatch
def load(self, x: ASTLiteralNode, *, sig=None):
if (sig is None): sig = Signature.build(x, ns=self.ns)
if (hasattr(sig, 'fmt')):
t, v = sig.__class__.__name__, struct.pack(sig.fmt, int(x.literal))
elif (isinstance(sig, stdlib.int)):
t, v = 'i', writeVarInt(eval_literal(x))
elif (isinstance(sig, stdlib.str)):
t, v = 's', eval_literal(x).encode('utf-8')
elif (isinstance(sig, stdlib.char)):
t, v = 'c', eval_literal(x).encode('utf-8') # TODO
else: raise NotImplementedError(sig)
self.add(CONST)
self.instrs += t.encode('utf-8')+b'\0' + writeVarInt(len(v)) + v
@dispatch
def load(self, x: ASTIdentifierNode, **kwargs):
self.load(x.identifier, **kwargs)
@dispatch
def load(self, x: ASTValueNode, **kwargs):
self.load(x.value, **kwargs)
@dispatch
def load(self, x: ASTFunccallNode):
nargs = int()
for i in x.callargs.callargs:
self.load(i)
nargs += 1
if (x.callargs.starargs): raise NotImplementedError()
if (x.callkwargs.callkwargs): raise NotImplementedError()
if (x.callkwargs.starkwargs): raise NotImplementedError()
if (isinstance(x.callable, ASTValueNode) and isinstance(x.callable.value, ASTIdentifierNode) and x.callable.value.identifier in self.ns.signatures):
raise NotImplementedError()
#name = f"{x.callable.value.identifier}({CallArguments.build(x, self.ns)})"
#self.load(name)
#self.load(name+'.len')
#self.add(EXEC)
else:
self.load(x.callable)
self.add(CALL, nargs)
@dispatch
def load(self, x: ASTAttrgetNode):
if (str(x.value) != 'stdio'): raise NotImplementedError(x)
if (x.optype.special != '.'): raise NotImplementedError(x)
self.builtin(str(x.attr))
@dispatch
def load(self, x: ASTBinaryExprNode, **kwargs):
self.load(x.lvalue, **kwargs)
self.load(x.rvalue, **kwargs)
self.builtin(self.binopmap[x.operator.operator], 2)
@dispatch
def load(self, x: str, **kwargs):
self.add(SGET, self.scpcells[x])
@dispatch
def store(self, x: ASTIdentifierNode):
self.store(x.identifier)
@dispatch
def store(self, x: str):
self.add(SSET, self.scpcells[x])
@dispatch
def assign(self, builtin: str,
nargs_code: lambda nargs_code: isinstance(nargs_code, int) and 0 <= nargs_code < 4,
nargs_stack: lambda nargs_stack: isinstance(nargs_stack, int) and 0 <= nargs_stack < 64,
opcode: int = None):
self.builtin(builtin)
if (opcode is None): opcode = first(sorted(set(range(0x10, 0x100)) - set(self.opmap.values())))
self.add(ASGN, opcode, (nargs_code << 6) | nargs_stack)
self.opmap[builtin, nargs_code, nargs_stack] = opcode
@dispatch
def builtin(self, builtin: str):
self.add(BLTIN)
self.instrs += builtin.encode('ascii')+b'\0'
@dispatch
def builtin(self, builtin: str, nargs: int):
if (nargs is not None and len(self.opmap) < 0xf0): self.assign(builtin, 0, nargs)
if ((builtin, 0, nargs) in self.opmap): self.add(self.opmap[builtin, 0, nargs])
else: self.builtin(builtin); self.add(CALL, nargs)
class SBCCompiler(Compiler):
ext = '.sbc'
@classmethod
def compile_ast(cls, ast, ns, *, filename):
instrs = Instrs(name='<module>', ns=ns, filename=filename)
instrs.add(ast)
code = instrs.compile()
return code
compiler = SBCCompiler
# by Sdore, 2020

View File

@ -7,13 +7,13 @@ def read_token(src, *, lineno, offset, lineoff):
(l, src), line = lstripcount(src[offset:], whitespace), src
offset += l
if (src[:1] in '\n;'): return (offset, None)
length = int()
err = (0, 0)
for ii, i in enumerate(Token.types):
r = globals()['find_'+i.casefold()](src) or 0
if (isinstance(r, int) and r <= 0): length = max(length, -r); continue
n, s = r if (isinstance(r, tuple)) else (r, src[:r])
r = globals()['find_'+i.casefold()](src)
n, s = r if (isinstance(r, tuple)) else (r, src[:r]) if (isinstance(r, int) and r > 0) else (0, None)
if (isinstance(n, int) and n <= 0): err = max(err, (-n, s if (isinstance(s, int)) else 0)); continue
return (offset+n, Token(ii, s, lineno=lineno, offset=offset+lineoff))
else: raise SlSyntaxError("Invalid token", line, lineno=lineno, offset=offset+lineoff, length=length+l)
else: raise SlSyntaxError("Invalid token", [None]*(lineno-1)+line.split('\n'), lineno=lineno, offset=offset+lineoff, length=err[0]+l, char=err[1])
def parse_expr(src, *, lineno=1, lineoff=0):
r = list()

164
repl.py
View File

@ -6,21 +6,28 @@ from .ast import *
from .lexer import *
from utils.nolog import *
def get_node_value(node, ns):
while (isinstance(node, ASTNode)):
node = execute_node(node, ns)
if (isiterablenostr(node)):
try: node = type(node)(get_node_value(i, ns) for i in node)
except Exception: pass
return node
@dispatch
def execute_node(node: ASTCodeNode, ns):
r = None
for i in node.nodes:
if (isinstance(i, ASTElseClauseNode) and r is not None): continue
r = execute_node(i, ns)
if (r not in (None, ...)):
if (ns.flags.interactive and r not in (None, ...)):
ns.values['_'] = r
print(repr(r))
else: return
return r
@dispatch
def execute_node(node: ASTVardefNode, ns):
ns.values[node.name] = node.value
if (node.value is not None): ns.values[node.name] = get_node_value(node.value, ns)
@dispatch
def execute_node(node: ASTBlockNode, ns):
@ -28,19 +35,52 @@ def execute_node(node: ASTBlockNode, ns):
@dispatch
def execute_node(node: ASTFuncdefNode, ns):
ns.values[node.name.identifier] = (node.argdefs, node.code)
ns.values[node.name.identifier] = node
@dispatch
def execute_node(node: ASTAssignmentNode, ns):
ns.values[node.name] = execute_node(ASTBinaryExprNode(node.name, node.inplace_operator, node.value, lineno=node.lineno, offset=node.offset), ns) if (node.inplace_operator is not None) else node.value
ns.values[node.name] = execute_node(ASTBinaryExprNode(node.name, node.inplace_operator, node.value, lineno=node.value.lineno, offset=node.value.offset), ns) if (node.inplace_operator is not None) else get_node_value(node.value, ns)
@dispatch
def execute_node(node: ASTUnaryOperationNode, ns):
def _op(): execute_node(ASTAssignmentNode(node.name, node.isattr, ASTSpecialNode('=', lineno=node.unary_operator.lineno, offset=node.unary_operator.offset), ASTOperatorNode(node.unary_operator.operator[0], lineno=node.unary_operator.lineno, offset=node.unary_operator.offset), ASTLiteralNode(1 if (node.unary_operator.operator[0] in '+-') else get_node_value(node.name, ns), lineno=node.unary_operator.lineno, offset=node.unary_operator.offset), lineno=node.lineno, offset=node.offset), ns)
if (isinstance(node, ASTUnaryPreOperationNode)): _op()
res = ns.values[node.name]
if (isinstance(node, ASTUnaryPostOperationNode)): _op()
return res
@dispatch
def execute_node(node: ASTItemgetNode, ns):
return get_node_value(node.value, ns)[get_node_value(node.key, ns)]
@dispatch
def execute_node(node: ASTAttrgetNode, ns):
if (node.optype.special == '.'):
if (isinstance(node.value.value, ASTIdentifierNode) and node.value.value.identifier == 'stdio'):
if (node.attr.identifier == 'println'): return stdlib.stdio.println
elif (node.attr.identifier == 'map'): return stdlib._map
elif (node.attr.identifier == 'each'): return stdlib._each
raise NotImplementedError(node)
@dispatch
def execute_node(node: ASTFunccallNode, ns):
code_ns = ns.derive(node.callable.value.identifier)
argdefs, func = ns.values[node.callable]
func = execute_node(node.callable, ns)
if (isinstance(func, type) and issubclass(func, stdlib.Builtin)):
if (func is stdlib.stdio.println): f = print
elif (func is stdlib._map): f = lambda l: [execute_node(ASTFunccallNode(node.callable.value.value, ASTCallargsNode([i], [], lineno=node.callargs.lineno, offset=node.callargs.offset), ASTCallkwargsNode([], [], lineno=node.callkwargs.lineno, offset=node.callkwargs.offset), lineno=node.callable.lineno, offset=node.callable.offset), ns) for i in l]
elif (func is stdlib._each): f = lambda _: [execute_node(ASTFunccallNode(node.callargs.callargs[0].value, ASTCallargsNode([i], [], lineno=node.callargs.lineno, offset=node.callargs.offset), ASTCallkwargsNode([], [], lineno=node.callkwargs.lineno, offset=node.callkwargs.offset), lineno=node.callable.lineno, offset=node.callable.offset), ns) for i in get_node_value(node.callable.value.value, ns)]
else: raise NotImplementedError(func)
callarguments = CallArguments.build(node, ns)
assert (func.compatible_call(callarguments, ns) is not None)
return f(*(get_node_value(i, ns) for i in node.callargs.callargs),
*(get_node_value(j, ns) for i in node.callargs.starargs for j in get_node_value(i, ns)))
code_ns = ns.derive(str(node.callable), append=False)
for ii, i in enumerate(node.callargs.callargs):
code_ns.values[argdefs[ii].name] = i
return execute_node(func, code_ns)
code_ns.values[func.argdefs[ii].name] = get_node_value(i, ns)
return execute_node(func.code, code_ns)
@dispatch
def execute_node(node: ASTValueNode, ns):
@ -48,24 +88,35 @@ def execute_node(node: ASTValueNode, ns):
@dispatch
def execute_node(node: ASTIdentifierNode, ns):
try: return ns.values[node]
except KeyError: raise SlValidationError(f"{node} is not initialized", node, scope=ns.scope)
if (ns.values.get(node) is None): raise SlValidationError(f"{node} is not initialized", node, scope=ns.scope)
return ns.values[node]
@dispatch
def execute_node(node: ASTLiteralNode, ns):
return eval(node.literal) if (isinstance(node.literal, str)) else node.literal
if (isinstance(node.literal, str)):
try: return eval(node.literal)
except Exception as ex: raise SlReplError(ex, node, scope=ns.scope)
else: return node.literal
@dispatch
def execute_node(node: ASTListNode, ns):
return node.values
return list(node.values)
@dispatch
def execute_node(node: ASTTupleNode, ns):
return tuple(node.values)
@dispatch
def execute_node(node: ASTKeywordExprNode, ns):
#if (node.keyword.keyword == 'return'): return execute_node(node.value, ns) # TODO FIXME???
#el
if (node.keyword.keyword == 'delete'): ns.delete(node.value)
if (node.keyword.keyword == 'return'): return execute_node(node.value, ns)
elif (node.keyword.keyword == 'delete'): ns.delete(node.value)
else: raise NotImplementedError(node.keyword)
@dispatch
def execute_node(node: ASTKeywordDefNode, ns):
if (node.keyword.keyword == 'main'):
execute_node(node.code, ns)
@dispatch
def execute_node(node: ASTConditionalNode, ns):
if (execute_node(node.condition, ns)):
@ -75,23 +126,44 @@ def execute_node(node: ASTConditionalNode, ns):
@dispatch
def execute_node(node: ASTForLoopNode, ns):
ns.define(node.name, Signature.build(node.iterable, ns).valtype)
ns.weaken(node.name)
for i in execute_node(node.iterable, ns):
ns.values[node.name] = get_node_value(i, ns)
execute_node(node.code, ns)
else: return
return ...
@dispatch
def execute_node(node: ASTWhileLoopNode, ns):
while (get_node_value(execute_node(node.condition, ns), ns)):
execute_node(node.code, ns)
else: return
return ...
@dispatch
def execute_node(node: ASTElseClauseNode, ns):
execute_node(node.code, ns)
@dispatch
def execute_node(node: ASTUnaryExprNode, ns):
value = execute_node(node.value, ns)
return eval(f"{node.operator.operator} value")
value = get_node_value(node.value, ns)
try: return eval(f"{node.operator.operator} value")
except Exception as ex: raise SlReplError(ex, node, scope=ns.scope)
@dispatch
def execute_node(node: ASTBinaryExprNode, ns):
lvalue = execute_node(node.lvalue, ns)
rvalue = execute_node(node.rvalue, ns)
if (node.operator.operator == 'xor'): return eval("(lvalue and not rvalue) or (rvalue and not lvalue)")
elif (node.operator.operator == 'to'): return range(lvalue, rvalue)
else: return eval(f"lvalue {node.operator.operator} rvalue")
lvalue = get_node_value(node.lvalue, ns)
rvalue = get_node_value(node.rvalue, ns)
if (node.operator.operator == 'xor'):
try: return eval("(lvalue and not rvalue) or (rvalue and not lvalue)")
except Exception as ex: raise SlReplError(ex, node, scope=ns.scope)
elif (node.operator.operator == 'to'):
try: return range(lvalue, rvalue)
except Exception as ex: raise SlReplError(ex, node, scope=ns.scope)
else:
try: return eval(f"lvalue {node.operator.operator} rvalue")
except Exception as ex: raise SlReplError(ex, node, scope=ns.scope)
class Completer:
def __init__(self, namespace):
@ -155,9 +227,23 @@ class Completer:
matches.sort()
return matches
def repl():
class SlReplError(SlNodeException):
ex: ...
def __init__(self, ex, *args, **kwargs):
super().__init__(*args, **kwargs)
self.ex = ex
def __exline__(self):
return "Repl error"
def __exsubline__(self):
return f"\n\033[1;91mException\033[0m:\n "+'\n '.join(traceback.format_exception_only(type(self.ex), self.ex))
def repl(*, optimize=0):
ns = Namespace('<repl>')
ns.define(ASTIdentifierNode('_', lineno=None, offset=None), stdlib.Any())
ns.define(ASTIdentifierNode('_', lineno=None, offset=None), stdlib.Any)
ns.flags.interactive = True
completer = Completer(ns)
histfile = os.path.expanduser('~/.sli_history')
@ -187,8 +273,8 @@ def repl():
#if (len(tl) >= 2 and tl[-2][-1].token == '\\'): tl[-1] = tl[-2][:-1]+tl.pop() # TODO FIXME?: [['a', '+', '\\'], 'b'] --> [['a', '+', 'b']]
if (tl[0][-1].token == '{' and tl[-1][-1].token != '}'): continue
ast = build_ast(tl, interactive=True)
if (optimize): optimize_ast(ast, validate_ast(ast), optimize)
validate_ast(ast, ns)
optimize_ast(ast, ns)
execute_node(ast.code, ns)
except KeyboardInterrupt:
buf = readline.get_line_buffer()
@ -196,13 +282,33 @@ def repl():
except EOFError:
print(end='\r\033[K')
break
except (SlSyntaxError, SlValidationError) as ex:
ex.line = l[ex.lineno-1]
except (SlSyntaxException, SlNodeException) as ex:
if (not ex.srclines): ex.srclines = l
print(ex)
tl.clear()
l.clear()
finally: readline.write_history_file(histfile)
if (__name__ == '__main__'): repl()
def run_file(file, *, optimize=0):
src = file.read()
try:
tl = parse_string(src)
ast = build_ast(tl, file.name.join('""'))
if (optimize): optimize_ast(ast, validate_ast(ast), optimize)
ns = validate_ast(ast)
execute_node(ast.code, ns)
except (SlSyntaxException, SlNodeException) as ex:
if (not ex.srclines): ex.srclines = src.split('\n')
sys.exit(str(ex))
@apmain
@aparg('file', metavar='file.sl', nargs='?', type=argparse.FileType('r'))
@aparg('-O', metavar='level', help='Code optimization level', type=int, default=DEFAULT_OLEVEL)
def main(cargs):
if (cargs.file is not None): run_file(cargs.file, optimize=cargs.O)
else: repl(optimize=cargs.O)
if (__name__ == '__main__'): main(nolog=True)
# by Sdore, 2020

View File

@ -14,4 +14,5 @@ main {
stdio.println("\n")
}
# by Sdore, 2020
# by Sdore, 2021
# slang.sdore.me

View File

@ -3,7 +3,8 @@
import tokens:*
tuple read_token(str src, int lineno, int offset, int lineoff) {
tuple c = lstripcount(src#|[offset:]|#, whitespace)
tuple c = lstripcount(src[offset:], whitespace)
int l = c[0]
src = c[1]
str line = src
@ -11,39 +12,42 @@ tuple read_token(str src, int lineno, int offset, int lineoff) {
if not src or src[0] in '\n;' {
return (offset, None)
}
int length = 0
int ii = -1
int length = 0, ii = -1
for i in Token.types {
ii += 1
auto r = globals['find_'+i.casefold()](src) or 0
if r isof int and r <= 0 {
length = max(length, -r)
continue
}
tuple c
if r isof tuple {
c = r
}
else {
c = (r, src[:r])
}
if r isof tuple: c = r
else: c = (r, src[:r])
int n = c[0]
str s = c[1]
return (offset+n, Token(ii, s, lineno=lineno, offset=offset+lineoff))
} #else raise SlSyntaxError("Invalid token", line, lineno=lineno, offset=offset+lineoff, length=length)
} else: raise SlSyntaxError("Invalid token", line, lineno=lineno, offset=offset+lineoff, length=length)
return (0, Token())
}
tuple parse_expr(str src, int lineno = 1, int lineoff = 0) {
list r = [Token]
int offset
while 1 {
while true {
offset, tok = read_token(src, lineno=lineno, offset=offset, lineoff=lineoff)
if tok is None {
break
}
if tok is None: break
r.append(tok)
}
return (offset, r)
}
@ -52,23 +56,25 @@ list parse_string(str src) {
list tl = [Token]
int lines = src.count('\n')
int lineoff = 0
while src {
tuple c = parse_expr(src, lineno=lines-src.count('\n')+1, lineoff=lineoff)
int offset = c[0]
list r = c[1]
lineoff += offset
if offset < src.len {
if src[offset] == '\n' {
lineoff = 0
}
else {
lineoff += 1
}
if src[offset] == '\n': lineoff = 0
else: lineoff += 1
}
src = src[offset+1:]
tl.append(r)
}
return tl
}
# by Sdore, 2020
# by Sdore, 2021
# slang.sdore.me

View File

@ -1,49 +1,35 @@
# Slang tokens
str whitespace = ' \t\r\v\f'
const str whitespace = " \t\r\v\f"
tuple lstripcount(str s, str chars) {
int ii = -1
char i
for i in s {
ii += 1
if i not in chars {
break
}
}
else {
ii = 0
}
return (ii, s#|[ii:]|#)
if i not in chars: break
} else: ii = 0
return (ii, s[ii:])
}
class Token {
int type
const tuple types = ('SPECIAL', 'OPERATOR', 'LITERAL', 'KEYWORD', 'IDENTIFIER') # order is also resolution order
int type, lineno, offset
str token
int lineno
int offset
tuple types# = ('SPECIAL', 'OPERATOR', 'LITERAL', 'KEYWORD', 'IDENTIFIER') # order is also resolution order
constr (int .type, str .token, int .lineno, int .offset);
constr(int type, str token, int lineno, int offset) {
#.type, .token, .lineno, .offset = type, token, lineno, offset
}
repr = f"<Token {.typename} «{repr(.token)[1:-1]}» at line {.lineno}, offset {.offset}>"
#|repr {
return "<Token {.types[.type]} «{repr(.token)[1:-1]}» at line {.lineno}, offset {.offset}>"
}|#
eq = (super == x or .token == x)
#|eq {
#return super() == x or .token == x
}|#
property typename = .types[.type]
#|property typename {
#return .types[.type]
}
property length {
#return .token.length
}|#
property length = .token.length
}
# by Sdore, 2020
# by Sdore, 2021
# slang.sdore.me

View File

@ -47,6 +47,7 @@
<context ref="def:in-line-comment"/>
</include>
</context>
<context id="multiline-comment" style-ref="comment" class-disabled="no-spell-check" class="comment">
<start>#\|</start>
<end>\|#</end>
@ -54,7 +55,8 @@
<context ref="def:in-comment"/>
</include>
</context>
<context id="close-comment-outside-comment" style-ref="error">
<context id="close-comment-outside-comment" style-ref="def:error">
<match>\|#(?!#\|)</match>
</context>
@ -206,6 +208,7 @@
<context id="builtin-objects" style-ref="builtin-object">
<prefix>(?&lt;![\w\.])</prefix>
<suffix>(\.\w+)?</suffix>
<keyword>stdio</keyword>
<keyword>globals</keyword>
</context>

2
sld.py
View File

@ -1,6 +1,6 @@
#!/usr/bin/python3
# Slang .sld parser
# .sld files are used for definition of built-in, implementation-provided, external and native components.
# .sld files are used for definition of built-in, implementation-provided, external and native components, and documentation.
from .tokens import *
from utils import *

132
stdlib.py
View File

@ -1,14 +1,11 @@
#!/usr/bin/python3
# Slang stdlib
from .ast import Signature, Function, Object, Collection, CallArguments
from .ast import Signature, Function, Object, Collection, CallArguments, MultiCollection
from .tokens import *
from utils import *
class Builtin(Signature):
def __init__(self):
pass
@classitemget
def attrops(cls, optype, attr):
if (optype == '.'):
@ -25,18 +22,12 @@ class Builtin(Signature):
return self.__class__.__name__
class BuiltinFunction(Builtin, Function):
@property
def name(self):
return self.__class__.__name__
def __init__(self):
super().__init__(name=self.__class__.__name__)
class BuiltinObject(Builtin, Object): pass
class BuiltinType(Builtin):
@init_defaults
@autocast
def __init__(self, *, modifiers: paramset):
self.modifiers = modifiers
def __eq__(self, x):
return (super().__eq__(x) or issubclass(x.__class__, self.__class__) or issubclass(self.__class__, x.__class__))
@ -51,10 +42,13 @@ class BuiltinType(Builtin):
if (op in operators[10]+operators[11]+operators[12]): return type(valsig) # binary `and'. `xor', `or'
raise KeyError()
@singleton
class Any(BuiltinType):
def __eq__(self, x):
return True
class auto(BuiltinType): pass
class void(BuiltinType):
@staticitemget
def operators(op, valsig=None):
@ -81,7 +75,7 @@ class int(BuiltinType):
if (op in map(UnaryOperator, '+-~')): return int
if (op == UnaryOperator('!')): return bool
if (isinstance(valsig, (int, float))):
if (op in map(BinaryOperator, ('**', *'+-*'))): return valsig
if (op in map(BinaryOperator, ('**', *'+-*%'))): return valsig
if (op in map(BinaryOperator, ('//', '<<', '>>', *'&^|'))): return int
if (op == BinaryOperator('/')): return float
if (isinstance(valsig, int)):
@ -98,7 +92,7 @@ class float(BuiltinType):
if (op in map(UnaryOperator, '+-')): return float
if (op == UnaryOperator('!')): return bool
if (isinstance(valsig, (int, float))):
if (op in map(BinaryOperator, ('**', *'+-*'))): return float
if (op in map(BinaryOperator, ('**', *'+-*%'))): return float
if (op == BinaryOperator('/')): return float
if (op == BinaryOperator('//')): return int
raise KeyError()
@ -121,22 +115,22 @@ class str(BuiltinType):
raise KeyError()
class rstrip(BuiltinFunction):
callargssigstr = "rstrip(char)"
callargssigstr: "rstrip(char)"
@staticitemget
@staticmethod
@instantiate
def call(callargssig):
if (callargssig.kwargs or callargssig.starkwargs): raise KeyError()
return str
def compatible_call(callarguments, ns):
if (callarguments.kwargs or callarguments.starkwargs): return None
return (None, str())
class count(BuiltinFunction):
callargssig = "count(char)"
callargssig: "count(char)"
@staticitemget
@staticmethod
@instantiate
def call(callargssig):
if (callargssig.kwargs or callargssig.starkwargs): raise KeyError()
return int
def compatible_call(callarguments, ns):
if (callarguments.kwargs or callarguments.starkwargs): return None
return (None, int())
class char(BuiltinType):
@staticitemget
@ -150,58 +144,80 @@ class char(BuiltinType):
if (isinstance(valsig, (char, int)) and op in map(BinaryOperator, '+-')): return char
raise KeyError()
class i8(int): fmt = 'b'
class u8(int): fmt = 'B'
class i16(int): fmt = 'h'
class u16(int): fmt = 'H'
class i32(int): fmt = 'i'
class u32(int): fmt = 'I'
class i64(int): fmt = 'q'
class u64(int): fmt = 'Q'
#class i128(int): fmt = '
#class u128(int): fmt = '
class i8(int): fmt: 'b'
class u8(int): fmt: 'B'
class i16(int): fmt: 'h'
class u16(int): fmt: 'H'
class i32(int): fmt: 'i'
class u32(int): fmt: 'I'
class i64(int): fmt: 'q'
class u64(int): fmt: 'Q'
#class i128(int): fmt: '
#class u128(int): fmt: '
#class f8(float): fmt = '
#class f16(float): fmt = 'e'
#class f32(float): fmt = 'f'
#class f64(float): fmt = 'd'
#class f128(float): fmt = '
#class f8(float): fmt: '
#class f16(float): fmt: 'e'
#class f32(float): fmt: 'f'
#class f64(float): fmt: 'd'
#class f128(float): fmt: '
#uf8 = uf16 = uf32 = uf64 = uf128 = float
class range(BuiltinType):
keytype = int
valtype = int
keytype: int
valtype: int
@staticitemget
def operators(op, valsig=None):
raise KeyError()
class iterable(BuiltinType, Collection): pass
class iterable(Collection, BuiltinType): pass
class list(iterable):
keytype = int()
valtype = Any()
keytype: int
class tuple(iterable):
keytype = int()
valtype = Any()
def __init__(self, *, valtype=Any):
super().__init__(keytype=self.keytype, valtype=valtype)
@staticitemget
def attrops(optype, attr):
if (optype == '.'):
if (attr == 'each'): return _each()
raise KeyError()
class tuple(iterable, MultiCollection):
keytype: int
def __init__(self, *, valtypes=()):
super().__init__(keytype=self.keytype, valtypes=valtypes)
class stdio(BuiltinObject):
class println(BuiltinFunction):
callargssigstr = "println(...)"
callargssigstr: "println(...)"
@staticitemget
@instantiate
def call(callargssig):
if (callargssig.kwargs or callargssig.starkwargs): raise KeyError()
return void
@staticmethod
def compatible_call(callarguments, ns):
if (callarguments.kwargs or callarguments.starkwargs): return None
return (None, void())
class _map(BuiltinFunction):
@staticitemget
@instantiate
def call(callargssig):
if (callargssig.kwargs or callargssig.starkwargs): raise KeyError()
return list
@staticmethod
def compatible_call(callarguments, ns):
if (len(callarguments.args) != 1 or
callarguments.starargs or
callarguments.kwargs or
callarguments.starkwargs): return None
return (None, list(valtype=Any))
class _each(BuiltinFunction):
@staticmethod
def compatible_call(callarguments, ns):
if (len(callarguments.args) != 1 or
callarguments.starargs or
callarguments.kwargs or
callarguments.starkwargs): return None
#fsig = Signature.build(callarguments.args[0], ns)
return (None, list(valtype=Any))
builtin_names = {k: v for i in map(allsubclassdict, Builtin.__subclasses__()) for k, v in i.items()}
builtin_names.update({i: globals()[i] for i in (i+j for j in map(builtins.str, (8, 16, 32, 64, 128)) for i in (*'iuf', 'uf') if i+j in globals())})

View File

@ -19,13 +19,14 @@ class int {
int operator *int;
int operator //int;
int operator **int;
int operator %int;
int operator <<int;
int operator >>int;
int operator &int;
int operator ^int;
int operator |int;
range operator to int;
range operator 'to' int;
int popcount();
int length(int base=2);
@ -42,6 +43,7 @@ class float {
float operator /float;
int operator //float;
float operator **float;
float operator %float;
int round();
bool isint();

View File

@ -1,3 +1,5 @@
#| Slang `class' test. |#
class Test {
int a = 1
@ -9,29 +11,27 @@ class Test {
.a = 5
}
constr (int a) {
.a = a
}
constr (int .a);
}
main {
stdio.println(Test.a)
stdio.println(Test.a) # 1
Test t
stdio.println(t.a)
stdio.println(t.a) # 3
t.a = 2
stdio.println(t.a)
stdio.println(t.a) # 2
delete t
Test t = Test()
stdio.println(t.a)
stdio.println(t.a) # 5
delete t
Test t = Test(7)
stdio.println(t.a)
stdio.println(t.a) # 7
delete t
#Test t(10)
#stdio.println(t.a)
#stdio.println(t.a) # 10
#delete t
}

7
tests/colonkw.sl Normal file
View File

@ -0,0 +1,7 @@
#| Slang `colonkw' test. |#
int a(int y) = y
main {
a(y: 3)
}

7
tests/divzero.sl Normal file
View File

@ -0,0 +1,7 @@
#| Slang `divzero' test. |#
main {
u8 a = 1
u8 b = 0
stdio.println(a/b)
}

8
tests/each.sl Normal file
View File

@ -0,0 +1,8 @@
#| Slang `each' test. |#
main {
list l = [int: 1, 2, 3]
l.each(stdio.println)
[int: 4, 5].each(stdio.println)
}

View File

@ -2,24 +2,24 @@
#| and that is a
multiline one. |#
const u32 n = 123123 # n is of type const u32
const i64 m = 10**18 # m is of type const i64
const int z = 2**128 # z is of type const int (unsized)
const auto q = 2**256 # q is of type const int
const u32 n = 123123 # n: const u32 (unsigned 32 bit)
const i64 m = 10**18 # m: const i64 (signed 64 bit)
const int z = 2**128 # z: const int (signed unsized)
const auto q = 2**256 # q: const int (signed unsized)
char f(str x) { # f() is of type char, x is of type str
auto c = x[1] # c is of type char
return c
char f(str x) { # f(): char, x: str
auto c = x[1] # c: char
return c # char
}
auto g(str x) { # g() is of type char, x is of type str
return x[0] # retval is of type char
auto g(str x) { # g(): char, x: str
return x[0] # char
}
int h(int x) = x+1 # h() is of type int, x is of type int
int h(int x) = x+1 # h(): int, x: int
main {
stdio.println(h(n), \ # comment here too
f('123asd') + g('32') + 1) #--> «123124 f»
stdio.println(q/z/2**96) #--> «4294967296.0»
stdio.println(q/z/2**96) #--> «4294967296.0»
}

View File

@ -1,3 +1,5 @@
#| Slang `fib' test. |#
int fib(int n) {
if (n < 2) {return 1}
return fib(n-1) + fib(n-2)

View File

@ -1,3 +1,5 @@
#| Slang `for' test. |#
main {
for i in (0 to 5) {
stdio.println(i)

View File

@ -1,2 +1,4 @@
#| Slang `funcdef' test. |#
void f() {}
void f(int a?) {}

View File

@ -1,3 +1,5 @@
#| Slang `list' test. |#
list l = [int: 1, 2, 3]
main {
@ -7,4 +9,6 @@ main {
stdio.println(i, l[i])
#i -=- 1
}
for i in l stdio.println(i)
stdio.println(l, *l)
}

View File

@ -1,3 +1,5 @@
#| Slang `map' test. |#
void f(int x) = stdio.println(x)
main {

5
tests/nestedcomm.sl Normal file
View File

@ -0,0 +1,5 @@
#| Slang `nestedcomm' test. |#
#| test #| a |#|#
main { #a
}

View File

@ -1,4 +1,6 @@
#| Slang `nl' test. |#
main {
stdio.println(1, #|
|# 2)
}
}

View File

@ -1,5 +1,16 @@
#| Slang `opti' test. |#
int a = 3
void f() {a = 5}
void g() {a += 2+2}
main {
stdio.println(2**a)
stdio.println(a)
a = 4
stdio.println(a)
f()
stdio.println(a)
g()
stdio.println(a)
}

View File

@ -1,7 +1,10 @@
#| Slang `overload' test. |#
int f(int x) = x+1
int f(int x, int y) = x+y+1
main {
stdio.println(f(1))
stdio.println(f(1, 2))
f()
}

View File

@ -1,3 +1,8 @@
#| Slang `println' test. |#
main {
stdio.println('test')
stdio.println(3)
}
stdio.println(5)
stdio.println(5)

5
tests/range.sl Normal file
View File

@ -0,0 +1,5 @@
#| Slang `range' test. |#
main {
stdio.println(1 to 3)
}

View File

@ -1,6 +1,8 @@
#| Slang `redefine' test. |#
main {
int a
a = 3
a, b := (int a, char 'b')
stdio.println(a, b)
}
}

16
tests/scope.sl Normal file
View File

@ -0,0 +1,16 @@
#| Slang `scope' test. |#
int a = 3
int g() = 5
int f() {
int a = 4
int g = g()
return a+g
}
main {
int a = f()
stdio.println(a)
}

View File

@ -1,3 +1,5 @@
#| Slang `strcat' test. |#
str a = "a"
str b = "B"
str c = a+b

View File

@ -1,4 +1,6 @@
i8 a = 3
i8 b = 5
i8 c = a+b
#| Slang `sum' test. |#
i64 a = 97
i64 b = 5
i64 c = a+b
stdio.println(c)

View File

@ -1,5 +1,9 @@
#| Slang `test' test. |#
const int a = 3
int b = 2; int c = 0; int x = 9
int b = 2
int c = 0
int x = 9
int sum(int a, int z) = a + z
stdio.println(sum(a, b & 3))
@ -8,7 +12,7 @@ stdio.println(1, 2)
stdio.println('a')
stdio.println(2*(3)+-2*(5+1)/2)
stdio.println(-2*x**(2+2)*a*b+10*c)
stdio.println(*'a'+'b'*2)
stdio.println(*"a"+"b"*2)
stdio.println(-2-2-2-2-2-2-2-2)
main {
@ -20,9 +24,10 @@ main {
int test() = 2*2
stdio.println(sum(b, test()))
for i in 0 to 5 stdio.println(i)
else stdio.println(0)
for i in (0 to 5): stdio.println(i)
else: stdio.println(0)
int i = 5
while i {stdio.println(i); i -= 1;} else stdio.println('ВСЁ!')
while i: stdio.println(i); i -= 1
else: stdio.println('ВСЁ!')
}

6
tests/tuple.sl Normal file
View File

@ -0,0 +1,6 @@
#| Slang `tuple' test. |#
main {
tuple a = (int 1, str "test")
stdio.println(a, *a)
}

View File

@ -1 +1,3 @@
int _a_b_
#| Slang `underscore' test. |#
int _a_b_

View File

@ -1,4 +1,6 @@
#| Slang `wrongnl' test. |#
main {
stdio.println(1, \ 2,
3)
}
}

114
tokens.py
View File

@ -15,6 +15,9 @@ class Modifier(Keyword): pass
class ReservedKeyword(Keyword): pass
class Operator(str): pass
#def __eq__(self, x):
# if (isinstance(x, Operator)): return (type(x) is type(self) and x == self)
# return super().__eq__(x)
class UnaryOperator(Operator): pass
class BinaryOperator(Operator): pass
class BothOperator(UnaryOperator, BinaryOperator): pass
@ -51,10 +54,13 @@ keywords = (
ReservedKeyword('raise'),
ReservedKeyword('with'),
ReservedKeyword('yield'),
ReservedKeyword('include'),
ReservedKeyword('using'),
ReservedKeyword('default'),
)
operators = (*(tuple(map(*i)) for i in ( # ordered by priority
(UnaryOperator, '!+-~'),
(UnaryOperator, (*'!+-~', '++', '--', '**')),
(BinaryOperator, ('**',)),
(BinaryOperator, ('//', *'*/%')),
(BinaryOperator, '+-'),
@ -69,7 +75,8 @@ operators = (*(tuple(map(*i)) for i in ( # ordered by priority
(BinaryOperator, ('||', 'or')),
(BinaryOperator, ('to',)),
)),)
bothoperators = '+-'
unaryops = ('++', '--', '**')
bothoperators = (*'+-', *unaryops)
attrops = ('->', '@.', *'@.:')
logical_operators = ('is', 'is not', 'in', 'not in', 'not', 'and', 'but', 'xor', 'or', 'isof')
@ -83,6 +90,7 @@ def find_identifier(s):
if (not s[i].isalnum() and s[i] != '_'): break
else: i += 1
if (s[:i].isidentifier()): return i
return (0, i)
def find_keyword(s):
if (not s): return
@ -98,7 +106,9 @@ def find_literal(s):
for i in range(1, len(s)):
if (esc): esc = False; continue
if (s[i] == '\\'): esc = True; continue
if (s[i] == '\n'): break
if (s[i] == s[0]): return i+1
return (0, i)
if (s[0].isdigit() or s[0] == '.'):
i = int()
digits = '0123456789abcdef'
@ -108,7 +118,7 @@ def find_literal(s):
for i in range(1, len(s)):
if (i == 1 and s[0] == '0'):
if (s[1] not in 'box'):
if (s[1].isalnum()): return
if (s[1].isalnum()): break
return 1
else:
radix = (2, 8, 16)['box'.index(s[1])]
@ -117,29 +127,38 @@ def find_literal(s):
if (s[i].casefold() not in digits[:radix]):
if (s[i] == '_'): continue
if (s[i] == '.' and not dp): dp = True; continue
if (not digit or s[i].isalpha()): return
if (not digit or s[i].isalnum()): break
return i
digit = True
if (s[i].casefold() in digits[:radix] or s[i] == '.' and dp): return i+1
else:
if (s[i].casefold() in digits[:radix] or s[i] == '.' and dp): return i+1
c = i
while (i < len(s) and (s[i].isalnum() or s[i] == '.')): i += 1
return (-i+1, c)
def find_operator(s):
if (not s): return
for i in sorted(itertools.chain(*operators), key=len, reverse=True):
if (s.startswith(i)):
l = len(i)
if (not (i[-1].isalpha() and s[l:l+1].isidentifier())): return (len(i), BothOperator(i) if (i in bothoperators) else i)
if (not (i[-1].isalpha() and s[l:l+1].isidentifier())): return (l, BothOperator(i) if (i in bothoperators) else i)
def find_special(s):
if (not s): return
if (s[0] == '.' and s[1:2].isdigit()): return
if (s[:2] == '#|'):
l = s.find('|#', 2)
if (l == -1): return -2
return l+2
o = 2
l = 1
while (s[o:] and l > 0):
#if (s[o:o+3] in ('#|#', '|#|')): o += 2; break
if (s[o:o+2] == '#|'): l += 1; o += 2
elif (s[o:o+2] == '|#'): l -= 1; o += 2
else: o += 1
if (l > 0): return (0, o)
return o
if (s[0] == '#'):
l = s.find('\n', 1)
if (l == -1): return len(s)
return l
s = s.partition('\n')[0]
return len(s) if (s[1:] and not s[1:].isspace()) else (0, 1)
if (s[0] == '\\'): return 1
for i in sorted(specials, key=len, reverse=True):
if (s[:len(i)] == i):
@ -148,12 +167,15 @@ def find_special(s):
def operator_precedence(op):
for ii, i in enumerate(operators):
if (op in i): return ii
#else: return len(operators)
if (any(op == j and type(op) is type(j) for j in i)): return ii
else: raise WTFException(op)
class Token:
__slots__ = ('type', 'token', 'lineno', 'offset')
types = ('SPECIAL', 'OPERATOR', 'LITERAL', 'KEYWORD', 'IDENTIFIER') # order is also resolution order
class Token(Slots):
types = ('SPECIAL', 'OPERATOR', 'LITERAL', 'KEYWORD', 'IDENTIFIER') # order in tuple is resolution order
type: ...
token: ...
lineno: ...
offset: ...
def __init__(self, type, token, *, lineno, offset):
self.type, self.token, self.lineno, self.offset = type, token, lineno, offset
@ -162,7 +184,7 @@ class Token:
return f"<Token {self.types[self.type]} «{repr(self.token)[1:-1]}» at line {self.lineno}, offset {self.offset}>"
def __eq__(self, x):
return super() == x or self.token == x
return (self.token == x)
@property
def typename(self):
@ -172,43 +194,38 @@ class Token:
def length(self):
return len(self.token)
def lstripcount(s, chars):
for ii, i in enumerate(s):
if (i not in chars): break
else: ii = 0
return (ii, s[ii:])
class SlSyntaxException(Exception): pass
class SlSyntaxException(Exception, Slots): pass
class SlSyntaxNoToken(SlSyntaxException): pass
class SlSyntaxEmpty(SlSyntaxNoToken): pass
class SlSyntaxError(SlSyntaxException):
__slots__ = ('desc', 'line', 'lineno', 'offset', 'length', 'usage')
desc: ...
srclines: ...
lineno: ...
offset: ...
length: ...
char: ...
usage: None
#@dispatch
def __init__(self, desc='Syntax error', line='', *, lineno, offset, length, scope=None):
self.desc, self.line, self.lineno, self.offset, self.length = (f'\033[2m(in {scope})\033[0m ' if (scope is not None) else '')+desc, line, lineno, offset, length
self.usage = None
#@dispatch
#def __init__(self, desc='Syntax error', *, token):
# self.desc, self.line, self.lineno, self.offset, self.length = desc, '', token.lineno, token.offset, token.length
def __init__(self, desc='Syntax error', srclines=(), *, lineno, offset, length, char=0, scope=None):
self.desc, self.srclines, self.lineno, self.offset, self.length, self.char = (f'\033[2m(in {scope})\033[0m ' if (scope is not None) else '')+desc, srclines, lineno, offset, length, char
def __str__(self):
l, line = lstripcount(self.line.partition('\n')[0].replace('\t', ' '), ' \t')
l, line = lstripcount(self.srclines[self.lineno-1].partition('\n')[0]) if (self.srclines) else (0, '')
offset = (self.offset-l) if (self.offset >= 0) else (len(line)+self.offset+1)
#Syntax error:
return f"{self.desc}{self.at}"+(':\n'+\
' \033[1m'+line[:offset]+'\033[91m'*(self.offset >= 0)+line[offset:]+'\033[0m\n'+\
' '+' '*offset+'\033[95m^'+'~'*(self.length-1)+'\033[0m' if (line) else '') + \
(f"\n\n\033[1;95mCaused by:\033[0m\n{self.__cause__}" if (self.__cause__ is not None) else '')
return f"{self.desc} {self.at}"+(':\n'+\
' '*2+'\033[1m'+line[:offset]+'\033[91m'*(self.offset >= 0)+line[offset:]+'\033[0m\n'+\
' '*(2+offset)+'\033[95m'+'~'*self.char+'^'+'~'*(self.length-1-self.char)+'\033[0m' if (line) else '') + \
(f"\n\n\033[1;95mCaused by:\033[0m\n{self.__cause__ if (isinstance(self.__cause__, (SlSyntaxError, SlValidationError, SlCompilationError))) else ' '+str().join(traceback.format_exception(type(self.__cause__), self.__cause__, self.__cause__.__traceback__))}" if (self.__cause__ is not None) else '')
@property
def at(self):
return f" at line {self.lineno}, offset {self.offset}"
return f"at line {self.lineno}, offset {self.offset}" if (self.offset >= 0) else f"at the end of line {self.lineno}"
class SlSyntaxExpectedError(SlSyntaxError):
__slots__ = ('expected', 'found')
expected: ...
found: ...
def __init__(self, expected='nothing', found='nothing', *, lineno=None, offset=None, length=0, scope=None):
assert (expected != found)
@ -227,14 +244,16 @@ class SlSyntaxExpectedMoreTokensError(SlSyntaxExpectedError):
super().__init__(expected=f"More tokens for {for_}", offset=offset, **kwargs)
class SlSyntaxMultiExpectedError(SlSyntaxExpectedError):
__slots__ = ('sl', 'errlist')
sl: ...
errlist: ...
def __init__(self, expected, found, *, scope=None, errlist=None, **kwargs):
self.errlist = errlist
self.sl = len(scope)+6 if (scope is not None) else 0
super().__init__(
expected=S(',\n'+' '*(self.sl+9)).join(Stuple((i.expected+f" at offset {i.offset if (i.offset >= 0) else '<end of line>'}{i.offset+1 if (i.offset < -1) else ''}" if (not isinstance(i, SlSyntaxMultiExpectedError)) else i.expected)+(f' (for {i.usage})' if (i.usage) else '') for i in expected).strip('nothing').uniquize(str.casefold), last=',\n'+' '*(self.sl+6)+'or ') or 'nothing',
found=S(',\n'+' '*(self.sl+6)).join(Stuple(i.found+f" at {f'offset {i.offset}' if (i.offset >= 0) else '<end of line>'}{i.offset+1 if (i.offset < -1) else ''}" if (not isinstance(i, SlSyntaxMultiExpectedError)) else i.expected for i in found).strip('nothing').uniquize(str.casefold), last=',\n'+' '*(self.sl+2)+'and ') or 'nothing',
expected=S(',\n'+' '*(self.sl+9)).join(Stuple((i.expected+f" at {f'offset {i.offset}' if (i.offset >= 0) else 'the end of line'}" if (not isinstance(i, SlSyntaxMultiExpectedError)) else i.expected)+(f' (for {i.usage})' if (i.usage) else '') for i in expected).strip('nothing').uniquize(str.casefold), last=',\n'+' '*(self.sl+6)+'or ') or 'nothing',
found=S(',\n'+' '*(self.sl+6)).join(Stuple(i.found+f" at {f'offset {i.offset}' if (i.offset >= 0) else 'the end of line'}" if (not isinstance(i, SlSyntaxMultiExpectedError)) else i.expected for i in found).strip('nothing').uniquize(str.casefold), last=',\n'+' '*(self.sl+2)+'and ') or 'nothing',
#{i.offset+1 if (i.offset < -1) else ''}
scope=scope,
**kwargs
)
@ -253,11 +272,12 @@ class SlSyntaxMultiExpectedError(SlSyntaxExpectedError):
i.expected = ' '*loff+S(i.expected).indent(sl+loff).lstrip()
i.found = ' '*loff+S(i.found).indent(sl+loff).lstrip()
lineno = max(err, key=operator.attrgetter('lineno')).lineno
return cls(
expected=sorted(err, key=operator.attrgetter('expected'), reverse=True),
found=sorted(err, key=operator.attrgetter('offset')),
lineno=max(err, key=operator.attrgetter('lineno')).lineno,
offset=max(err, key=lambda x: x.offset if (x.offset >= 0) else inf).offset,
lineno=lineno,
offset=max((i for i in err if i.lineno == lineno), key=lambda x: x.offset if (x.offset >= 0) else inf).offset,
length=min(i.length for i in err if i.length) if (not any(i.offset < 0 for i in err)) else 0,
scope=scope,
errlist=list(err),