diff --git a/.gitignore b/.gitignore index 145b0b9..9df6826 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,8 @@ old.c **.db **.log -**.old **.pyc +**.old +**.bak **__pycache__ **_config.py \ No newline at end of file diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..56b2020 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "compilers/pyssembly/typeshed"] + path = compilers/pyssembly/typeshed + url = https://github.com/python/typeshed.git diff --git a/SBC/builtins.c b/SBC/builtins.c new file mode 100644 index 0000000..6320749 --- /dev/null +++ b/SBC/builtins.c @@ -0,0 +1,50 @@ +#include +#include + +typedef struct atom { + enum {b, i8, u8, i16, u16, i32, u32, i64, u64, i128, u128} type; + union { + _Bool* b; + int8_t* i8; + uint8_t* u8; + int16_t* i16; + uint16_t* u16; + int32_t* i32; + uint32_t* u32; + int64_t* i64; + uint64_t* u64; + void* data; + }; +} atom_t; + +typedef atom_t (*builtin_function)(int nargs, atom_t args[nargs]); + +typedef struct builtin { + const char* name; + builtin_function fp; +} builtin_t; + + +/// XXX /// + + +atom_t _builtin_println(int nargs, atom_t args[nargs]) { + static int res; + res = puts(args[0].data); + return (atom_t){i32, .i32 = &res}; +} + + +/// XXX /// + + +builtin_t builtins[] = { + {"println", _builtin_println}, +{NULL, NULL}}; + +builtin_function get_builtin(const char* name) { + for (builtin_t* i = builtins; i->name != NULL; i++) + if (strcmp(i->name, name) == 0) + return i->fp; + return NULL; +} diff --git a/SBC/bytecode.md b/SBC/bytecode.md index 3aeba0f..9b83a46 100644 --- a/SBC/bytecode.md +++ b/SBC/bytecode.md @@ -6,9 +6,22 @@ ### NOP 0x00 > Does nothing. -### RET +### END 0x01 +> Closes previously opened block. +### POP +0x02 +> Drops `TOS`. +### RET +0x03 > Returns `TOS` to caller. +### BLTIN +0x04 +> Reads string from the bytecode until null byte (max 255 bytes) and pushes builtin function with that name. +### CODE +0x05 +> Reads code from the bytecode until the corresponding `END` instruction and pushes a reference to it and then its length. +> Opens a block. ## Unary @@ -44,7 +57,7 @@ 0x19 > Pushes integer of smallest possible width equal to rounded `TOS`. ### CTOS -0x1a +0x1A > Pushes string consisting of char `TOS` and a null byte. @@ -62,7 +75,7 @@ ### DIV 0x23 > Pushes `TOS1 / TOS`. -### FLRDIV +### IDIV 0x24 > Pushes `TOS1 // TOS`. ### MOD @@ -71,46 +84,88 @@ ### POW 0x26 > Pushes `TOS1 ** TOS`. -### SHL +### LSH 0x27 > Pushes `TOS1 << TOS`. -### SHR +### RSH 0x28 > Pushes `TOS1 >> TOS`. ### AND 0x29 > Pushes `TOS1 & TOS`. ### OR -0x2a +0x2A > Pushes `TOS1 | TOS`. ### XOR -0x2b +0x2B > Pushes `TOS1 ^ TOS`. +## Comparisons + +### EQ +0x30 +> Pushes `TOS1 == TOS`. +### NE +0x31 +> Pushes `TOS1 != TOS`. +### LT +0x32 +> Pushes `TOS1 < TOS`. +### GT +0x33 +> Pushes `TOS1 > TOS`. +### LE +0x34 +> Pushes `TOS1 <= TOS`. +### GE +0x35 +> Pushes `TOS1 >= TOS`. +### IS +0x36 +> Pushes `TOS1 is TOS`. +### ISNOT +0x37 +> Pushes `TOS1 is not TOS`. + + +## Flow control +### IF +0x40 +> If `TOS` is false, skips bytecode until corresponding `ELSE` (if exists) or `END`. +> Opens a block. +### ELSE +0x41 +> Pops last `IF` result from `IF`-stack, and if it is true, skips bytecode to corresponding `END`. +> Opens a block. +### EXEC +0x42 +> Executes code block `TOS1` of length `TOS` and pushes the result. + + ## With argument ### ALLOC*(bytes)* -0xa0 +0xA0 > Pushes reference to `calloc(1, bytes)`. ### EXTEND*(bytes)* -0xa1 +0xA1 > Extends integer `TOS` width to `bytes` bytes if narrower. ### CONST*(bytes)* -0xa2 +0xA2 > Reads next `bytes` bytes of bytecode and pushes a reference to a copy of them. ### JUMPF*(offset)* -0xa3 +0xA3 > Jumps `offset` bytes of bytecode forward. ### JUMPB*(offset)* -0xa4 +0xA4 > Jumps `offset` bytes of bytecode backward. ### SCPGET*(cell)* -0xa5 +0xA5 > Pushes the value of cell `cell` of local scope variables array. ### SCPSET*(cell)* -0xa6 +0xA6 > Sets the value of cell `cell` of local scope variables array to `TOS`. ### CALL*(nargs)* -0xa7 -> Pops `nargs ^ (1 << 7)` TODO \ No newline at end of file +0xA7 +> Calls `TOS` with `nargs` arguments popped from stack (below the callable). diff --git a/SBC/dissbc.py b/SBC/dissbc.py new file mode 100755 index 0000000..7441c1a --- /dev/null +++ b/SBC/dissbc.py @@ -0,0 +1,50 @@ +#!/usr/bin/python3 +# SBC Disassembler + +from utils.nolog import * + +opcodes = {int(m[2], 16): m[1] for m in re.finditer(r'### (\w+).*?\n(\w+)', open(os.path.join(os.path.dirname(sys.argv[0]), 'bytecode.md')).read())} +HASARG = 0xa0 + +def dissbc(code, no_opnames=False): + cp = 0 + codesize = len(code) + blocklvl = 0 + while (cp < codesize): + opcode = code[cp]; cp += 1 + opname = opcodes.get(opcode) + if (opname == 'END'): blocklvl -= 1 + print('\t'*blocklvl, end='') + if (opname is None): print("\033[2mUNKNOWN: %02x\033[0m" % opcode); continue + print(f"\033[1m0x{opcode:02x}{' '+opname if (not no_opnames) else ''}\033[0m", end='') + if (opcode > HASARG): + arg = code[cp]; cp += 1 + print("(%d|0x%02x)" % (arg, arg), end='') + + if (opname == 'CONST'): + const = code[cp:cp+arg] + cp += arg + print(':', '0x'+const.hex(), end='') + print(' |', str().join(chr(i) if (32 <= i < 128) else '.' for i in const), end='') + elif (opname == 'BLTIN'): + name = bytearray() + while (code[cp] != 0): + name.append(code[cp]); cp += 1 + else: cp += 1 + print(':', name.decode('ascii'), end='') + + if (opname in ('IF', 'ELSE', 'CODE')): + print(':', end='') + blocklvl += 1 + + print() + +@apmain +@aparg('file', metavar='') +@aparg('--no-opnames', action='store_true') +def main(cargs): + dissbc(open(cargs.file, 'rb').read(), no_opnames=cargs.no_opnames) + +if (__name__ == '__main__'): exit(main(nolog=True), nolog=True) + +# by Sdore, 2020 diff --git a/SBC/sbc b/SBC/sbc new file mode 100755 index 0000000..f245d59 Binary files /dev/null and b/SBC/sbc differ diff --git a/SBC/sbc.c b/SBC/sbc.c new file mode 100644 index 0000000..4bb7ae5 --- /dev/null +++ b/SBC/sbc.c @@ -0,0 +1,217 @@ +// SBC + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include "builtins.c" +#include "stack.c" + +typedef uint8_t code_t; + +enum { + // Standalone + NOP = 0x00, + END = 0x01, + POP = 0x02, + RET = 0x03, + BLTIN = 0x04, + CODE = 0x05, + + // Unary + POS = 0x10, + NEG = 0x11, + NOT = 0x12, + INV = 0x13, + ATOI = 0x14, + ITOA = 0x15, + ITOF = 0x16, + CEIL = 0x17, + FLR = 0x18, + RND = 0x19, + CTOS = 0x1A, + + // Binary + ADD = 0x20, + SUB = 0x21, + MUL = 0x22, + DIV = 0x23, + IDIV = 0x24, + MOD = 0x25, + POW = 0x26, + SHL = 0x27, + SHR = 0x28, + AND = 0x29, + OR = 0x2A, + XOR = 0x2B, + + // Comparisons + EQ = 0x30, + NE = 0x31, + LT = 0x32, + GT = 0x33, + LE = 0x34, + GE = 0x35, + IS = 0x36, + ISNOT = 0x37, + + // Flow control + IF = 0x40, + ELSE = 0x41, + EXEC = 0x42, + + // With argument + ALLOC = 0xA0, + EXTEND = 0xA1, + CONST = 0xA2, + JUMPF = 0xA3, + JUMPB = 0xA4, + SCPGET = 0xA5, + SCPSET = 0xA6, + CALL = 0xA7, + + HASARG = 0xA0, +}; + +atom_t exec(code_t* code, uint32_t codesize) { // TODO: freeing + stack_t* st = stack(); + atom_t scp[255]; // TODO + code_t* cb[255]; // TODO + uint32_t cbi = 0; + + uint32_t cp = 0; + while (cp < codesize) { + code_t opcode = code[cp++]; + uint32_t ocp = cp; + + switch (opcode) { + // Standalone + case NOP: break; + case END: break; + case POP: stack_pop(st); break; + case RET: return st->top->data; + case BLTIN: { + char name[256]; + strncpy(name, (char*)code+cp, 255); + while (code[cp++] != '\0'); + stack_push(st, (atom_t){.data = get_builtin(name)}); + }; break; + case CODE: { + code_t* code_block = malloc(codesize); + static uint32_t cbp = 0; + uint32_t blocklvl = 1; + while (cp < codesize) { + code_t c = code[cp++]; + if (c == CODE || + c == IF || + c == ELSE) blocklvl++; + else if (c == END) blocklvl--; + if (blocklvl <= 0) break; + code_block[cbp++] = c; + if (c > HASARG) code_block[cbp++] = code[cp++]; + if (c == CONST) + for (uint8_t i = code_block[cbp-1]; i > 0; i--) + code_block[cbp++] = code[cp++]; + else if (c == BLTIN) + do code_block[cbp++] = code[cp++]; + while (code[cp-1] != '\0'); + } + cb[cbi++] = realloc(code_block, cbp); + free(code_block); + stack_push(st, (atom_t){u32, .u32 = &cbp}); + stack_push(st, (atom_t){.data = &cb[cbi-1]}); + }; break; + + // Unary + case POS: *st->top->data.i32 = abs(*st->top->data.i32); break; + case ITOA: { + char s[12]; + fprintf(stderr, "-- %x\n", *st->top->data.i32); + snprintf(s, sizeof(s)/sizeof(*s), "%d", *st->top->data.i32); + st->top->data.data = strdup(s); + }; break; + + // Binary (TODO) + case ADD: *st->top->data.i32 += *stack_pop(st).i32; break; + case SUB: *st->top->data.i32 -= *stack_pop(st).i32; break; + + // Comparisons + case LT: *st->top->data.b = *stack_pop(st).i32 > *st->top->data.i32; break; + + // Flow control + case EXEC: { + uint32_t exec_codesize = *stack_pop(st).u32; + code_t* exec_code = stack_pop(st).data; + stack_push(st, exec(exec_code, exec_codesize)); + }; break; + + // With argument + case CONST: { + uint8_t len = code[cp++]; + stack_push(st, (atom_t){.data = memcpy(malloc(len), code+cp, len)}); + fprintf(stderr, "-- l=%02x: %x\n", len, *st->top->data.i32); + cp += len; + }; break; + case SCPGET: { + uint8_t cell = code[cp++]; + stack_push(st, scp[cell]); + }; break; + case SCPSET: { + uint8_t cell = code[cp++]; + scp[cell] = stack_pop(st); + fprintf(stderr, "-- c%d = %d\n", cell, *scp[cell].i32); + }; break; + case CALL: { + uint8_t nargs = code[cp++]; + fprintf(stderr, "-- nargs=%d\n", nargs); + builtin_function func = stack_pop(st).data; + atom_t args[nargs]; + for (uint8_t i = 0; i < nargs; i++) + args[i] = stack_pop(st); + stack_push(st, func(nargs, args)); + }; break; + + default: fprintf(stderr, "Not Implemented opcode: 0x%02x\n", opcode); exit(3); + } + + fprintf(stderr, "[%02x", opcode); + if (opcode > HASARG) fprintf(stderr, "(%d|0x%02x)", code[ocp], code[ocp++]); + fputc(':', stderr); + do fprintf(stderr, " %02x", code[ocp++]); + while (ocp < cp && ocp < codesize); + if (st->top != NULL) { + fprintf(stderr, ", TOS = %u(0x%02x)", *st->top->data.i32, *st->top->data.i32); + if (isprint(*(char*)st->top->data.data)) { + fprintf(stderr, " | "); + for (char* p = st->top->data.data; *p != '\0'; p++) + fputc(isprint(*p)?*p:'.', stderr); + } + } + fprintf(stderr, "]\n"); + } + + return (atom_t){.data = NULL}; +} + +int main(int argc, char* argv[]) { + if (argc != 2) { + fprintf(stderr, "Usage: %s \n", basename(argv[0])); + exit(1); + } + + FILE* fd = fopen(argv[1], "rb"); + fseek(fd, 0, SEEK_END); + long fsize = ftell(fd); + fseek(fd, 0, SEEK_SET); + + code_t code[fsize]; + fread(code, sizeof(*code), fsize, fd); + fclose(fd); + + exec(code, fsize); +} + +// by Sdore, 2020 diff --git a/SBC/stack.c b/SBC/stack.c new file mode 100644 index 0000000..53238d9 --- /dev/null +++ b/SBC/stack.c @@ -0,0 +1,53 @@ +/* +#define CAT(a, b) a##_##b +#define TEMPLATE(name, type) CAT(name, type) + +#define stack TEMPLATE(stack, T) +#define stack_t TEMPLATE(stack_t, T) +#define stack_item TEMPLATE(stack_item, T) +#define stack_push TEMPLATE(stack_push, T) +#define stack_pop TEMPLATE(stack_pop, T) +*/ + +#define T atom_t + +typedef struct stack { + struct stack_item* top; +} stack_t; + +struct stack_item { + T data; + struct stack_item* below; +}; + +stack_t* stack() { + stack_t* st = malloc(sizeof(*st)); + st->top = NULL; + return st; +} + +void stack_push(stack_t* st, T data) { + struct stack_item* new = malloc(sizeof(*new)); + new->data = data; + if (st->top != NULL) new->below = st->top; + st->top = new; +} + +T stack_pop(stack_t* st) { + assert (st->top != NULL); + struct stack_item* item = st->top; + st->top = item->below; + T data = item->data; + free(item); + return data; +} + +/* +#undef stack +#undef stack_t +#undef stack_item +#undef stack_push +#undef stack_pop +*/ + +#undef T diff --git a/SBC/tests/hw.sbc b/SBC/tests/hw.sbc index 02c7e6e..cf0a31f 100644 Binary files a/SBC/tests/hw.sbc and b/SBC/tests/hw.sbc differ diff --git a/Slang.md b/Slang.md index 2db0867..49c220d 100644 --- a/Slang.md +++ b/Slang.md @@ -18,19 +18,17 @@ char f(str x) { # f() is of type char, x is of type str return c } -char g(str x) { # g() is of type char, x is of type str +auto g(str x) { # g() is of type char, x is of type str return x[0] # retval is of type char } int h(int x) = x+1 # h() is of type int, x is of type int -void main() { - print(h(n), \ +main { + stdio.println(h(n), \ # comment here too f('123asd') + g('32') + 1) #--> «123124 f» - print(q/z/2**96) #--> «4294967296.0» + stdio.println(q/z/2**96) #--> «4294967296.0» } - -main() ``` ## Tokens @@ -83,6 +81,7 @@ _Note: `*` after syntax unit means any number of them._ * ` [expr]` — `keywordexpr` (keyword expression) * ` [= ]` — `vardef` (variable definition) * ` = ` — `assignment` +* `[, ]* = ` — `unpackassignment` * `([ | | , ])` — `funccall` (function call) * `` — expr evaluation (only in REPL) * `if () ` — `conditional` @@ -106,7 +105,7 @@ _Note: `*` after syntax unit means any number of them._ Non-empty sequence of alphanumeric characters plus underscore («_»), not starting with a digit character. -Regex: `[_\w]+[_\w\d]*` +Regex: `[_\w][_\w\d]*` ### Data types @@ -133,6 +132,12 @@ _Note: `*` after syntax unit here means any number of them, `+` means at least o * `<<0> | . | .>` — number * `<"" | ''>` — string +### Literal structures + +* `[ [, ]* ]` — list +* `( [type] [, [type] ]* )` — tuple +* `{ <: >* }` — map + ## Operators * ` ` — unary operators usage @@ -143,7 +148,7 @@ _Note: `*` after syntax unit here means any number of them, `+` means at least o A set of pre-defined character operators: -* `!$%&*+-:^~` — unary charset +* `!+-~` — unary charset * `%&*+-/<=>^|` — binary charset * `==`, `**`, `//`, `<<`, `>>` — double-char binary operators @@ -167,4 +172,4 @@ A set of pre-defined keyword operators: All character class checks are performed in current locale. --- -_by Sdore, 2019_ +_by Sdore, 2020_ diff --git a/Slang.py b/Slang.py index 8171977..d2ae515 100755 --- a/Slang.py +++ b/Slang.py @@ -2,12 +2,10 @@ # Slang from .ast import * -from .repl import * from .compilers import * -from .compilers.pyssembly import * from utils.nolog import *; logstart('Slang') -def debug_compile(src, filename=''): +def compile(src, filename='', *, compiler): try: #print(f"Source: {{\n{S(src).indent()}\n}}\n") @@ -15,46 +13,36 @@ def debug_compile(src, filename=''): #print(f"Tokens:\n{pformat(tl)}\n") ast = build_ast(tl, filename) - #print(f"Code: {repr(ast.code)}\n") + #print(f"Code: {ast.code}\n") #print(f"Nodes: {pformat(list(walk_ast_nodes(ast)))}\n") optimize_ast(ast, validate_ast(ast)) - #print(f"Optimized: {repr(ast.code)}\n") + #print(f"Optimized: {ast.code}\n") - code = PyssemblyCompiler.compile_ast(ast, validate_ast(ast), filename=filename) + code = compiler.compile_ast(ast, validate_ast(ast), filename=filename) #print("Compiled.\n") - - #print("Running.\n") - #exec(code, {}) - #print("\nFinished.\n") except (SlSyntaxError, SlValidationError, SlCompilationError) as ex: - ex.line = src.split('\n')[ex.lineno-1] + if (not ex.line): ex.line = src.split('\n')[ex.lineno-1] sys.exit(ex) - except pyssembly.PyssemblyError as ex: - print('Error:', ex) - try: code = ex.code.to_code() - except pyssembly.PyssemblyError: pass - else: - print("\nHere is full pyssembly code 'til the errorneous line:\n") - dis.dis(code) - sys.exit(1) - else: return code + return code +@apmain +@aparg('file', metavar='', type=argparse.FileType('r')) +@aparg('-o', dest='output') +@aparg('-f', dest='compiler', required=True) def main(cargs): - if (cargs.o is None and not cargs.file.name.rpartition('.')[0]): - argparser.add_argument('-o', metavar='', required=True) + if (cargs.output is None and not cargs.file.name.rpartition('.')[0]): + argparser.add_argument('-o', dest='output', required=True) cargs = argparser.parse_args() src = cargs.file.read() filename = cargs.file.name - code = debug_compile(src, filename=filename.join('""')) - open(cargs.o or cargs.file.name.rpartition('.')[0]+'.pyc', 'wb').write(pyssembly.asm(code)) + _cns = importlib.import_module('.compilers.'+cargs.compiler, package=__package__).__dict__.values() + compiler = first(i for i in allsubclasses(Compiler) if i in _cns) + code = compile(src, filename=filename.join('""'), compiler=compiler) + open(cargs.output or cargs.file.name.rpartition('.')[0]+compiler.ext, 'wb').write(code) -if (__name__ == '__main__'): - argparser.add_argument('file', metavar='', type=argparse.FileType('r')) - argparser.add_argument('-o', metavar='') - cargs = argparser.parse_args() - logstarted(); exit(main(cargs)) +if (__name__ == '__main__'): exit(main()) else: logimported() -# by Sdore, 2019 +# by Sdore, 2020 diff --git a/TODO.md b/TODO.md index 49dbb26..ea47a6f 100644 --- a/TODO.md +++ b/TODO.md @@ -1 +1,14 @@ -- arrays \ No newline at end of file +- doc `import [[:][/]:]` +- allow `?` modifier only if ≥2 args ? +- https://esolangs.org/wiki/Stlang +- https://esolangs.org/wiki/Object_disoriented +- https://esolangs.org/wiki/Funciton ? +- `O.each { code }` (i.e. `O.each({code})`) +- Proposal: `if`, `for` and `while` without brackets +- OOP +- Increments +- FIXME: whitespace in srclength +- renew all `__repr__`s *AND* `__str__`s +- Proposal: `f(kw: arg)` = `f(kw=arg)` +- Proposal: [https://docs.microsoft.com/ru-ru/dotnet/csharp/language-reference/operators/default] +- FIXME: `auto' rettype diff --git a/asm/hw b/asm/hw new file mode 100755 index 0000000..a25226f Binary files /dev/null and b/asm/hw differ diff --git a/asm/hw.asm b/asm/hw.asm new file mode 100644 index 0000000..c4d77a3 --- /dev/null +++ b/asm/hw.asm @@ -0,0 +1,16 @@ +bits 64 + +;section .rodata +hw: db "hw", 0 + +.text +global _start +extern puts +_start: + push rbp + mov rbp, rsp + lea rdi, [hw+rip] + call puts + mov rax, 0 + pop rbp + ret diff --git a/asm/hw.c b/asm/hw.c new file mode 100644 index 0000000..5264558 --- /dev/null +++ b/asm/hw.c @@ -0,0 +1 @@ +main() {puts("hw");} \ No newline at end of file diff --git a/asm/hw.o b/asm/hw.o new file mode 100644 index 0000000..c5bdd3d Binary files /dev/null and b/asm/hw.o differ diff --git a/ast.py b/ast.py index ea277c2..942b511 100644 --- a/ast.py +++ b/ast.py @@ -2,7 +2,8 @@ # Slang AST import abc -from .tokens import * +from . import sld +from .lexer import * from utils import * DEBUG_PRECEDENCE = False @@ -17,11 +18,10 @@ def literal_type(x): return type(r) def common_type(l, ns): # TODO - r = set() - for i in l: r.add(Signature.build(i, ns)) + r = tuple(Signature.build(i, ns) for i in l) if (not r): return None if (len(r) > 1): raise TODO(r) - return next(iter(r)) + return first(r) class ASTNode(abc.ABC): __slots__ = ('lineno', 'offset', 'flags') @@ -99,10 +99,10 @@ class ASTCodeNode(ASTNode): self.nodes, self.name = nodes, name def __repr__(self): - return (S('\n').join(self.nodes).indent().join('\n\n') if (self.nodes) else '').join('{}') + return f"""""" def __str__(self): - return f"""""" + return (S('\n').join(self.nodes).indent().join('\n\n') if (self.nodes) else '').join('{}') @classmethod def build(cls, name): @@ -132,7 +132,7 @@ class ASTTokenNode(ASTNode): super().build(tl) off = int() for ii, i in enumerate(tl.copy()): - if (i.typename == 'SPECIAL' and (i.token[0] == '#' or i.token == '\\\n')): del tl[ii-off]; off += 1 + if (i.typename == 'SPECIAL' and (i.token[0] == '#' or i.token == '\\')): del tl[ii-off]; off += 1 if (not tl): raise SlSyntaxEmpty() @property @@ -257,16 +257,18 @@ class ASTValueNode(ASTPrimitiveNode): super().build(tl) lineno, offset = tl[0].lineno, tl[0].offset - types = [ASTLiteralNode, ASTFunccallNode, ASTAttrgetNode, ASTItemgetNode, ASTIdentifierNode, ASTLambdaNode] - if (fcall): types.remove(ASTFunccallNode); types.remove(ASTLambdaNode) if (tl[0].typename == 'LITERAL'): value = ASTLiteralNode.build(tl) else: + types = allsubclasses(ASTLiteralStructNode)+[ASTLiteralNode, ASTFunccallNode, ASTAttrgetNode, ASTItemgetNode, ASTIdentifierNode, ASTLambdaNode] + if (fcall): types.remove(ASTFunccallNode); types.remove(ASTLambdaNode) # XXX lambda too? + err = set() for i in types: tll = tl.copy() try: value = i.build(tll) + except SlSyntaxExpectedError as ex: err.add(ex); continue except SlSyntaxException: continue else: tl[:] = tll; break - else: raise SlSyntaxExpectedError('Value', tl[0]) + else: raise SlSyntaxMultiExpectedError.from_list(err) return cls(value, lineno=lineno, offset=offset) @@ -278,7 +280,7 @@ class ASTValueNode(ASTPrimitiveNode): def optimize(self, ns): super().optimize(ns) - if (isinstance(self.value, ASTIdentifierNode) and self.value in ns.values): self.value = ASTLiteralNode(repr(ns.values[self.value]), lineno=self.lineno, offset=self.offset) # TODO FIXME in functions + if (isinstance(self.value, ASTIdentifierNode) and ns.values.get(self.value)): self.value = ASTLiteralNode(repr(ns.values[self.value]), lineno=self.lineno, offset=self.offset) # TODO FIXME in functions class ASTItemgetNode(ASTPrimitiveNode): __slots__ = ('value', 'key') @@ -298,7 +300,18 @@ class ASTItemgetNode(ASTPrimitiveNode): value = ASTIdentifierNode.build(tl) # TODO: value/expr bracket = ASTSpecialNode.build(tl) if (bracket.special != '['): raise SlSyntaxExpectedError("'['", bracket) - key = ASTExprNode.build(tl) + start = None + stop = None + step = None + if (tl and not (tl[0].typename == 'SPECIAL' and tl[0].token == ':')): start = ASTExprNode.build(tl) + if (tl and tl[0].typename == 'SPECIAL' and tl[0].token == ':'): + ASTSpecialNode.build(tl) + if (tl and not (tl[0].typename == 'SPECIAL' and tl[0].token in ']:')): stop = ASTExprNode.build(tl) + if (tl and tl[0].typename == 'SPECIAL' and tl[0].token == ':'): + ASTSpecialNode.build(tl) + if (tl and not (tl[0].typename == 'SPECIAL' and tl[0].token == ']')): step = ASTExprNode.build(tl) + key = slice(start, stop, step) + else: key = start bracket = ASTSpecialNode.build(tl) if (bracket.special != ']'): raise SlSyntaxExpectedError("']'", bracket) @@ -308,7 +321,7 @@ class ASTItemgetNode(ASTPrimitiveNode): super().validate(ns) valsig = Signature.build(self.value, ns) keysig = Signature.build(self.key, ns) - if (keysig not in valsig.itemget): raise SlValidationError(f"'{valsig}' does not support itemget by key of type '{keysig}'", self, scope=ns.scope) + if ((keysig, self.key) not in valsig.itemget): raise SlValidationError(f"'{valsig}' does not support itemget by key of type '{keysig}'", self, scope=ns.scope) class ASTAttrgetNode(ASTPrimitiveNode): __slots__ = ('value', 'optype', 'attr') @@ -334,7 +347,8 @@ class ASTAttrgetNode(ASTPrimitiveNode): def validate(self, ns): super().validate(ns) - if (self.optype not in Signature.build(self.value, ns).attrops): raise SlValidationError(f"'{self.value}' does not support attribute operation '{self.optype}'", self, scope=ns.scope) + valsig = Signature.build(self.value, ns) + if ((self.optype.special, self.attr.identifier) not in valsig.attrops): raise SlValidationError(f"'{valsig}' does not support attribute operation '{self.optype}' with attr '{self.attr}'", self, scope=ns.scope) class ASTExprNode(ASTPrimitiveNode): @classmethod @@ -343,11 +357,10 @@ class ASTExprNode(ASTPrimitiveNode): lineno, offset = tl[0].lineno, tl[0].offset for ii, p in enumerate(operators[::-1]): - for i in p: - tll = tl.copy() - try: value = ASTBinaryExprNode.build(tll, i) - except SlSyntaxException: continue - else: tl[:] = tll; return value + tll = tl.copy() + try: value = ASTBinaryExprNode.build(tll, p) + except SlSyntaxException: continue + else: tl[:] = tll; return value tll = tl.copy() try: value = ASTUnaryExprNode.build(tll) @@ -356,30 +369,33 @@ class ASTExprNode(ASTPrimitiveNode): tll = tl.copy() try: value = ASTValueNode.build(tll, fcall=fcall) - except SlSyntaxException: pass + except SlSyntaxException as ex: pass else: tl[:] = tll; return value - parenthesis = ASTSpecialNode.build(tl) - if (parenthesis.special != '('): raise SlSyntaxExpectedError('Expr', parenthesis) - parenthesized = list() - lvl = 1 - while (tl): - if (tl[0].typename == 'SPECIAL'): lvl += 1 if (tl[0].token == '(') else -1 if (tl[0].token == ')') else 0 - if (lvl == 0): - parenthesis = ASTSpecialNode.build(tl) - if (parenthesis.special != ')'): raise SlSyntaxExpectedError("')'", parenthesis) - break - assert lvl > 0 - parenthesized.append(tl.pop(0)) - value = ASTExprNode.build(parenthesized) - if (parenthesized): raise SlSyntaxExpectedNothingError(parenthesized[0]) - return value + try: + parenthesis = ASTSpecialNode.build(tl) + if (parenthesis.special != '('): raise SlSyntaxExpectedError('Expr', parenthesis) + parenthesized = list() + lvl = 1 + while (tl): + if (tl[0].typename == 'SPECIAL'): lvl += 1 if (tl[0].token == '(') else -1 if (tl[0].token == ')') else 0 + if (lvl == 0): + parenthesis = ASTSpecialNode.build(tl) + if (parenthesis.special != ')'): raise SlSyntaxExpectedError("')'", parenthesis) + break + assert (lvl > 0) + parenthesized.append(tl.pop(0)) + value = ASTExprNode.build(parenthesized) + if (parenthesized): raise SlSyntaxExpectedNothingError(parenthesized[0]) + except SlSyntaxException: pass # TODO + else: return value + raise SlSyntaxExpectedError('Expr', lineno=lineno, offset=offset) class ASTUnaryExprNode(ASTExprNode): __slots__ = ('operator', 'value') def __init__(self, operator, value, **kwargs): - super(ASTPrimitiveNode, self).__init__(**kwargs) + super().__init__(**kwargs) self.operator, self.value = operator, value def __str__(self): @@ -391,50 +407,53 @@ class ASTUnaryExprNode(ASTExprNode): lineno, offset = tl[0].lineno, tl[0].offset operator = ASTOperatorNode.build(tl) - if (operator.operator in bothoperators): operator.operator = UnaryOperator(operator.operator) - elif (not isinstance(operator.operator, UnaryOperator)): raise SlSyntaxExpectedError('UnaryOperator', operator) + if (not isinstance(operator.operator, UnaryOperator)): raise SlSyntaxExpectedError('UnaryOperator', operator) value = ASTExprNode.build(tl) return cls(operator, value, lineno=lineno, offset=offset) def validate(self, ns): super().validate(ns) - sig = Signature.build(self.value, ns) + valsig = Signature.build(self.value, ns) op = self.operator.operator - if (op not in sig.operators): raise SlValidationError(f"'{sig}' does not support unary operator '{op}'", self, scope=ns.scope) + if (op not in valsig.operators): raise SlValidationError(f"'{valsig}' does not support unary operator '{op}'", self, scope=ns.scope) def optimize(self, ns): super().optimize(ns) - sig = Signature.build(self.value, ns) - if (self.value in ns.values): return ASTValueNode(ASTLiteralNode(eval(f"{self.operator} {ns.values[self.value]}"), lineno=self.lineno, offset=self.offset), lineno=self.lineno, offset=self.offset) + if (ns.values.get(self.value)): return ASTValueNode(ASTLiteralNode(eval(f"{'not' if (self.operator.operator == '!') else self.operator} {ns.values[self.value]}"), lineno=self.lineno, offset=self.offset), lineno=self.lineno, offset=self.offset) class ASTBinaryExprNode(ASTExprNode): __slots__ = ('lvalue', 'operator', 'rvalue') def __init__(self, lvalue, operator, rvalue, **kwargs): - super(ASTPrimitiveNode, self).__init__(**kwargs) + super().__init__(**kwargs) self.lvalue, self.operator, self.rvalue = lvalue, operator, rvalue def __str__(self): return f"{str(self.lvalue).join('()') if (DEBUG_PRECEDENCE or isinstance(self.lvalue, ASTBinaryExprNode) and operator_precedence(self.lvalue.operator.operator) > operator_precedence(self.operator.operator)) else self.lvalue}{str(self.operator).join(' ') if (operator_precedence(self.operator.operator) > 0) else self.operator}{str(self.rvalue).join('()') if (DEBUG_PRECEDENCE or isinstance(self.rvalue, ASTBinaryExprNode) and operator_precedence(self.rvalue.operator.operator) > operator_precedence(self.operator.operator)) else self.rvalue}" @classmethod - def build(cls, tl, op): + def build(cls, tl, opset): ASTPrimitiveNode.build(tl) lineno, offset = tl[0].lineno, tl[0].offset - lasti = None + lasti = list() lvl = int() for ii, i in enumerate(tl): if (i.typename == 'SPECIAL'): lvl += 1 if (i.token == '(') else -1 if (i.token == ')') else 0 if (lvl > 0): continue - if (i.typename == 'OPERATOR' and isinstance(i.token, BinaryOperator) and i.token == op): lasti = ii - if (lasti is None): raise SlSyntaxExpectedError('BinaryOperator', tl[0]) - tll, tl[:] = tl[:lasti], tl[lasti:] - lvalue = ASTExprNode.build(tll) - if (tll): raise SlSyntaxExpectedNothingError(tll[0]) - operator = ASTOperatorNode.build(tl) - rvalue = ASTExprNode.build(tl) + if (i.typename == 'OPERATOR' and isinstance(i.token, BinaryOperator) and i.token in opset): lasti.append(ii) + for i in lasti[::-1]: + tlr, tll = tl[:i], tl[i:] + err = set() + try: + lvalue = ASTExprNode.build(tlr) + if (tlr): raise SlSyntaxExpectedNothingError(tlr[0]) + operator = ASTOperatorNode.build(tll) + rvalue = ASTExprNode.build(tll) + except SlSyntaxException: pass + else: tl[:] = tll; break + else: raise SlSyntaxExpectedError('BinaryOperator', tl[0]) return cls(lvalue, operator, rvalue, lineno=lineno, offset=offset) @@ -447,8 +466,83 @@ class ASTBinaryExprNode(ASTExprNode): def optimize(self, ns): super().optimize(ns) - if (self.operator.operator == '**' and self.lvalue in ns.values and ns.values[self.lvalue] == 2): self.operator.operator, self.lvalue.value, ns.values[self.lvalue] = BinaryOperator('<<'), ASTLiteralNode('1', lineno=self.lvalue.value.lineno, offset=self.lvalue.value.offset), 1 - if (self.lvalue in ns.values and self.rvalue in ns.values and self.operator.operator != 'to'): return ASTValueNode(ASTLiteralNode(repr(eval(str(self))), lineno=self.lineno, offset=self.offset), lineno=self.lineno, offset=self.offset) + if (self.operator.operator == '**' and ns.values.get(self.lvalue) == 2 and (ns.values.get(self.rvalue) or 0) > 0): self.operator.operator, self.lvalue.value, ns.values[self.lvalue] = BinaryOperator('<<'), ASTLiteralNode('1', lineno=self.lvalue.value.lineno, offset=self.lvalue.value.offset), 1 + if (ns.values.get(self.lvalue) and ns.values.get(self.rvalue) and self.operator.operator != 'to'): return ASTValueNode(ASTLiteralNode(repr(eval(str(self))), lineno=self.lineno, offset=self.offset), lineno=self.lineno, offset=self.offset) + +class ASTLiteralStructNode(ASTNode): pass + +class ASTListNode(ASTLiteralStructNode): + __slots__ = ('type', 'values') + + def __init__(self, type, values, **kwargs): + super().__init__(**kwargs) + self.type, self.values = type, values + + def __repr__(self): + return f"" + + def __str__(self): + return f"[{self.type}{': ' if (self.values) else ''}{S(', ').join(self.values)}]" + + @classmethod + def build(cls, tl): + super().build(tl) + lineno, offset = tl[0].lineno, tl[0].offset + + bracket = ASTSpecialNode.build(tl) + if (bracket.special != '['): raise SlSyntaxExpectedError("'['", bracket) + type = ASTIdentifierNode.build(tl) + values = list() + if (not (tl[0].typename == 'SPECIAL' and tl[0].token == ']')): + colon = ASTSpecialNode.build(tl) + if (colon.special != ':'): raise SlSyntaxExpectedError("':'", colon) + while (tl and tl[0].typename != 'SPECIAL'): + values.append(ASTExprNode.build(tl)) + if (tl and tl[0].typename == 'SPECIAL' and tl[0].token == ','): ASTSpecialNode.build(tl) + bracket = ASTSpecialNode.build(tl) + if (bracket.special != ']'): raise SlSyntaxExpectedError("']'", bracket) + + return cls(type, values, lineno=lineno, offset=offset) + + def validate(self, ns): + typesig = Signature.build(self.type, ns) + for i in self.values: + if (Signature.build(i, ns) != typesig): raise SlValidationError(f"List item '{i}' does not match list type '{self.type}'", self, scope=ns.scope) + +class ASTTupleNode(ASTLiteralStructNode): + __slots__ = ('types', 'values') + + def __init__(self, types, values, **kwargs): + super().__init__(**kwargs) + self.types, self.values = types, values + + def __repr__(self): + return f"" + + def __str__(self): + return f"({S(', ').join((str(self.types[i])+' ' if (self.types[i] is not None) else '')+str(self.values[i]) for i in range(len(self.values)))}{','*(len(self.values) == 1)})" + + @classmethod + def build(cls, tl): + super().build(tl) + lineno, offset = tl[0].lineno, tl[0].offset + + parenthesis = ASTSpecialNode.build(tl) + if (parenthesis.special != '('): raise SlSyntaxExpectedError("'('", parenthesis) + types = list() + values = list() + while (tl and not (tl[0].typename == 'SPECIAL' and tl[0].token == ')')): + types.append(ASTIdentifierNode.build(tl) if (len(tl) >= 2 and tl[0].typename == 'IDENTIFIER' and tl[1].token != ',') else None) + values.append(ASTExprNode.build(tl)) + if (len(values) < 2 or tl and tl[0].typename == 'SPECIAL' and tl[0].token == ','): ASTSpecialNode.build(tl) + parenthesis = ASTSpecialNode.build(tl) + if (parenthesis.special != ')'): raise SlSyntaxExpectedError("')'", parenthesis) + + return cls(types, values, lineno=lineno, offset=offset) + + def validate(self, ns): + for i in range(len(self.values)): + if (Signature.build(self.values[i], ns) != Signature.build(self.types[i], ns)): raise SlValidationError(f"Tuple item '{self.values[i]}' does not match its type '{self.types[i]}'", self, scope=ns.scope) class ASTNonFinalNode(ASTNode): pass @@ -493,7 +587,7 @@ class ASTArgdefNode(ASTNonFinalNode): type = ASTTypedefNode.build(tl) name = ASTIdentifierNode.build(tl) modifier = ASTOperatorNode.build(tl) if (tl and tl[0].typename == 'OPERATOR' and tl[0].token in '+**') else ASTSpecialNode.build(tl) if (tl and tl[0].typename == 'SPECIAL' and tl[0].token in '?=') else None - value = ASTValueNode.build(tl) if (modifier == '=') else None + value = ASTValueNode.build(tl) if (isinstance(modifier, ASTSpecialNode) and modifier.special == '=') else None return cls(type, name, modifier, value, lineno=lineno, offset=offset) @@ -568,26 +662,28 @@ class ASTCallkwargsNode(ASTNonFinalNode): class ASTCallableNode(ASTNode): def validate(self, ns): super().validate(ns) - #dlog('-->', self.code.name) code_ns = ns.derive(self.code.name) - for i in self.argdefs: code_ns.define(i, redefine=True) + code_ns.define(self, redefine=True) + for i in self.argdefs: + code_ns.define(i, redefine=True) self.code.validate(code_ns) + + def optimize(self, ns): + super().optimize(ns) + code_ns = ns.derive(self.code.name) + self.code.optimize(code_ns) + +class ASTFunctionNode(ASTCallableNode): + def validate(self, ns): + super().validate(ns) + code_ns = ns.derive(self.code.name) rettype = common_type((i.value for i in self.code.nodes if (isinstance(i, ASTKeywordExprNode) and i.keyword.keyword == 'return')), code_ns) or stdlib.void() if (self.type.type.identifier == 'auto'): self.type.type.identifier = rettype.typename else: expected = Signature.build(self.type, ns) if (rettype != expected): raise SlValidationError(f"Returning value of type '{rettype}' from function with return type '{expected}'", self, scope=ns.scope) - #dlog('<--', self.code.name) - def optimize(self, ns): - super().optimize(ns) - #dlog('-->', self.code.name) - code_ns = ns.derive(self.code.name) - for i in self.argdefs: code_ns.define(i, redefine=True) - self.code.optimize(code_ns) - #dlog('<--', self.code.name) - -class ASTLambdaNode(ASTNonFinalNode, ASTCallableNode): +class ASTLambdaNode(ASTNonFinalNode, ASTFunctionNode): __slots__ = ('argdefs', 'type', 'code') def __init__(self, argdefs, type, code, **kwargs): @@ -601,7 +697,7 @@ class ASTLambdaNode(ASTNonFinalNode, ASTCallableNode): return f"" def __str__(self): - return f"{self.__fsig__()} {f'= {self.code.nodes[0].value}' if (len(self.code.nodes) == 1 and isinstance(self.code.nodes[0], ASTKeywordExprNode) and self.code.nodes[0].keyword.keyword == 'return') else repr(self.code)}" + return f"{self.__fsig__()} {f'= {self.code.nodes[0].value}' if (len(self.code.nodes) == 1 and isinstance(self.code.nodes[0], ASTKeywordExprNode) and self.code.nodes[0].keyword.keyword == 'return') else self.code}" @classmethod def build(cls, tl): @@ -634,7 +730,7 @@ class ASTBlockNode(ASTNonFinalNode): self.code = code def __str__(self): - return repr(self.code) if (len(self.code.nodes) > 1) else repr(self.code)[1:-1].strip() + return str(self.code) if (len(self.code.nodes) > 1) else str(self.code)[1:-1].strip() @classmethod def build(cls, tl): @@ -654,10 +750,10 @@ class ASTFinalNode(ASTNode): pass class ASTDefinitionNode(ASTNode): def validate(self, ns): + Signature.build(self, ns) super().validate(ns) - ns.define(self) -class ASTFuncdefNode(ASTFinalNode, ASTDefinitionNode, ASTCallableNode): +class ASTFuncdefNode(ASTFinalNode, ASTDefinitionNode, ASTFunctionNode): __slots__ = ('type', 'name', 'argdefs', 'code') def __init__(self, type, name, argdefs, code, **kwargs): @@ -671,8 +767,8 @@ class ASTFuncdefNode(ASTFinalNode, ASTDefinitionNode, ASTCallableNode): return f"" def __str__(self): - isexpr = len(self.code.nodes) == 1 and isinstance(self.code.nodes[0], ASTKeywordExprNode) and self.code.nodes[0].keyword.keyword == 'return' - r = f"{self.__fsig__()} {f'= {self.code.nodes[0].value}' if (isexpr) else repr(self.code)}" + isexpr = (len(self.code.nodes) == 1 and isinstance(self.code.nodes[0], ASTKeywordExprNode) and self.code.nodes[0].keyword.keyword == 'return') + r = f"{self.__fsig__()} {f'= {self.code.nodes[0].value}' if (isexpr) else self.code}" return r if (isexpr) else r.join('\n\n') @classmethod @@ -703,6 +799,45 @@ class ASTFuncdefNode(ASTFinalNode, ASTDefinitionNode, ASTCallableNode): return cls(type, name, argdefs, code, lineno=lineno, offset=offset) +class ASTClassdefNode(ASTFinalNode, ASTDefinitionNode, ASTCallableNode): + __slots__ = ('name', 'bases', 'code', 'type') + + argdefs = () + + def __init__(self, name, bases, code, **kwargs): + super().__init__(**kwargs) + self.name, self.bases, self.code = name, bases, code + self.type = ASTTypedefNode([], self.name, lineno=self.lineno, offset=self.offset) + + def __repr__(self): + return f"" + + def __str__(self): + return f"class {self.name}{S(', ').join(self.bases).join('()') if (self.bases) else ''} {self.code}" + + @classmethod + def build(cls, tl): + super().build(tl) + lineno, offset = tl[0].lineno, tl[0].offset + + class_ = ASTKeywordNode.build(tl) + if (class_.keyword != 'class'): raise SlSyntaxExpectedError("'class'", class_) + name = ASTIdentifierNode.build(tl) + bases = list() + if (tl and tl[0].typename == 'SPECIAL' and tl[0].token == '('): + parenthesis = ASTSpecialNode.build(tl) + if (parenthesis.special != '('): raise SlSyntaxExpectedError("'('", parenthesis) + while (tl and tl[0].typename != 'SPECIAL'): + bases.append(ASTIdentifierNode.build(tl)) + if (tl and tl[0].typename == 'SPECIAL' and tl[0].token == ','): ASTSpecialNode.build(tl) + parenthesis = ASTSpecialNode.build(tl) + if (parenthesis.special != ')'): raise SlSyntaxExpectedError("')'", parenthesis) + cdef = ASTSpecialNode.build(tl) + if (cdef.special != '{'): raise SlSyntaxExpectedError("'{'", cdef) + code = (yield from ASTCodeNode.build(name.identifier)) + + return cls(name, bases, code, lineno=lineno, offset=offset) + class ASTKeywordExprNode(ASTFinalNode): __slots__ = ('keyword', 'value') @@ -720,28 +855,113 @@ class ASTKeywordExprNode(ASTFinalNode): keyword = ASTKeywordNode.build(tl) if (not isinstance(keyword.keyword, ExprKeyword)): raise SlSyntaxExpectedError('ExprKeyword', keyword) - if (not tl): value = None - elif (keyword.keyword == 'import'): + if (keyword.keyword == 'import'): + if (not tl): raise SlSyntaxExpectedMoreTokensError('import', lineno=lineno) lineno_, offset_ = tl[0].lineno, tl[0].offset value = ASTIdentifierNode(str().join(tl.pop(0).token for _ in range(len(tl))), lineno=lineno_, offset=offset_) # TODO Identifier? (or document it) - else: value = ASTExprNode.build(tl) + elif (keyword.keyword == 'delete'): + value = ASTIdentifierNode.build(tl) + #if (not value): raise SlSyntaxExpectedError('identifier', lineno=lineno, offset=-1) + elif (tl): value = ASTExprNode.build(tl) + else: value = None return cls(keyword, value, lineno=lineno, offset=offset) def validate(self, ns): if (self.keyword.keyword == 'import'): - ns.define(ASTIdentifierNode(self.value.identifier.partition('::')[2], lineno=self.value.lineno, offset=self.value.offset)) - return + m = re.fullmatch(r'(?:(?:(\w+):)?(?:([\w./]+)/)?([\w.]+):)?([\w*]+)', self.value.identifier) + assert (m is not None) + namespace, path, pkg, name = m.groups() + if (namespace is None): namespace = 'sl' + if (path is None): path = '.' + if (pkg is None): pkg = name + pkg = pkg.replace('.', '/') + if (namespace != 'sl'): + filename = f"{os.path.join(path, pkg)}.sld" + f = sld.parse(open(filename).read()) + module_ns = f.namespace + else: + filename = f"{os.path.join(path, pkg)}.sl" + src = open(filename, 'r').read() + try: + tl = parse_string(src) + ast = build_ast(tl, filename) + optimize_ast(ast, validate_ast(ast)) + module_ns = validate_ast(ast) + except (SlSyntaxError, SlValidationError) as ex: + ex.line = src.split('\n')[ex.lineno-1] + raise SlValidationError(f"Error importing {self.value}", self, scope=ns.scope) from ex + if (name != '*'): ns.define(ASTIdentifierNode(name, lineno=self.value.lineno, offset=self.value.offset)) # TODO object + else: ns.signatures.update(module_ns.signatures) # TODO? + #return # XXX? + elif (self.keyword.keyword == 'delete'): + if (self.value.identifier not in ns): raise SlValidationNotDefinedError(self.value, scope=ns.scope) + ns.delete(self.value) super().validate(ns) +class ASTKeywordDefNode(ASTFinalNode): + __slots__ = ('keyword', 'name', 'argdefs', 'code') + + def __init__(self, keyword, name, argdefs, code, **kwargs): + super().__init__(**kwargs) + self.keyword, self.name, self.argdefs, self.code = keyword, name, argdefs, code + if (self.name is None): self.name = ASTIdentifierNode(self.code.name, lineno=self.lineno, offset=self.offset) + + def __repr__(self): + return f"" + + def __str__(self): + return f"{self.keyword}{' '+S(', ').join(self.argdefs).join('()') if (isinstance(self.keyword.keyword, DefArgsKeyword)) else f' {self.name}' if (isinstance(self.keyword.keyword, DefNamedKeyword)) else ''} {self.code}" + + @classmethod + def build(cls, tl): + super().build(tl) + lineno, offset = tl[0].lineno, tl[0].offset + + keyword = ASTKeywordNode.build(tl) + if (not isinstance(keyword.keyword, DefKeyword)): raise SlSyntaxExpectedError('DefKeyword', keyword) + name = None + argdefs = None + if (isinstance(keyword.keyword, DefNamedKeyword)): + name = ASTIdentifierNode.build(tl) + elif (isinstance(keyword.keyword, DefArgsKeyword)): + parenthesis = ASTSpecialNode.build(tl) + if (parenthesis.special != '('): raise SlSyntaxExpectedError("'('", parenthesis) + argdefs = list() + while (tl and tl[0].typename != 'SPECIAL'): + argdef = ASTArgdefNode.build(tl) + if (argdefs and argdef.value is None and argdefs[-1].value is not None): raise SlSyntaxError(f"Non-default argument {argdef} follows default argument {argdefs[-1]}") + argdefs.append(argdef) + if (tl and tl[0].typename == 'SPECIAL' and tl[0].token == ','): ASTSpecialNode.build(tl) + parenthesis = ASTSpecialNode.build(tl) + if (parenthesis.special != ')'): raise SlSyntaxExpectedError("')'", parenthesis) + cdef = ASTSpecialNode.build(tl) + if (cdef.special != '{'): raise SlSyntaxExpectedError('{', cdef) + code = (yield from ASTCodeNode.build(f"<{keyword}>")) + + return cls(keyword, name, argdefs, code, lineno=lineno, offset=offset) + + def validate(self, ns): + super().validate(ns) + code_ns = ns.derive(self.code.name) + if (isinstance(self.keyword.keyword, DefArgsKeyword)): + for i in self.argdefs: + code_ns.define(i, redefine=True) + self.code.validate(code_ns) + + def optimize(self, ns): + super().optimize(ns) + code_ns = ns.derive(self.code.name) + self.code.optimize(code_ns) + class ASTAssignvalNode(ASTNode): def validate(self, ns): super().validate(ns) if (self.name.identifier not in ns): raise SlValidationNotDefinedError(self.name, scope=ns.scope) - vartype = ns.signatures[self.name.identifier] + vartype = Signature.build(self.name, ns) if (self.value is not None): valtype = Signature.build(self.value, ns) - if (vartype != valtype): raise SlValidationError(f"Assignment of '{self.value}' of type '{valtype}' to '{self.name}' of type '{vartype}'", self, scope=ns.scope) + if (valtype != vartype and vartype != valtype): raise SlValidationError(f"Assignment of value '{self.value}' of type '{valtype}' to variable '{self.name}' of type '{vartype}'", self, scope=ns.scope) class ASTVardefNode(ASTFinalNode, ASTAssignvalNode, ASTDefinitionNode): __slots__ = ('type', 'name', 'value') @@ -760,49 +980,124 @@ class ASTVardefNode(ASTFinalNode, ASTAssignvalNode, ASTDefinitionNode): type = ASTTypedefNode.build(tl) name = ASTIdentifierNode.build(tl) - assignment = ASTSpecialNode.build(tl) if (tl and tl[0].typename == 'SPECIAL') else None - if (assignment is not None and assignment.special != '='): raise SlSyntaxExpectedError('assignment', assignment) - value = ASTExprNode.build(tl) if (assignment is not None) else None + assignment = None + value = None + if (tl and tl[0].typename == 'SPECIAL'): + assignment = ASTSpecialNode.build(tl) + if (assignment.special != '='): raise SlSyntaxExpectedError('assignment', assignment) + value = ASTExprNode.build(tl) return cls(type, name, value, lineno=lineno, offset=offset) def validate(self, ns): if (self.type.type.identifier == 'auto'): self.type.type.identifier = Signature.build(self.value, ns).typename + ns.define(self) super().validate(ns) def optimize(self, ns): super().optimize(ns) - #if (ns.signatures[self.name.identifier].modifiers.const): self.flags.optimized_out = True # TODO + #if (Signature.build(self.name, ns).modifiers.const): self.flags.optimized_out = True # TODO class ASTAssignmentNode(ASTFinalNode, ASTAssignvalNode): - __slots__ = ('name', 'inplace_operator', 'value') + __slots__ = ('name', 'isattr', 'assignment', 'inplace_operator', 'value') - def __init__(self, name, inplace_operator, value, **kwargs): + def __init__(self, name, isattr, assignment, inplace_operator, value, **kwargs): super().__init__(**kwargs) - self.name, self.inplace_operator, self.value = name, inplace_operator, value + self.name, self.isattr, self.assignment, self.inplace_operator, self.value = name, isattr, assignment, inplace_operator, value def __str__(self): - return f"{self.name} {self.inplace_operator or ''}= {self.value}" + return f"{'.'*self.isattr}{self.name} {self.inplace_operator or ''}= {self.value}" @classmethod def build(cls, tl): super().build(tl) lineno, offset = tl[0].lineno, tl[0].offset + isattr = bool() + if (tl and tl[0].typename == 'SPECIAL' and tl[0].token == '.'): ASTSpecialNode.build(tl); isattr = True name = ASTIdentifierNode.build(tl) + inplace_operator = None + if (tl and tl[0].typename == 'OPERATOR'): + inplace_operator = ASTOperatorNode.build(tl) + if (not isinstance(inplace_operator.operator, BinaryOperator)): raise SlSyntaxExpectedError('BinaryOperator', inplace_operator) + assignment = ASTSpecialNode.build(tl) + if (assignment.special not in ('=', ':=')): raise SlSyntaxExpectedError('assignment', assignment) + value = ASTExprNode.build(tl) + + return cls(name, isattr, assignment, inplace_operator, value, lineno=lineno, offset=offset) + + def validate(self, ns): + valtype = Signature.build(self.value, ns) + if (self.assignment.special == ':='): ns.define(self.name, valtype, redefine=True) + if (self.isattr): return # TODO + super().validate(ns) + vartype = Signature.build(self.name, ns) + if (vartype.modifiers.const): raise SlValidationError(f"Assignment to const '{self.name}'", self, scope=ns.scope) + +class ASTUnpackAssignmentNode(ASTFinalNode, ASTAssignvalNode): + __slots__ = ('names', 'assignment', 'inplace_operator', 'value') + + def __init__(self, names, assignment, inplace_operator, value, **kwargs): + super().__init__(**kwargs) + self.names, self.assignment, self.inplace_operator, self.value = names, assignment, inplace_operator, value + + def __str__(self): + return f"{S(', ').join(self.names)} {self.inplace_operator or ''}= {self.value}" + + @classmethod + def build(cls, tl): + super().build(tl) + lineno, offset = tl[0].lineno, tl[0].offset + + names = list() + while (tl and tl[0].typename != 'SPECIAL'): + names.append(ASTIdentifierNode.build(tl)) + if (tl and tl[0].typename == 'SPECIAL' and tl[0].token == ','): ASTSpecialNode.build(tl) inplace_operator = ASTOperatorNode.build(tl) if (tl and tl[0].typename == 'OPERATOR') else None if (inplace_operator is not None and not isinstance(inplace_operator.operator, BinaryOperator)): raise SlSyntaxExpectedError('BinaryOperator', inplace_operator) assignment = ASTSpecialNode.build(tl) - if (assignment.special != '='): raise SlSyntaxExpectedError('assignment', assignment) + if (assignment.special not in ('=', ':=')): raise SlSyntaxExpectedError('assignment', assignment) value = ASTExprNode.build(tl) - return cls(name, inplace_operator, value, lineno=lineno, offset=offset) + return cls(names, assignment, inplace_operator, value, lineno=lineno, offset=offset) def validate(self, ns): - super().validate(ns) - vartype = ns.signatures[self.name.identifier] valtype = Signature.build(self.value, ns) - if (vartype.modifiers.const): raise SlValidationError(f"Assignment to const '{self.name}'", self, scope=ns.scope) + if (self.assignment.special == ':='): + for name, type in zip(self.names, valtype.valtypes): + ns.define(name, type, redefine=True) + vartypes = tuple(Signature.build(i, ns) for i in self.names) + for name, vartype in zip(self.names, vartypes): + if (vartype.modifiers.const): raise SlValidationError(f"Assignment to const '{name}'", self, scope=ns.scope) + if (vartypes != valtype.valtypes): raise SlValidationError(f"Unpacking assignment of '{valtype}' to variables of types {vartypes}", self, scope=ns.scope) + +class ASTAttrsetNode(ASTFinalNode): + __slots__ = ('value', 'assignment') + + def __init__(self, value, assignment, **kwargs): + super().__init__(**kwargs) + self.value, self.assignment = value, assignment + + def __str__(self): + return f"{self.value}{self.assignment}" + + @classmethod + def build(cls, tl): + super().build(tl) + lineno, offset = tl[0].lineno, tl[0].offset + + value = ASTIdentifierNode.build(tl) + assignment = ASTAssignmentNode.build(tl) + if (not assignment.isattr): raise SlSyntaxExpectedError('attrset', assignment) + + return cls(value, assignment, lineno=lineno, offset=offset) + + def validate(self, ns): + assert (self.assignment.isattr) + super().validate(ns) + # TODO: attr check + #valsig = Signature.build(self.value, ns) + #if ((self.optype.special, self.attr.identifier) not in valsig.attrops): raise SlValidationError(f"'{valsig}' does not support attribute operation '{self.optype}' with attr '{self.attr}'", self, scope=ns.scope) class ASTFunccallNode(ASTFinalNode): __slots__ = ('callable', 'callargs', 'callkwargs') @@ -832,7 +1127,7 @@ class ASTFunccallNode(ASTFinalNode): def validate(self, ns): super().validate(ns) fsig = Signature.build(self.callable, ns) - if (not isinstance(fsig, Function)): raise SlValidationError(f"'{self.callable}' of type '{fsig}' is not callable", self.callable, scope=ns.scope) + if (not isinstance(fsig, Callable)): raise SlValidationError(f"'{self.callable}' of type '{fsig}' is not callable", self.callable, scope=ns.scope) callargssig = CallArguments.build(self, ns) # TODO: starargs if (callargssig not in fsig.call): raise SlValidationError(f"Parameters '({callargssig})' don't match any of '{self.callable}' signatures:\n{S(fsig.callargssigstr).indent()}\n", self, scope=ns.scope) @@ -843,8 +1138,11 @@ class ASTConditionalNode(ASTFinalNode): super().__init__(**kwargs) self.condition, self.code = condition, code + def __repr__(self): + return f"" + def __str__(self): - return f"if ({self.condition}) {self.code}" + return f"if {self.condition} {self.code}" @classmethod def build(cls, tl): @@ -853,11 +1151,7 @@ class ASTConditionalNode(ASTFinalNode): if_ = ASTKeywordNode.build(tl) if (if_.keyword != 'if'): raise SlSyntaxExpectedError("'if'", if_) - parenthesis = ASTSpecialNode.build(tl) - if (parenthesis.special != '('): raise SlSyntaxExpectedError("'('", parenthesis) condition = ASTExprNode.build(tl) - parenthesis = ASTSpecialNode.build(tl) - if (parenthesis.special != ')'): raise SlSyntaxExpectedError("')'", parenthesis) code = (yield from ASTBlockNode.build(tl)) return cls(condition, code, lineno=lineno, offset=offset) @@ -869,8 +1163,11 @@ class ASTForLoopNode(ASTFinalNode): super().__init__(**kwargs) self.name, self.iterable, self.code = name, iterable, code + def __repr__(self): + return f"" + def __str__(self): - return f"for {self.name} in ({self.iterable}) {self.code}" + return f"for {self.name} in {self.iterable} {self.code}" @classmethod def build(cls, tl): @@ -880,17 +1177,19 @@ class ASTForLoopNode(ASTFinalNode): for_ = ASTKeywordNode.build(tl) if (for_.keyword != 'for'): raise SlSyntaxExpectedError("'for'", for_) name = ASTIdentifierNode.build(tl) - in_ = ASTKeywordNode.build(tl) - if (in_.keyword != 'in'): raise SlSyntaxExpectedError("'in'", in_) - parenthesis = ASTSpecialNode.build(tl) - if (parenthesis.special != '('): raise SlSyntaxExpectedError("'('", parenthesis) + in_ = ASTOperatorNode.build(tl) + if (in_.operator != 'in'): raise SlSyntaxExpectedError("'in'", in_) iterable = ASTExprNode.build(tl) - parenthesis = ASTSpecialNode.build(tl) - if (parenthesis.special != ')'): raise SlSyntaxExpectedError("')'", parenthesis) code = (yield from ASTBlockNode.build(tl)) return cls(name, iterable, code, lineno=lineno, offset=offset) + def validate(self, ns): + super().validate(ns) + # TODO: validate iterability + ns.define(self.name, Signature.build(self.iterable, ns).valtype) + ns.weaken(self.name) + class ASTWhileLoopNode(ASTFinalNode): __slots__ = ('condition', 'code') @@ -898,8 +1197,11 @@ class ASTWhileLoopNode(ASTFinalNode): super().__init__(**kwargs) self.condition, self.code = condition, code + def __repr__(self): + return f"" + def __str__(self): - return f"while ({self.condition}) {self.code}" + return f"while {self.condition} {self.code}" @classmethod def build(cls, tl): @@ -908,11 +1210,7 @@ class ASTWhileLoopNode(ASTFinalNode): while_ = ASTKeywordNode.build(tl) if (while_.keyword != 'while'): raise SlSyntaxExpectedError("'while'", while_) - parenthesis = ASTSpecialNode.build(tl) - if (parenthesis.special != '('): raise SlSyntaxExpectedError("'('", parenthesis) condition = ASTExprNode.build(tl) - parenthesis = ASTSpecialNode.build(tl) - if (parenthesis.special != ')'): raise SlSyntaxExpectedError("')'", parenthesis) code = (yield from ASTBlockNode.build(tl)) return cls(condition, code, lineno=lineno, offset=offset) @@ -924,6 +1222,9 @@ class ASTElseClauseNode(ASTFinalNode): super().__init__(**kwargs) self.code = code + def __repr__(self): + return f"" + def __str__(self): return f"else {self.code}" @@ -946,6 +1247,7 @@ def build_ast(code, name=None, *, interactive=False): final_nodes = ASTFinalNode.__subclasses__() if (interactive): final_nodes += (ASTExprNode,) + for ii, tl in enumerate(code): if (not tl): continue lineno, offset = tl[0].lineno, tl[0].offset @@ -960,17 +1262,18 @@ def build_ast(code, name=None, *, interactive=False): try: r = next(r) except StopIteration as ex: code_stack.pop(); r = ex.args[0] else: - assert r is None + assert (r is None) if (c): if (c[-1].typename == 'SPECIAL' and c[-1].token == '}'): code.insert(ii+1, [c.pop()]) code.insert(ii+1, c) err.clear() break - assert r is not None + assert (r is not None) if (c): raise SlSyntaxExpectedNothingError(c[0]) except SlSyntaxEmpty: err.clear(); break - except SlSyntaxNoToken: err.add(SlSyntaxExpectedError(f"More tokens for {i.__name__[3:-4]}", lineno=lineno, offset=offset)) - except SlSyntaxExpectedError as ex: ex.expected += f" at offset {ex.offset if (ex.offset != -1) else ''} (for {i.__name__[3:-4]})"; err.add(ex) + except SlSyntaxNoToken: err.add(SlSyntaxExpectedMoreTokensError(i.__name__[3:-4], lineno=lineno, offset=-2)) + except SlSyntaxMultiExpectedError as ex: pass#err.add(ex) # TODO FIXME + except SlSyntaxExpectedError as ex: ex.usage = i.__name__[3:-4]; err.add(ex) else: code_stack[-1][0].send(r); err.clear(); break else: if (len(code_stack) > 1 and tl and tl[0].typename == 'SPECIAL' and tl[0].token == '}'): @@ -980,9 +1283,9 @@ def build_ast(code, name=None, *, interactive=False): else: raise WTFException() elif (not err): raise SlSyntaxError("Unknown structure", lineno=lineno, offset=offset, length=0, scope='.'.join(i[1] for i in code_stack if i[1])) - if (err): raise SlSyntaxMultiExpectedError(S(sorted(set(map(operator.attrgetter('expected'), err)), reverse=True)).uniquize(), S(sorted(err, key=operator.attrgetter('offset'))).uniquize(), lineno=max(err, key=operator.attrgetter('lineno')).lineno, offset=max(err, key=lambda x: x.offset if (x.offset != -1) else inf).offset, length=min(i.length for i in err if i.length), scope='.'.join(i[1] for i in code_stack if i[1]) if (code_stack[0][1] is not None) else None) + if (err): raise SlSyntaxMultiExpectedError.from_list(err, scope='.'.join(i[1] for i in code_stack if i[1]) if (code_stack[0][1] is not None) else None) - assert len(code_stack) == 1 + assert (len(code_stack) == 1) try: next(code_stack.pop()[0]) except StopIteration as ex: return ex.args[0] @@ -994,33 +1297,41 @@ def walk_ast_nodes(node): for i in node.__slots__: yield from walk_ast_nodes(getattr(node, i)) class _SignatureBase: pass - class Signature(_SignatureBase): __slots__ = ('typename', 'modifiers') operators = dict() - call = dict() - itemget = dict() - attrops = () @init_defaults @autocast - def __init__(self, *, typename, modifiers: paramset, attrops: tuple): - self.typename, self.modifiers, self.attrops = typename, modifiers, attrops + def __init__(self, *, typename, modifiers: paramset): + self.typename, self.modifiers = typename, modifiers @property def __reprname__(self): - return type(self).__name__ + return self.__class__.__name__ def __repr__(self): - return f"<{self.__reprname__} {type(self).__name__}>" + return f"<{self.__reprname__} '{self.name}'>" def __eq__(self, x): - return self.typename == x.typename + return (x is not None and self.typename == x.typename) def __hash__(self): return hash(tuple(getattr(self, i) for i in self.__slots__)) + @property + def name(self): + return self.typename + + @staticitemget + def itemget(x): + raise KeyError() + + @staticitemget + def attrops(optype, attr): + raise KeyError() + @classmethod @dispatch def build(cls, x: ASTArgdefNode, ns): # TODO: modifiers @@ -1028,17 +1339,18 @@ class Signature(_SignatureBase): @classmethod @dispatch - def build(cls, x: ASTVardefNode, ns): + def build(cls, x: ASTAssignvalNode, ns): r = cls.build(x.type, ns) - ns.signatures[x.name.identifier] = r + #ns.signatures[x.name.identifier] = r if (x.value is not None): ns.values[x.name] = x.value if (not r.modifiers.volatile and r.modifiers.const) else None return r @classmethod @dispatch def build(cls, x: ASTTypedefNode, ns): - if (x.type.identifier not in builtin_names): raise SlValidationNotDefinedError(x.type, scope=ns.scope) - return builtin_names[x.type.identifier](modifiers=x.modifiers) + r = cls.build(x.type, ns) + r.modifiers.update(x.modifiers) + return r @classmethod @dispatch @@ -1057,6 +1369,16 @@ class Signature(_SignatureBase): if (x.identifier not in ns): raise SlValidationNotDefinedError(x, scope=ns.scope) return ns.signatures[x.identifier] + @classmethod + @dispatch + def build(cls, x: ASTListNode, ns): + return Collection(keytype=stdlib.int, valtype=Signature.build(x.type, ns)) + + @classmethod + @dispatch + def build(cls, x: ASTTupleNode, ns): + return MultiCollection(keytype=stdlib.int, valtypes=tuple(Signature.build(t if (t is not None) else v, ns) for t, v in zip(x.types, x.values))) + @classmethod @dispatch def build(cls, x: ASTFunccallNode, ns): @@ -1067,15 +1389,25 @@ class Signature(_SignatureBase): def build(cls, x: ASTFuncdefNode, ns): return Function.build(x, ns) + @classmethod + @dispatch + def build(cls, x: ASTClassdefNode, ns): + return Class.build(x, ns) + + @classmethod + @dispatch + def build(cls, x: ASTKeywordDefNode, ns): + return KeywordDef.build(x, ns) + @classmethod @dispatch def build(cls, x: ASTItemgetNode, ns): - return cls.build(x.value, ns).itemget[cls.build(x.key, ns)] + return cls.build(x.value, ns).itemget[cls.build(x.key, ns), x.key] @classmethod @dispatch def build(cls, x: ASTAttrgetNode, ns): - return cls.build(x.value, ns).attrops[x.optype] + return cls.build(x.value, ns).attrops[x.optype.special, x.attr.identifier] @classmethod @dispatch @@ -1092,33 +1424,131 @@ class Signature(_SignatureBase): def build(cls, x: _SignatureBase, ns): return x -class Function(Signature): - __slots__ = ('name', 'call') +class Callable(Signature, abc.ABC): + __slots__ = ('call',) + + @abc.abstractproperty + def callargssigstr(self): + pass + +class Function(Callable): + __slots__ = ('name',) def __init__(self, *, name, **kwargs): super().__init__(typename='function', **kwargs) self.name = name self.call = listmap() - def __repr__(self): - return f"" - @property def callargssigstr(self): return '\n'.join(f"{self.name}({args})" for args, ret in self.call.items()) + @staticitemget + def attrops(optype, attr): + if (optype == '.'): + if (attr == 'map'): return stdlib._map() + raise KeyError() + @classmethod @dispatch def build(cls, x: ASTFuncdefNode, ns, *, redefine=False): name = x.name.identifier if (name not in ns): ns.signatures[name] = cls(name=name) callargssig = CallArguments(args=tuple(Signature.build(i, ns) for i in x.argdefs)) - if (not redefine and callargssig in ns.signatures[name].call): raise SlValidationRedefinedError(x, ns.signatures[name].call[callargssig], scope=ns.scope) + if (not redefine and callargssig in ns.signatures[name].call): raise SlValidationRedefinedError(x.name, ns.signatures[name].call[callargssig], scope=ns.scope) ns.signatures[name].call[callargssig] = Signature.build(x.type, ns) return ns.signatures[name] -class Object(Signature): - __slots__ = ('attrops',) +class KeywordDef(Callable): + __slots__ = ('name',) + + def __init__(self, *, name, **kwargs): + super().__init__(typename='keyworddef', **kwargs) + self.name = name + self.call = listmap() + + @property + def callargssigstr(self): + return '\n'.join(f"{self.name}({args})" for args, ret in self.call.items()) + + @classmethod + @dispatch + def build(cls, x: ASTKeywordDefNode, ns, *, redefine=False): + name = x.name.identifier + if (name not in ns): ns.signatures[name] = cls(name=name) + callargssig = CallArguments(args=tuple(Signature.build(i, ns) for i in x.argdefs or ())) + if (not redefine and callargssig in ns.signatures[name].call): raise SlValidationRedefinedError(x.name, ns.signatures[name].call[callargssig], scope=ns.scope) + ns.signatures[name].call[callargssig] = stdlib.void + return ns.signatures[name] + +class Object(Signature): pass + +class Collection(Object): + __slots__ = ('keytype', 'valtype') + + def __init__(self, *, keytype, valtype): + self.keytype, self.valtype = keytype, valtype + + @property + def typename(self): + return self.valtype.typename + + @itemget + @instantiate + def itemget(self, keysig, key): + if (keysig == self.keytype): return self.valtype + raise KeyError() + +class MultiCollection(Collection): + __slots__ = ('valtypes', 'typename') + + def __init__(self, *, keytype, valtypes): + Object.__init__(self, typename='tuple') + self.keytype, self.valtypes = keytype, valtypes + + @itemget + @instantiate + def itemget(self, keysig, key): + if (keysig == self.keytype): return self.valtypes[int(key)] + raise KeyError() + +class Class(Object, Callable): + __slots__ = ('name', 'scope', 'constructor') + + def __init__(self, *, name, scope, **kwargs): + super().__init__(typename='class', **kwargs) + self.name, self.scope = name, scope + self.constructor = listmap() + + def __str__(self): + return self.name + + @property + def callargssigstr(self): + return '\n'.join(f"{self.name}({args})" for args, ret in self.call.items()) + + @itemget + def call(self, callargssig): + return self.constructor[callargssig] + + @itemget + def attrops(self, optype, attr): + if (optype == '.'): + return self.scope.signatures[attr] + raise KeyError() + + @classmethod + @dispatch + def build(cls, x: ASTClassdefNode, ns, *, redefine=False): + name = x.name.identifier + if (not redefine and name in ns): raise SlValidationRedefinedError(x.name, ns.signatures[name], scope=ns.scope) + else: ns.signatures[name] = cls(name=name, scope=ns.derive(x.code.name)) + for i in x.code.nodes: + if (isinstance(i, ASTKeywordDefNode) and i.keyword.keyword == 'constr'): + callargssig = CallArguments(args=tuple(Signature.build(j, ns) for j in i.argdefs or ())) + if (not redefine and callargssig in ns.signatures[name].constructor): raise SlValidationRedefinedError(x.name, ns.signatures[name].constructor[callargssig], scope=ns.scope) + ns.signatures[name].constructor[callargssig] = ns.signatures[name] + return ns.signatures[name] class CallArguments: __slots__ = ('args', 'starargs', 'kwargs', 'starkwargs') @@ -1147,7 +1577,7 @@ class CallArguments: return cls(args=tuple(Signature.build(i, ns) for i in x.callargs.callargs), kwargs=tuple((k, Signature.build(v, ns)) for k, v in x.callkwargs.callkwargs)) class Namespace: - __slots__ = ('signatures', 'scope', 'values', 'refcount', 'flags', 'warnclasses') + __slots__ = ('signatures', 'scope', 'values', 'weak', 'refcount', 'flags', 'warnclasses') class _Values: @init_defaults @@ -1169,7 +1599,7 @@ class Namespace: @dispatch def __getitem__(self, x: ASTIdentifierNode): - return self.values[x.identifier] + return self[x.identifier] @dispatch def __getitem__(self, x: str): @@ -1201,44 +1631,90 @@ class Namespace: @dispatch def __delitem__(self, x: ASTIdentifierNode): - del self.values[x.identifier] + del self[x.identifier] + + @dispatch + def __delitem__(self, x: str): + del self.values[x] + + def get(self, x): + try: return self[x] + except (DispatchError, KeyError): return None + + def items(self): + return self.values.items() def copy(self): return self.__class__(values=self.values.copy()) @init_defaults - def __init__(self, scope, *, signatures: dict, values: _Values, refcount: lambda: Sdict(int), warnclasses: paramset, **kwargs): - self.scope, self.signatures, self.values, self.refcount, self.warnclasses = scope, signatures, values, refcount, warnclasses + def __init__(self, scope, *, signatures: dict, values: _Values, weak: set, refcount: lambda: Sdict(int), warnclasses: paramset, **kwargs): + self.scope, self.signatures, self.values, self.weak, self.refcount, self.warnclasses = scope, signatures, values, weak, refcount, warnclasses self.flags = paramset(k for k, v in kwargs.items() if v) - #self.derive.clear_cache() + #self.derive.clear_cache() # TODO FIXME (also cachedproperty) def __repr__(self): return f"" def __contains__(self, x): - return x in builtin_names or x in self.signatures + return (x in builtin_names or x in self.signatures) @cachedfunction def derive(self, scope): - return Namespace(signatures=self.signatures.copy(), values=self.values.copy(), scope=self.scope+'.'+scope) + return Namespace(signatures=self.signatures.copy(), values=self.values.copy(), weak=self.weak.copy(), scope=self.scope+'.'+scope) - @dispatch - def define(self, x: ASTIdentifierNode): - if (x.identifier in self): raise SlValidationRedefinedError(x, self.signatures[x.identifier], scope=self.scope) - self.values[x.identifier] = None - self.signatures[x.identifier] = None + #@dispatch + #def define(self, x: ASTIdentifierNode): + # if (x.identifier in self and x.identifier not in self.weak): raise SlValidationRedefinedError(x, self.signatures[x.identifier], scope=self.scope) + # self.values[x] = None + # self.signatures[x.identifier] = None + # self.weak.discard(x.identifier) @dispatch def define(self, x: ASTFuncdefNode): return self.define(x, redefine=True) + #@dispatch + #def define(self, x: str, *, value=None): + # assert (x not in self) + # self.values[x] = value + # self.signatures[x] = None + @dispatch - def define(self, x, redefine=False): + def define(self, x: lambda x: hasattr(x, 'name'), sig=None, *, redefine=False): if (redefine): try: del self.values[x.name] except KeyError: pass - elif (x.name.identifier in self): raise SlValidationRedefinedError(x.name, self.signatures[x.name.identifier], scope=self.scope) - self.signatures[x.name.identifier] = Signature.build(x, self) + try: del self.signatures[x.name.identifier] + except KeyError: pass + self.define(x.name, sig if (sig is not None) else Signature.build(x, self), redefine=redefine) + + @dispatch + def define(self, x: ASTIdentifierNode, sig, *, redefine=False): + if (redefine): + try: del self.values[x] + except KeyError: pass + try: del self.signatures[x.identifier] + except KeyError: pass + elif (x.identifier in self and x.identifier not in self.weak): raise SlValidationRedefinedError(x, self.signatures[x.identifier], scope=self.scope) + self.signatures[x.identifier] = sig + self.weak.discard(x.identifier) + + @dispatch + def weaken(self, x: ASTIdentifierNode): + self.weak.add(x.identifier) + + @dispatch + def delete(self, x: ASTIdentifierNode): + ok = bool() + try: del self.values[x] + except KeyError: pass + else: ok = True + try: del self.signatures[x.identifier] + except KeyError: pass + else: ok = True + self.weak.discard(x.identifier) + if (not ok): raise SlValidationNotDefinedError(x, scope=self.scope) from . import stdlib from .stdlib import builtin_names @@ -1257,10 +1733,11 @@ class SlValidationError(SlValidationException): def __str__(self): l, line = lstripcount(self.line.partition('\n')[0].replace('\t', ' '), ' \t') - offset = (self.node.offset-l) if (self.node.offset != -1) else len(line) + offset = (self.node.offset-l) if (self.node.offset >= 0) else (len(line)+self.node.offset+1) return (f'\033[2m(in {self.scope})\033[0m ' if (self.scope is not None) else '')+f"Validation error: {self.desc}{self.at}"+(':\n'+\ - ' \033[1m'+line[:offset]+'\033[91m'+line[offset:]+'\033[0m\n'+\ - ' '+' '*offset+'\033[95m^'+'~'*(self.node.length-1) if (line) else '') + ' \033[1m'+line[:offset]+'\033[91m'*(self.node.offset >= 0)+line[offset:]+'\033[0m\n'+\ + ' '+' '*offset+'\033[95m^'+'~'*(self.node.length-1)+'\033[0m' if (line) else '') + \ + (f"\n\n\033[1;95mCaused by:\033[0m\n{self.__cause__}" if (self.__cause__ is not None) else '') @property def at(self): @@ -1280,4 +1757,4 @@ class SlValidationRedefinedError(SlValidationError): def optimize_ast(ast, ns): return ast.code.optimize(ns) -# by Sdore, 2019 +# by Sdore, 2020 diff --git a/compilers/__init__.py b/compilers/__init__.py index 06ad96b..e563661 100644 --- a/compilers/__init__.py +++ b/compilers/__init__.py @@ -1,7 +1,8 @@ #!/usr/bin/python3 # Slang compilers -import abc +from ..ast import SlSyntaxError, SlValidationError +import abc, traceback class Compiler(abc.ABC): @abc.abstractclassmethod @@ -23,9 +24,10 @@ class SlCompilationError(Exception): def __str__(self): l, line = lstripcount(self.line.partition('\n')[0].replace('\t', ' '), ' \t') offset = (self.node.offset-l) if (self.node.offset != -1) else len(line) - return (f'\033[2m(in {self.scope})\033[0m ' if (self.scope is not None) else '')+f"Compilation error: {self.desc}{self.at}"+(':\n'+\ - ' \033[1m'+line[:offset]+'\033[91m'+line[offset:]+'\033[0m\n'+\ - ' '+' '*offset+'\033[95m^'+'~'*(self.node.length-1) if (line) else '') + return (f'\033[2m(in {self.scope})\033[0m ' if (self.scope is not None) else '') + f"Compilation error: {self.desc}{self.at}"+(':\n' + \ + ' \033[1m'+line[:offset]+'\033[91m'+line[offset:]+'\033[0m\n' + \ + ' '+' '*offset+'\033[95m^'+'~'*(self.node.length-1)+'\033[0m' if (line) else '') + \ + (f"\n\n\033[1;95mCaused by:\033[0m\n{self.__cause__ if (isinstance(self.__cause__, (SlSyntaxError, SlValidationError, SlCompilationError))) else ' '+str().join(traceback.format_exception(type(self.__cause__), self.__cause__, self.__cause__.__traceback__))}" if (self.__cause__ is not None) else '') @property def at(self): diff --git a/compilers/esolang/gibberish.py b/compilers/esolang/gibberish.py new file mode 100644 index 0000000..0256dbd --- /dev/null +++ b/compilers/esolang/gibberish.py @@ -0,0 +1,135 @@ +#!/usr/bin/python3 +# Slang Esolang Gibberish compilation target (PoC) +# https://esolangs.org/wiki/Gibberish_(programming_language) + +from .. import * +from Slang.ast import * +from utils import * + +class Instrs: + binopmap = { + '+': b'ea', + '-': b'es', + '*': b'em', + '/': b'ed', + '<<': b'fl', + '>>': b'fr', + '&': b'ga', + } + + @init_defaults + @autocast + def __init__(self, ns, stack: Slist, functions: dict): + self.ns, self.stack, self.functions = ns, stack, functions + self.code = bytearray() + + @dispatch + def add(self, x: ASTRootNode): + self.add(x.code) + + @dispatch + def add(self, x: ASTCodeNode): + for i in x.nodes: + self.add(i) + + @dispatch + def add(self, x: ASTVardefNode): + if (x.value is not None): + self.load(x.value) + self.stack[-1] = Signature.build(x, self.ns) + + @dispatch + def add(self, x: ASTAssignmentNode): + sig = Signature.build(x.name, self.ns) + for ii, i in enumerate(self.stack): + if (i == sig): self.stack[ii] = None + self.load(x.value) + self.stack[-1] = sig + + @dispatch + def add(self, x: ASTFunccallNode): + self.load(x) + self.code += b'ev' + self.stack.pop() + + @dispatch + def add(self, x: ASTFuncdefNode): + code_ns = self.ns.derive(x.name.identifier) + if (x.name.identifier == 'main'): + assert not x.argdefs + name = x.name.identifier + else: name = f"{x.name.identifier}__{self.ns.signatures[x.name.identifier].call.index(CallArguments(args=tuple(Signature.build(i, code_ns) for i in x.argdefs)))}" + f_instrs = Instrs(ns=code_ns, stack=(Signature.build(i, code_ns) for i in x.argdefs), functions=self.functions.copy()) + f_instrs.add(x.code) + #dlog(f"{x.__fsig__()} instrs:\n"+'\n'.join(f_instrs.instrs)+'\n') + self.functions[name] = f_instrs + + @dispatch + def add(self, x: ASTKeywordExprNode): + if (x.keyword.keyword == 'return'): + self.load(x.value) + else: raise NotImplementedError(x.keyword) + + @dispatch + def load(self, x: ASTLiteralNode): + s = str(eval(str(x.literal))).encode().split(b']') + if (s): + s[0] = b'['+s[0] + s[-1] += b']' + self.code += b'][93]eigtec['.join(s) + b'c'*(len(s)-1) + if (issubclass(literal_type(x.literal), (int, float))): self.code += b'ei' + self.stack.append(None) + + @dispatch + def load(self, x: ASTIdentifierNode): + dlog(self.stack) + sig = Signature.build(x, self.ns) + i = self.stack.rindex(sig) + self.code += (b'[%d]eip' if (i >= 10) else b'%dep') % i + self.stack.append(sig) + + @dispatch + def load(self, x: ASTValueNode): + self.load(x.value) + + @dispatch + def load(self, x: ASTFunccallNode): + assert not (x.callargs.starargs or x.callkwargs.callkwargs or x.callkwargs.starkwargs) + if (x.callable.value.identifier == 'print'): + for i in x.callargs.callargs[:-1]: + self.load(i) + self.code += b'eq[32]eigteq' + self.stack.pop() + if (x.callargs.callargs): + self.load(x.callargs.callargs[-1]) + self.stack.pop() + else: self.code += b'[]' + self.code += b'eo' + self.stack.append(None) + return + for i in x.callargs.callargs[::-1]: + self.load(i) + self.stack.pop() + self.code += self.functions[f"{x.callable.value.identifier}__{self.ns.signatures[x.callable.value.identifier].call.index(CallArguments.build(x, self.ns))}" if (isinstance(x.callable, ASTValueNode) and isinstance(x.callable.value, ASTIdentifierNode) and x.callable.value.identifier in self.ns.signatures) else x.callable.value.identifier].code#.join((b'{', b'}')) + + @dispatch + def load(self, x: ASTBinaryExprNode): + self.load(x.lvalue) + self.load(x.rvalue) + self.code += self.binopmap[x.operator.operator] + self.stack.pop() + self.stack.pop() + self.stack.append(None) + +class GibberishCompiler(Compiler): + ext = '.gib' + + @classmethod + def compile_ast(cls, ast, ns, *, filename): + instrs = Instrs(ns=ns) + instrs.add(ast) + code = bytes(instrs.code) + dlog("Code:\n"+code.decode()) + return code + +# by Sdore, 2019 diff --git a/compilers/pyssembly.py b/compilers/pyssembly.py deleted file mode 100644 index be6bf74..0000000 --- a/compilers/pyssembly.py +++ /dev/null @@ -1,278 +0,0 @@ -#!/usr/bin/python3 -# Slang Pyssembly compiler target - -import pyssembly -from . import * -from Slang.ast import * -from utils import * - -class Instrs: - unopmap = { - '+': 'POS', - '-': 'NEG', - '!': 'NOT', - '~': 'INV', - 'not': 'NOT', - } - - binopmap = { - '+': 'ADD', - '-': 'SUB', - '*': 'MUL', - '/': 'DIV', - '//': 'FLOORDIV', - '%': 'MOD', - '**': 'POW', - '<<': 'LSHIFT', - '>>': 'RSHIFT', - '&': 'AND', - '|': 'OR', - '^': 'XOR', - 'and': 'AND', - 'or': 'OR', - } - - @init_defaults - def __init__(self, *, name, ns, filename, argdefs=()): - self.name, self.ns, self.filename = name, ns, filename - self.instrs = list() - self.consts = list() - self.argnames = list() - for i in argdefs: - if (i.modifier is not None): raise SlCompilationError("argument modifiers are not supported yet", i.modifier) - self.argnames.append(i.name.identifier) - self.cellvars = self.argnames.copy() - self.srclnotab = list() - - def compile(self): - return pyssembly.Code('\n'.join(self.instrs), name=self.name, filename=self.filename.strip('"'), srclnotab=self.srclnotab, consts=self.consts, argnames=self.argnames) - - @dispatch - def add(self, x: ASTRootNode): - self.add(x.code) - - @dispatch - def add(self, x: ASTCodeNode): - lastln = int() - for i in x.nodes: - self.srclnotab.append(i.lineno-lastln) - lastln = i.lineno - l = len(self.instrs) - self.add(i) - self.srclnotab += [0]*(len(self.instrs)-l-1) - - @dispatch - def add(self, x: ASTValueNode): - self.add(x.value) - - @dispatch - def add(self, x: ASTVardefNode): - if (x.value is not None): - self.load(x.value) - self.store(x.name) - - @dispatch - def add(self, x: ASTAssignmentNode): - if (x.inplace_operator is not None): self.instrs.append(f"LOAD ({x.name})") - self.load(x.value) - if (x.inplace_operator is not None): self.instrs.append(f"INP_{self.binopmap[x.inplace_operator.operator]}") - self.store(x.name) - - @dispatch - def add(self, x: ASTFunccallNode): - self.load(x) - self.instrs.append("POP") - - @dispatch - def add(self, x: ASTBlockNode): - self.add(x.code) - - @dispatch - def add(self, x: ASTFuncdefNode): - code_ns = self.ns.derive(x.name.identifier) - name = f"{x.name.identifier}__{self.ns.signatures[x.name.identifier].call.index(CallArguments(args=tuple(Signature.build(i, code_ns) for i in x.argdefs)))}" - f_instrs = Instrs(name=f"{self.name}.{name}", ns=code_ns, filename=self.filename, argdefs=x.argdefs) - f_instrs.add(x.code) - #dlog(f"{x.__fsig__()} instrs:\n"+'\n'.join(f_instrs.instrs)+'\n') - self.consts.append(f_instrs.compile().to_code()) - self.instrs += [ - f"LOAD {len(self.consts)-1}", - f"LOAD ('{self.name}.{name}')", - f"MAKE_FUNCTION 0", # TODO: flags - ] - self.store(name) - - @dispatch - def add(self, x: ASTKeywordExprNode): - if (x.keyword.keyword == 'import'): - ns, _, name = x.value.identifier.partition('::') - assert ns == 'py' - self.instrs += [ - "LOAD (0)", # TODO - "LOAD (None)", # TODO - ] - self.instrs.append(f"IMPORT_NAME ({name})") - self.store(name) - elif (x.keyword.keyword == 'return'): - self.load(x.value) - self.instrs.append("RET") - else: raise NotImplementedError(x.keyword) - - @dispatch - def add(self, x: ASTConditionalNode): - self.load(x.condition) - self.instrs.append("JPOPF :else") - self.add(x.code) - self.instrs += [ - "JUMPF :end", - ":else", - ":end", - ] - - @dispatch - def add(self, x: ASTForLoopNode): - self.load(x.iterable) - #self.cellvars.append(x.name.identifier) # TODO FIXME - self.instrs += [ - "SETUP_LOOP :end", - "ITER", - ":for", - "FOR :else", - ] - self.store(x.name) - self.add(x.code) - self.instrs += [ - "JUMPA :for", - ":else", - "POP_BLOCK", - ":end", - ] - - @dispatch - def add(self, x: ASTWhileLoopNode): - self.instrs += [ - "SETUP_LOOP :end", - ":while", - ] - self.load(x.condition) - self.instrs.append("JPOPF :else") - self.add(x.code) - self.instrs += [ - "JUMPA :while", - ":else", - "POP_BLOCK", - ":end", - ] - - @dispatch - def add(self, x: ASTElseClauseNode): - assert (self.instrs[-1] == ":end") - popped = [self.instrs.pop()] - if (self.instrs[-1] == "POP_BLOCK"): popped.append(self.instrs.pop()) - self.add(x.code) - self.instrs += reversed(popped) - - @dispatch - def load(self, x: ASTLiteralNode): - self.instrs.append(f"LOAD ({x.literal})") - - @dispatch - def load(self, x: ASTIdentifierNode): - self.load(x.identifier) - - @dispatch - def load(self, x: ASTValueNode): - self.load(x.value) - - @dispatch - def load(self, x: ASTFunccallNode): - if (isinstance(x.callable, ASTValueNode) and isinstance(x.callable.value, ASTIdentifierNode) and x.callable.value.identifier in self.ns.signatures): - self.load(f"{x.callable.value.identifier}__{self.ns.signatures[x.callable.value.identifier].call.index(CallArguments.build(x, self.ns))}") - else: self.load(x.callable) - n = int() - - for i in x.callargs.callargs: - self.load(i) - n += 1 - if (x.callargs.starargs): - if (n): - self.instrs.append(f"BUILD_TUPLE {n}") - n = 1 - for i in x.callargs.starargs: - self.load(i) - n += 1 - self.instrs.append(f"BUILD_TUPLE_UNPACK_WITH_CALL {n}") - n = 0 - - for i in x.callkwargs.callkwargs: - self.load(f"'{i[0]}'") - self.load(i[1]) - n += 1 - if (n and (x.callargs.starargs or x.callkwargs.starkwargs)): - self.instrs.append(f"BUILD_MAP {n}") - n = 1 - if (x.callkwargs.starkwargs): - for i in x.callkwargs.starkwargs: - self.load(i) - n += 1 - self.instrs.append(f"BUILD_MAP_UNPACK_WITH_CALL {n}") - n = 1 - - self.instrs.append(f"CALL{'EX' if (x.callargs.starargs or x.callkwargs.starkwargs) else 'KW' if (x.callkwargs.callkwargs) else ''} {n}") - - @dispatch - def load(self, x: ASTAttrgetNode): - self.load(x.value) - assert x.optype.special == '.' # TODO - self.instrs.append(f"GETATTR ({x.attr})") - - @dispatch - def load(self, x: ASTUnaryExprNode): - self.load(x.value) - self.instrs.append(self.unopmap[x.operator.operator]) - - @dispatch - def load(self, x: ASTBinaryExprNode): - self.load(x.lvalue) - char = isinstance(Signature.build(x.lvalue, self.ns), stdlib.char) - if (char): self.instrs.append("CALL (ord) 1") - if (x.operator.operator == 'xor'): self.instrs.append("BOOL") - self.load(x.rvalue) - if (char and isinstance(Signature.build(x.rvalue, self.ns), stdlib.char)): self.instrs.append("CALL (ord) 1") - if (x.operator.operator == 'xor'): self.instrs.append("BOOL") - if (x.operator.operator == 'to'): self.instrs.append("CALL (range) 2") - else: self.instrs.append(self.binopmap[x.operator.operator]) - if (x.operator.operator == 'xor'): self.instrs.append("BOOL") - if (char and x.operator.operator not in keyword_operators): self.instrs.append("CALL (chr) 1") - - @dispatch - def load(self, x: ASTItemgetNode): - self.load(x.value) - self.load(x.key) - self.instrs.append("SUBSCR") - - @dispatch - def load(self, x: str): - self.instrs.append(f"LOAD {f'${x}' if (x in self.cellvars) else f'({x})'}") - - @dispatch - def store(self, x: ASTIdentifierNode): - self.store(x.identifier) - - @dispatch - def store(self, x: str): - self.instrs.append(f"STORE {f'${x}' if (x in self.cellvars) else f'({x})'}") - -class PyssemblyCompiler(Compiler): - @classmethod - def compile_ast(cls, ast, ns, *, filename): - instrs = Instrs(name='', ns=ns, filename=filename) - instrs.add(ast) - #dlog("Instrs:\n"+'\n'.join(instrs.instrs)+'\n') - code = instrs.compile().to_code() - #dis.show_code(code) - #dis.dis(code) - #print() - return code - -# by Sdore, 2019 diff --git a/compilers/pyssembly/__init__.py b/compilers/pyssembly/__init__.py new file mode 100644 index 0000000..f0447ef --- /dev/null +++ b/compilers/pyssembly/__init__.py @@ -0,0 +1,486 @@ +#!/usr/bin/python3 +# Slang Pyssembly compiler target + +import pyssembly +from . import std +from .. import * +from ...ast import * +from utils import * + +class Instrs: + unopmap = { + '+': 'POS', + '-': 'NEG', + '!': 'NOT', + '~': 'INV', + 'not': 'NOT', + } + + binopmap = { + '+': 'ADD', + '-': 'SUB', + '*': 'MUL', + '/': 'DIV', + '//': 'IDIV', + '%': 'MOD', + '**': 'POW', + '<<': 'LSHIFT', + '>>': 'RSHIFT', + '&': 'AND', + '|': 'OR', + '^': 'XOR', + 'and': 'AND', + 'or': 'OR', + } + + _self_argdef = ASTArgdefNode(Class, ASTIdentifierNode('', lineno=None, offset=None), None, None, lineno=None, offset=None) + + def _class_init(self, *args, **kwargs): + getattr(self, '')() + getattr(self, f"")(*args, **kwargs) + _class_init = _class_init.__code__.replace(co_name='') + + @init_defaults + def __init__(self, *, name, ns, filename, argdefs=(), lastnodens: lambda: [None, None], firstlineno=0): + self.name, self.ns, self.filename, self.lastnodens, self.firstlineno = name, ns, filename, lastnodens, firstlineno + self.instrs = list() + self.consts = list() + self.argnames = list() + for i in argdefs: + if (i.modifier is not None): raise NotImplementedError("argument modifiers are not supported yet") + self.argnames.append(i.name.identifier) + self.cellvars = self.argnames.copy() + self.srclnotab = list() + self.lastln = self.firstlineno + self._class_init = self._class_init.replace(co_filename=self.filename, co_consts=tuple(i.replace(co_filename=self.filename) if (isinstance(i, CodeType)) else i for i in self._class_init.co_consts)) + + def compile(self): + return pyssembly.Code('\n'.join(self.instrs), name=self.name, filename=self.filename.strip('"'), srclnotab=self.srclnotab, firstlineno=self.firstlineno, consts=self.consts, argnames=self.argnames) + + @dispatch + def add(self, x: ASTRootNode): + self.add(x.code) + + @dispatch + def add(self, x: ASTCodeNode): + for ii, i in enumerate(x.nodes): + assert (i.lineno >= self.lastln) + if (i.lineno != self.lastln): self.instrs.append(f"#line {i.lineno}") + self.lastln = i.lineno + self.lastnodens[0] = i + self.add(i) + + @dispatch + def add(self, x: ASTValueNode): + self.add(x.value) + + @dispatch + def add(self, x: ASTVardefNode): + typesig = Signature.build(x.type, self.ns) + if (x.value is not None): + self.load(x.value) + self.store(x.name) + elif (isinstance(typesig, Class)): + self.instrs += [ + "LOAD (object)", + "GETATTR (__new__)", + ] + self.load(x.type.type) + self.instrs += [ + "CALL 1", + "DUP", + "GETATTR <>", + "CALL", + "POP", + ] + self.store(x.name) + + @dispatch + def add(self, x: ASTAssignmentNode): + if (x.inplace_operator is not None): self.instrs.append(f"LOAD ({x.name})") + self.load(x.value) + if (x.inplace_operator is not None): self.instrs.append(f"IP{self.binopmap[x.inplace_operator.operator]}") + if (x.isattr): + self.instrs += [ + "LOAD $", + f"SETATTR ({x.name})", + ] + else: self.store(x.name) + + @dispatch + def add(self, x: ASTUnpackAssignmentNode): + assert (x.inplace_operator is None) # TODO + self.load(x.value) + self.instrs.append(f"UNPACK {len(x.names)}") + for name in x.names: + self.store(name) + + @dispatch + def add(self, x: ASTAttrsetNode): + assert (x.assignment.isattr) + if (x.assignment.inplace_operator is not None): + self.load(x.value) + self.instrs += [ + "DUP", + "GETATTR ({x.assignment.name})", + ] + self.load(x.assignment.value) + self.instrs += [ + f"IP{self.binopmap[x.assignment.inplace_operator.operator]}", + "ROT", + ] + else: + self.load(x.assignment.value) + self.load(x.value) + self.instrs.append(f"SETATTR ({x.assignment.name})") + + @dispatch + def add(self, x: ASTFunccallNode): + self.load(x) + self.instrs.append("POP") + + @dispatch + def add(self, x: ASTBlockNode): + self.add(x.code) + + @dispatch + def add(self, x: ASTFuncdefNode): + code_ns = self.ns.derive(x.name.identifier) + name = f"{x.name.identifier}({CallArguments(args=tuple(Signature.build(i, code_ns) for i in x.argdefs))})" + self.lastnodens[1] = code_ns + fname = f"{self.name}.<{x.__fsig__()}>" + f_instrs = Instrs(name=fname, ns=code_ns, filename=self.filename, argdefs=x.argdefs, lastnodens=self.lastnodens, firstlineno=x.lineno) + f_instrs.add(x.code) + #dlog(f"{fname} instrs:\n"+'\n'.join(f_instrs.instrs)+'\n') + self.consts.append(f_instrs.compile().to_code()) + self.lastnodens[1] = self.ns + self.instrs += [ + f"LOAD {len(self.consts)-1}", + f"LOAD ('{fname}')", + "MKFUNC 0", # TODO: flags + ] + self.store(name) + + @dispatch + def add(self, x: ASTClassdefNode): + code_ns = self.ns.derive(x.name.identifier) + name = x.name.identifier + self.lastnodens[1] = code_ns + cname = str(name) + c_instrs = Instrs(name=cname, ns=code_ns, filename=self.filename, lastnodens=self.lastnodens, firstlineno=x.lineno) + c_instrs.consts.append(self._class_init) + c_instrs.instrs += [ + f"LOAD {len(self.consts)-1}", + f"LOAD ('')", + "MKFUNC 0", # TODO: flags + ] + c_instrs.store('__init__') + c_instrs.add(x.code) + self.consts.append(c_instrs.compile().to_code()) + self.lastnodens[1] = self.ns + self.instrs += [ + "LOAD_BUILD_CLASS", + f"LOAD {len(self.consts)-1}", + f"LOAD ('{cname}')", + "MKFUNC 0", # TODO: flags? + f"LOAD ('{cname}')", + "CALL 2", + ] + self.store(name) + + @dispatch + def add(self, x: ASTKeywordExprNode): + if (x.keyword.keyword == 'import'): + m = re.fullmatch(r'(?:(?:(\w+):)?(?:([\w./]+)/)?([\w.]+):)?([\w*]+)', x.value.identifier) + assert (m is not None) + namespace, path, pkg, name = m.groups() + if (namespace is None): namespace = 'sl' + if (path is None): path = '.' + if (pkg is None): pkg = name + if (namespace == 'py'): + assert (path == '.') + self.instrs += [ + "LOAD (0)", # TODO + "LOAD (())", # TODO + f"IMPORT ({pkg})", + ] + if (name == '*'): self.instrs.append("IMPALL") + else: + if (name != pkg): self.instrs.append("IMPFROM") + self.store(name) + elif (namespace == 'sl'): + pkg = pkg.replace('.', '/') + filename = f"{os.path.join(path, pkg)}.sl" + src = open(filename, 'r').read() + tl = parse_string(src) + ast = build_ast(tl, filename) + optimize_ast(ast, validate_ast(ast)) + instrs = Instrs(name='', ns=validate_ast(ast), filename=filename) + instrs.add(ast) + code = instrs.compile().to_code() + # TODO + else: raise WTFException(namespace) + elif (x.keyword.keyword == 'return'): + self.load(x.value) + self.instrs.append("RET") + elif (x.keyword.keyword == 'delete'): + self.delete(x.value) + elif (x.keyword.keyword == 'break'): + self.instrs.append("JF :end") + else: raise NotImplementedError(x.keyword) + + @dispatch + def add(self, x: ASTKeywordDefNode): + name = x.name.identifier + if (x.keyword.keyword == 'main'): + code_ns = self.ns.derive(name) + self.lastnodens[1] = code_ns + f_instrs = Instrs(name=name, ns=code_ns, filename=self.filename, lastnodens=self.lastnodens, firstlineno=x.lineno) + f_instrs.add(x.code) + #dlog(f"{name} instrs:\n"+'\n'.join(f_instrs.instrs)+'\n') + self.consts.append(f_instrs.compile().to_code()) + self.lastnodens[1] = self.ns + self.instrs += [ + "LOAD (__name__)", + "LOAD ('__main__')", + "CMP (==)", + "JFP :nomain", + f"LOAD {len(self.consts)-1}", + f"LOAD ('{name}')", + "MKFUNC 0", + "CALL 0", + "POP", + ":nomain", + ] + elif (x.keyword.keyword == 'init'): + code_ns = self.ns.derive(name) + self.lastnodens[1] = code_ns + f_instrs = Instrs(name=name, argdefs=(self._self_argdef,), ns=code_ns, filename=self.filename, lastnodens=self.lastnodens, firstlineno=x.lineno) + f_instrs.add(x.code) + #dlog(f"{name} instrs:\n"+'\n'.join(f_instrs.instrs)+'\n') + self.consts.append(f_instrs.compile().to_code()) + self.lastnodens[1] = self.ns + self.instrs += [ + f"LOAD {len(self.consts)-1}", + f"LOAD ('{name}')", + "MKFUNC 0", + ] + self.store(name) + elif (x.keyword.keyword == 'constr'): + code_ns = self.ns.derive(name) + self.ns.define(x, redefine=True) + self.lastnodens[1] = code_ns + name = f"" + f_instrs = Instrs(name=name, ns=code_ns, filename=self.filename, argdefs=(self._self_argdef, *x.argdefs), lastnodens=self.lastnodens, firstlineno=x.lineno) + f_instrs.add(x.code) + #dlog(f"{name} instrs:\n"+'\n'.join(f_instrs.instrs)+'\n') + self.consts.append(f_instrs.compile().to_code()) + self.lastnodens[1] = self.ns + self.instrs += [ + f"LOAD {len(self.consts)-1}", + f"LOAD ('{name}')", + "MKFUNC 0", # TODO: flags + ] + self.store(name) + else: raise NotImplementedError(x.keyword) + + @dispatch + def add(self, x: ASTConditionalNode): + self.load(x.condition) + self.instrs.append("JFP :else") + self.add(x.code) + self.instrs += [ + "JF :end", + ":else", + ":end", + ] + + @dispatch + def add(self, x: ASTForLoopNode): + self.load(x.iterable) + #self.cellvars.append(x.name.identifier) # TODO FIXME + self.instrs += [ + "ITER", + ":for", + "FOR :else", + ] + self.store(x.name) + self.add(x.code) # TODO: stack effect (py38) + self.instrs += [ + "JA :for", + ":else", + ":end", + ] + + @dispatch + def add(self, x: ASTWhileLoopNode): + self.instrs += [ + ":while", + ] + self.load(x.condition) + self.instrs.append("JFP :else") + self.add(x.code) # TODO: stack effect (py38) + self.instrs += [ + "JA :while", + ":else", + ":end", + ] + + @dispatch + def add(self, x: ASTElseClauseNode): + ii = -1 - self.instrs[-1].startswith('#line') + assert (self.instrs.pop(ii) == ":end") + self.add(x.code) # TODO: stack effect (py38) + self.instrs.append(":end") + + @dispatch + def load(self, x: ASTLiteralNode): + self.instrs.append(f"LOAD ({x.literal})") + + @dispatch + def load(self, x: ASTIdentifierNode): + self.load(x.identifier) + + @dispatch + def load(self, x: ASTListNode): + for i in x.values: + self.load(i) + self.instrs.append(f"BUILD_LIST {len(x.values)}") + + @dispatch + def load(self, x: ASTTupleNode): + for i in x.values: + self.load(i) + self.instrs.append(f"BUILD_TUPLE {len(x.values)}") + + @dispatch + def load(self, x: ASTValueNode): + self.load(x.value) + + @dispatch + def load(self, x: ASTFunccallNode): + if (isinstance(x.callable, ASTValueNode) and isinstance(x.callable.value, ASTIdentifierNode) and x.callable.value.identifier in self.ns.signatures and not isinstance(self.ns.signatures[x.callable.value.identifier], Class)): + self.load(f"{x.callable.value.identifier}({CallArguments.build(x, self.ns)})") + else: self.load(x.callable) + n = int() + + for i in x.callargs.callargs: + self.load(i) + n += 1 + if (x.callargs.starargs): + if (n): + self.instrs.append(f"BUILD_TUPLE {n}") + n = 1 + for i in x.callargs.starargs: + self.load(i) + n += 1 + self.instrs.append(f"BUILD_TUPLE_UNPACK_WITH_CALL {n}") + n = 0 + + for i in x.callkwargs.callkwargs: + self.load(f"'{i[0]}'") + self.load(i[1]) + n += 1 + if (n and (x.callargs.starargs or x.callkwargs.starkwargs)): + self.instrs.append(f"BUILD_MAP {n}") + n = 1 + if (x.callkwargs.starkwargs): + for i in x.callkwargs.starkwargs: + self.load(i) + n += 1 + self.instrs.append(f"BUILD_MAP_UNPACK_WITH_CALL {n}") + n = 1 + + self.instrs.append(f"CALL{'EX' if (x.callargs.starargs or x.callkwargs.starkwargs) else 'KW' if (x.callkwargs.callkwargs) else ''} {n}") + + @dispatch + def load(self, x: ASTAttrgetNode): + self.load(x.value) + assert (x.optype.special == '.') # TODO + self.instrs.append(f"GETATTR ({x.attr})") + + @dispatch + def load(self, x: ASTUnaryExprNode): + self.load(x.value) + self.instrs.append(self.unopmap[x.operator.operator]) + + @dispatch + def load(self, x: ASTBinaryExprNode): + self.load(x.lvalue) + char = isinstance(Signature.build(x.lvalue, self.ns), stdlib.char) + if (char): self.instrs.append("CALL (ord) 1") + if (x.operator.operator == 'xor'): self.instrs.append("BOOL") + self.load(x.rvalue) + if (char and isinstance(Signature.build(x.rvalue, self.ns), stdlib.char)): self.instrs.append("CALL (ord) 1") + if (x.operator.operator == 'xor'): self.instrs.append("BOOL") + if (x.operator.operator == 'to'): self.instrs.append("CALL (range) 2") + else: self.instrs.append(f"CMP ({x.operator.operator})" if (x.operator.operator in dis.cmp_op) else self.binopmap[x.operator.operator]) + if (x.operator.operator == 'xor'): self.instrs.append("BOOL") + if (char and x.operator.operator not in logical_operators): self.instrs.append("CALL (chr) 1") + + @dispatch + def load(self, x: ASTItemgetNode): + self.load(x.value) + self.load(x.key) + self.instrs.append("SUBSCR") + + @dispatch + def load(self, x: str): + self.instrs.append(f"LOAD {f'${x}' if (x in self.cellvars) else f'<{x}>'}") + + @dispatch + def load(self, x): + self.instrs.append(f"LOAD_CONST ({repr(x)})") + + @dispatch + def store(self, x: ASTIdentifierNode): + self.store(x.identifier) + + @dispatch + def store(self, x: str): + self.instrs.append(f"STORE {f'${x}' if (x in self.cellvars) else f'<{x}>'}") + + @dispatch + def delete(self, x: ASTIdentifierNode): + self.delete(x.identifier) + + @dispatch + def delete(self, x: str): + self.instrs.append(f"DELETE {f'${x}' if (x in self.cellvars) else f'<{x}>'}") + +class PyssemblyCompiler(Compiler): + ext = '.pyc' + + @classmethod + def compile_ast(cls, ast, ns, *, filename): + instrs = Instrs(name='', ns=ns, filename=filename) + + instrs.consts.append(compile(open(std.__file__).read(), '', 'exec')) + instrs.instrs += [ + f"LOAD {len(instrs.consts)-1}", + "LOAD ('')", + "MKFUNC 0", # TODO? + "CALL", + ] + + try: instrs.add(ast) + except Exception as ex: raise SlCompilationError('Compilation error', instrs.lastnodens[0], scope=instrs.lastnodens[1].scope) from ex + + #dlog("Instrs:\n"+'\n'.join(instrs.instrs)+'\n') + + try: + code = instrs.compile().to_code() + #dis.show_code(code); dis.dis(code); print() + code = pyssembly.asm(code) + except pyssembly.PyssemblyError as ex: + print('Error:', ex) + try: code = ex.code.to_code() + except pyssembly.PyssemblyError: pass + else: + print("\nHere is full pyssembly code 'til the errorneous line:\n") + dis.dis(code) + raise SlCompilationError('Pyssembly error', ast, scope=instrs.ns) from ex + + return code + +# by Sdore, 2020 diff --git a/compilers/pyssembly/std.py b/compilers/pyssembly/std.py new file mode 100644 index 0000000..b666fdc --- /dev/null +++ b/compilers/pyssembly/std.py @@ -0,0 +1,2 @@ +class stdio: + println = print diff --git a/compilers/pyssembly/typeshed b/compilers/pyssembly/typeshed new file mode 160000 index 0000000..b44cd29 --- /dev/null +++ b/compilers/pyssembly/typeshed @@ -0,0 +1 @@ +Subproject commit b44cd294c4f6cdb66fdd6c13aebabb10855e7dc2 diff --git a/compilers/sbc.py b/compilers/sbc.py index ec2839b..ba341eb 100644 --- a/compilers/sbc.py +++ b/compilers/sbc.py @@ -6,7 +6,11 @@ from Slang.ast import * from utils import * NOP = 0x00 -RET = 0x01 +END = 0x01 +POP = 0x02 +RET = 0x03 +BLTIN = 0x04 +CODE = 0x05 POS = 0x10 NEG = 0x11 @@ -24,15 +28,28 @@ ADD = 0x20 SUB = 0x21 MUL = 0x22 DIV = 0x23 -FLRDIV = 0x24 +IDIV = 0x24 MOD = 0x25 POW = 0x26 -SHL = 0x27 -SHR = 0x28 +LSH = 0x27 +RSH = 0x28 AND = 0x29 OR = 0x2a XOR = 0x2b +EQ = 0x30 +NE = 0x31 +LT = 0x32 +GT = 0x33 +LE = 0x34 +GE = 0x35 +IS = 0x36 +ISNOT = 0x37 + +IF = 0x40 +ELSE = 0x41 +EXEC = 0x42 + ALLOC = 0xa0 EXTEND = 0xa1 CONST = 0xa2 @@ -40,26 +57,81 @@ JUMPF = 0xa3 JUMPB = 0xa4 SCPGET = 0xa5 SCPSET = 0xa6 +CALL = 0xa7 HASARG = 0xa0 +def readVarInt(s): + r = int() + i = int() + while (True): + b = s.recv(1)[0] + r |= (b & (1 << 7)-1) << (7*i) + if (not b & (1 << 7)): break + i += 1 + return r + +def writeVarInt(v): + assert v >= 0 + r = bytearray() + while (True): + c = v & (1 << 7)-1 + v >>= 7 + if (v): c |= (1 << 7) + r.append(c) + if (not v): break + return bytes(r) + class Instrs: unops = '+-!~' binops = (*'+-*/%', '**', '<<', '>>', '&', '|', '^') + unopmap = { + '+': POS, + '-': NEG, + '!': NOT, + '~': INV, + 'not': NOT, + } + binopmap = { + '+': ADD, + '-': SUB, + '*': MUL, + '/': DIV, + '//': IDIV, + '%': MOD, + '**': POW, + '<<': LSH, + '>>': RSH, + '&': AND, + '|': OR, + '^': XOR, + + '==': EQ, + '!=': NE, + '<': LT, + '>': GT, + '<=': LE, + '>=': GE, + 'is': IS, + 'is not': ISNOT, + } @init_defaults - def __init__(self, *, name, ns, filename, argdefs=()): - self.ns = ns + def __init__(self, *, name, ns, filename, scpcells: indexset): + self.name, self.ns, self.filename, self.scpcells = name, ns, filename, scpcells self.instrs = bytearray() def compile(self): - raise TODO + return bytes(self.instrs) @dispatch - def add(self, opcode: int, oparg: int = None): + def add(self, opcode: lambda x: isinstance(x, int) and x < HASARG): self.instrs.append(opcode) - if (opcode >= HASARG): self.instrs.append(oparg) - else: assert oparg is None + + @dispatch + def add(self, opcode: lambda x: isinstance(x, int) and x >= HASARG, oparg: int): + self.instrs.append(opcode) + self.instrs.append(oparg) @dispatch def add(self, x: ASTRootNode): @@ -67,7 +139,6 @@ class Instrs: @dispatch def add(self, x: ASTCodeNode): - lastln = int() for i in x.nodes: self.add(i) @@ -83,15 +154,14 @@ class Instrs: @dispatch def add(self, x: ASTAssignmentNode): - if (x.inplace_operator is not None): self.instrs.append(f"LOAD ({x.name})") + if (x.inplace_operator is not None): raise NotImplementedError() self.load(x.value) - if (x.inplace_operator is not None): self.instrs.append(f"INP_{self.binopmap[x.inplace_operator.operator]}") self.store(x.name) @dispatch def add(self, x: ASTFunccallNode): self.load(x) - self.instrs.append("POP") + self.add(POP) @dispatch def add(self, x: ASTBlockNode): @@ -101,46 +171,52 @@ class Instrs: def add(self, x: ASTFuncdefNode): code_ns = self.ns.derive(x.name.identifier) name = f"{x.name.identifier}__{self.ns.signatures[x.name.identifier].call.index(CallArguments(args=tuple(Signature.build(i, code_ns) for i in x.argdefs)))}" - f_instrs = Instrs(name=f"{self.name}.{name}", ns=code_ns, filename=self.filename, argdefs=x.argdefs) + fname = f"{self.name}.<{x.__fsig__()}>" + self.scpcells[name] + f_instrs = Instrs(name=fname, ns=code_ns, filename=self.filename, scpcells=self.scpcells.copy()) + for i in x.argdefs: + f_instrs.scpcells[i.name.identifier] f_instrs.add(x.code) - #dlog(f"{x.__fsig__()} instrs:\n"+'\n'.join(f_instrs.instrs)+'\n') - self.consts.append(f_instrs.compile().to_code()) - self.instrs += [ - f"LOAD {len(self.consts)-1}", - f"LOAD ('{self.name}.{name}')", - f"MAKE_FUNCTION 0", # TODO: flags - ] + self.add(CODE) + self.instrs += f_instrs.instrs + self.add(END) + self.store(name+'.len') self.store(name) @dispatch def add(self, x: ASTKeywordExprNode): if (x.keyword.keyword == 'import'): ns, _, name = x.value.identifier.partition('::') - assert ns == 'py' - self.instrs += [ - "LOAD (0)", # TODO - "LOAD (None)", # TODO - ] - self.instrs.append(f"IMPORT_NAME ({name})") + raise TODO self.store(name) elif (x.keyword.keyword == 'return'): self.load(x.value) - self.instrs.append("RET") + self.add(RET) + else: raise NotImplementedError(x.keyword) + + @dispatch + def add(self, x: ASTKeywordDefNode): + if (x.keyword.keyword == 'main'): + name = '
' + code_ns = self.ns.derive(name) + f_instrs = Instrs(name=name, ns=code_ns, filename=self.filename) + f_instrs.add(x.code) + self.add(CODE) + self.instrs += f_instrs.instrs + self.add(END) + self.add(EXEC) + self.add(POP) else: raise NotImplementedError(x.keyword) @dispatch def add(self, x: ASTConditionalNode): self.load(x.condition) - self.instrs.append("JPOPF :else") + self.add(IF) self.add(x.code) - self.instrs += [ - "JUMPF :end", - ":else", - ":end", - ] + self.add(END) - @dispatch - def add(self, x: ASTForLoopNode): + #@dispatch + def add_(self, x: ASTForLoopNode): self.load(x.iterable) #self.cellvars.append(x.name.identifier) # TODO FIXME self.instrs += [ @@ -158,8 +234,8 @@ class Instrs: ":end", ] - @dispatch - def add(self, x: ASTWhileLoopNode): + #@dispatch + def add_(self, x: ASTWhileLoopNode): self.instrs += [ "SETUP_LOOP :end", ":while", @@ -174,8 +250,8 @@ class Instrs: ":end", ] - @dispatch - def add(self, x: ASTElseClauseNode): + #@dispatch + def add_(self, x: ASTElseClauseNode): assert (self.instrs[-1] == ":end") popped = [self.instrs.pop()] if (self.instrs[-1] == "POP_BLOCK"): popped.append(self.instrs.pop()) @@ -184,7 +260,16 @@ class Instrs: @dispatch def load(self, x: ASTLiteralNode): - self.instrs.append(f"LOAD ({x.literal})") + sig = Signature.build(x, ns=self.ns) + if (hasattr(sig, 'fmt')): + v = struct.pack(sig.fmt, int(x.literal)) + elif (isinstance(sig, stdlib.int)): + v = writeVarInt(int(x.literal)) + elif (isinstance(sig, stdlib.str)): + v = x.literal.encode('utf8') + else: raise NotImplementedError(sig) + self.add(CONST, len(v)) + self.instrs += v @dispatch def load(self, x: ASTIdentifierNode): @@ -196,42 +281,29 @@ class Instrs: @dispatch def load(self, x: ASTFunccallNode): - if (isinstance(x.callable, ASTValueNode) and isinstance(x.callable.value, ASTIdentifierNode) and x.callable.value.identifier in self.ns.signatures): - self.load(f"{x.callable.value.identifier}__{self.ns.signatures[x.callable.value.identifier].call.index(CallArguments.build(x, self.ns))}") - else: self.load(x.callable) - n = int() + nargs = int() for i in x.callargs.callargs: self.load(i) - n += 1 - if (x.callargs.starargs): - if (n): - self.instrs.append(f"BUILD_TUPLE {n}") - n = 1 - for i in x.callargs.starargs: - self.load(i) - n += 1 - self.instrs.append(f"BUILD_TUPLE_UNPACK_WITH_CALL {n}") - n = 0 + nargs += 1 + if (x.callargs.starargs): raise NotImplementedError() - for i in x.callkwargs.callkwargs: - self.load(f"'{i[0]}'") - self.load(i[1]) - n += 1 - if (n and (x.callargs.starargs or x.callkwargs.starkwargs)): - self.instrs.append(f"BUILD_MAP {n}") - n = 1 - if (x.callkwargs.starkwargs): - for i in x.callkwargs.starkwargs: - self.load(i) - n += 1 - self.instrs.append(f"BUILD_MAP_UNPACK_WITH_CALL {n}") - n = 1 + if (x.callkwargs.callkwargs): raise NotImplementedError() + if (x.callkwargs.starkwargs): raise NotImplementedError() - self.instrs.append(f"CALL{'EX' if (x.callargs.starargs or x.callkwargs.starkwargs) else 'KW' if (x.callkwargs.callkwargs) else ''} {n}") + if (isinstance(x.callable, ASTValueNode) and isinstance(x.callable.value, ASTIdentifierNode) and x.callable.value.identifier in self.ns.signatures): + name = f"{x.callable.value.identifier}__{self.ns.signatures[x.callable.value.identifier].call.index(CallArguments.build(x, self.ns))}" + self.load(name) + self.load(name+'.len') + self.add(EXEC) + else: # builtin + #self.add(ITOA) # TODO FIXME cast + self.add(BLTIN) + self.instrs += x.callable.value.identifier.encode('ascii')+b'\0' + self.add(CALL, nargs) - @dispatch - def load(self, x: ASTAttrgetNode): + #@dispatch + def load_(self, x: ASTAttrgetNode): self.load(x.value) assert x.optype.special == '.' # TODO self.instrs.append(f"GETATTR ({x.attr})") @@ -239,31 +311,24 @@ class Instrs: @dispatch def load(self, x: ASTUnaryExprNode): self.load(x.value) - self.instrs.append(self.unopmap[x.operator.operator]) + self.add(self.unopmap[x.operator.operator]) @dispatch def load(self, x: ASTBinaryExprNode): self.load(x.lvalue) - char = isinstance(Signature.build(x.lvalue, self.ns), stdlib.char) - if (char): self.instrs.append("CALL (ord) 1") - if (x.operator.operator == 'xor'): self.instrs.append("BOOL") self.load(x.rvalue) - if (char and isinstance(Signature.build(x.rvalue, self.ns), stdlib.char)): self.instrs.append("CALL (ord) 1") - if (x.operator.operator == 'xor'): self.instrs.append("BOOL") - if (x.operator.operator == 'to'): self.instrs.append("CALL (range) 2") - else: self.instrs.append(self.binopmap[x.operator.operator]) - if (x.operator.operator == 'xor'): self.instrs.append("BOOL") - if (char and x.operator.operator not in keyword_operators): self.instrs.append("CALL (chr) 1") + if (x.operator.operator == 'to'): raise NotImplementedError() + else: self.add(self.binopmap[x.operator.operator]) - @dispatch - def load(self, x: ASTItemgetNode): + #@dispatch + def load_(self, x: ASTItemgetNode): self.load(x.value) self.load(x.key) self.instrs.append("SUBSCR") @dispatch def load(self, x: str): - self.instrs.append(f"LOAD {f'${x}' if (x in self.cellvars) else f'({x})'}") + self.add(SCPGET, self.scpcells[x]) @dispatch def store(self, x: ASTIdentifierNode): @@ -271,18 +336,16 @@ class Instrs: @dispatch def store(self, x: str): - self.instrs.append(f"STORE {f'${x}' if (x in self.cellvars) else f'({x})'}") + self.add(SCPSET, self.scpcells[x]) + +class SBCCompiler(Compiler): + ext = '.sbc' -class PyssemblyCompiler(Compiler): @classmethod def compile_ast(cls, ast, ns, *, filename): instrs = Instrs(name='', ns=ns, filename=filename) instrs.add(ast) - #dlog("Instrs:\n"+'\n'.join(instrs.instrs)+'\n') - code = instrs.compile().to_code() - #dis.show_code(code) - #dis.dis(code) - #print() + code = instrs.compile() return code # by Sdore, 2019 diff --git a/lexer.py b/lexer.py index 55709f7..589f34e 100644 --- a/lexer.py +++ b/lexer.py @@ -13,21 +13,30 @@ def read_token(src, *, lineno, offset, lineoff): if (isinstance(r, int) and r <= 0): length = max(length, -r); continue n, s = r if (isinstance(r, tuple)) else (r, src[:r]) return (offset+n, Token(ii, s, lineno=lineno, offset=offset+lineoff)) - else: raise SlSyntaxError("Invalid token", line, lineno=lineno, offset=offset+lineoff, length=length) + else: raise SlSyntaxError("Invalid token", line, lineno=lineno, offset=offset+lineoff, length=length+l) def parse_expr(src, *, lineno=1, lineoff=0): r = list() + lines = src.count('\n') offset = int() + continueln = False while (True): - offset, tok = read_token(src, lineno=lineno, offset=offset, lineoff=lineoff) - if (tok is None): break + offset, tok = read_token(src, lineno=lines-src[offset:].count('\n')+lineno, offset=offset, lineoff=lineoff) + if (tok is None): + if (not continueln): break + continueln = False + offset += 1 + lineoff = -offset + continue + elif (continueln and tok.token[0] != '#'): raise SlSyntaxError("Expected newline or comment after line continuation", src, lineno=lines-src[offset:].count('\n')+lineno, offset=tok.offset, length=tok.length) r.append(tok) + if (tok.token[0] != '#'): continueln = (tok.token == '\\' and tok.offset) return offset, r -def parse_string(src): +def parse_string(src, lnooff=0): src = src.rstrip() tl = list() - lines = src.count('\n') + lines = src.count('\n')+lnooff lineoff = int() while (src): offset, r = parse_expr(src, lineno=lines-src.count('\n')+1, lineoff=lineoff) @@ -39,4 +48,4 @@ def parse_string(src): tl.append(r) return tl -# by Sdore, 2019 +# by Sdore, 2020 diff --git a/repl.py b/repl.py index 1965c54..91c8aeb 100644 --- a/repl.py +++ b/repl.py @@ -4,13 +4,15 @@ import readline from .ast import * from .lexer import * -from utils import * +from utils.nolog import * @dispatch def execute_node(node: ASTCodeNode, ns): + r = None for i in node.nodes: + if (isinstance(i, ASTElseClauseNode) and r is not None): continue r = execute_node(i, ns) - if (r is not None): + if (r not in (None, ...)): ns.values['_'] = r print(repr(r)) else: return @@ -20,13 +22,17 @@ def execute_node(node: ASTCodeNode, ns): def execute_node(node: ASTVardefNode, ns): ns.values[node.name] = node.value +@dispatch +def execute_node(node: ASTBlockNode, ns): + return execute_node(node.code, ns) + @dispatch def execute_node(node: ASTFuncdefNode, ns): ns.values[node.name.identifier] = (node.argdefs, node.code) @dispatch def execute_node(node: ASTAssignmentNode, ns): - ns.values[node.name] = node.value + ns.values[node.name] = execute_node(ASTBinaryExprNode(node.name, node.inplace_operator, node.value, lineno=node.lineno, offset=node.offset), ns) if (node.inplace_operator is not None) else node.value @dispatch def execute_node(node: ASTFunccallNode, ns): @@ -42,48 +48,161 @@ def execute_node(node: ASTValueNode, ns): @dispatch def execute_node(node: ASTIdentifierNode, ns): - return ns.values[node] + try: return ns.values[node] + except KeyError: raise SlValidationError(f"{node} is not initialized", node, scope=ns.scope) @dispatch def execute_node(node: ASTLiteralNode, ns): - return eval(str(node.literal)) + return eval(node.literal) if (isinstance(node.literal, str)) else node.literal + +@dispatch +def execute_node(node: ASTListNode, ns): + return node.values @dispatch def execute_node(node: ASTKeywordExprNode, ns): - if (node.keyword.keyword == 'return'): return execute_node(node.value, ns) - raise NotImplementedError(node.keyword) + #if (node.keyword.keyword == 'return'): return execute_node(node.value, ns) # TODO FIXME??? + #el + if (node.keyword.keyword == 'delete'): ns.delete(node.value) + else: raise NotImplementedError(node.keyword) + +@dispatch +def execute_node(node: ASTConditionalNode, ns): + if (execute_node(node.condition, ns)): + execute_node(node.code, ns) + else: return + return ... + +@dispatch +def execute_node(node: ASTForLoopNode, ns): + for i in execute_node(node.iterable, ns): + execute_node(node.code, ns) + else: return + return ... @dispatch def execute_node(node: ASTUnaryExprNode, ns): - return eval(f"{node.operator.operator} {execute_node(node.value, ns)}") + value = execute_node(node.value, ns) + return eval(f"{node.operator.operator} value") @dispatch def execute_node(node: ASTBinaryExprNode, ns): - return eval(f"{execute_node(node.lvalue, ns)} {node.operator.operator} {execute_node(node.rvalue, ns)}") + lvalue = execute_node(node.lvalue, ns) + rvalue = execute_node(node.rvalue, ns) + if (node.operator.operator == 'xor'): return eval("(lvalue and not rvalue) or (rvalue and not lvalue)") + elif (node.operator.operator == 'to'): return range(lvalue, rvalue) + else: return eval(f"lvalue {node.operator.operator} rvalue") + +class Completer: + def __init__(self, namespace): + self.namespace = namespace + + def complete(self, text, state): + if (state == 0): + if ('.' in text): self.matches = self.attr_matches(text) + else: self.matches = self.global_matches(text) + try: return self.matches[state] + except IndexError: return None + + def _callable_postfix(self, val, word): + if (isinstance(val, ASTCallableNode)): word += '(' + return word + + def global_matches(self, text): + matches = list() + seen = set() + n = len(text) + + for word in keywords: + if (word[:n] != text): continue + seen.add(word) + matches.append(word+' ') + + for word, val in self.namespace.values.items(): + if (word[:n] != text or word in seen): continue + seen.add(word) + matches.append(self._callable_postfix(val, word)) + + return matches + + def attr_matches(self, text): + m = re.match(r"(\w+(\.\w+)*)\.(\w*)", text) + if (m is None): return () + expr, attr = m.group(1, 3) + try: obj = self.namespace.values[expr] # TODO FIXME + except KeyError: return () + + words = set() # TODO FIXME attrs + + matches = list() + n = len(attr) + if (attr == ''): noprefix = '_' + elif (attr == '_'): noprefix = '__' + else: noprefix = None + + while (True): + for word in words: + if (word[:n] != attr or (noprefix and word[:n+1] == noprefix)): continue + match = f"{expr}.{word}" + try: val = getattr(obj, word) + except Exception: pass # Include even if attribute not set + else: match = self._callable_postfix(val, match) + matches.append(match) + if (matches or not noprefix): break + if (noprefix == '_'): noprefix = '__' + else: noprefix = None + + matches.sort() + return matches def repl(): ns = Namespace('') - #ns.values['print'] = print + ns.define(ASTIdentifierNode('_', lineno=None, offset=None), stdlib.Any()) + + completer = Completer(ns) + histfile = os.path.expanduser('~/.sli_history') + try: readline.read_history_file(histfile) + except FileNotFoundError: pass + for i in ( + 'set colored-completion-prefix on', + 'set enable-bracketed-paste on', + #'set horizontal-scroll-mode on', + 'set skip-completed-text on', + 'tab: complete', + ): readline.parse_and_bind(i) + readline.set_completer(completer.complete) + #readline.set_completion_display_matches_hook(completer.display) # TODO + l = list() tl = list() - while (True): - try: - l.append(input(f"\1\033[1;93m\2{'...' if (tl) else '>>>'}\1\033[0m\2 ")) - tll = parse_string(l[-1]) - if (not tll): continue - tl += tll - if (tl[-1][-1].token == '\\'): continue - if (len(tl) >= 2 and tl[-2][-1].token == '\\'): tl[-1] = tl[-2][:-1]+tl.pop() - if (tl[0][-1].token == '{' and tl[-1][-1].token != '}'): continue - ast = build_ast(tl, interactive=True) - validate_ast(ast, ns) - optimize_ast(ast, ns) - execute_node(ast.code, ns) - except (EOFError, KeyboardInterrupt): break - except (SlSyntaxError, SlValidationError) as ex: - ex.line = l[ex.lineno-1] - print(ex) - tl.clear() - l.clear() -# by Sdore, 2019 + try: + while (True): + try: + l.append(input(f"\1\033[1;93m\2{'...' if (tl) else '>>>'}\1\033[0m\2 ")) + tll = parse_string(l[-1], lnooff=len(l)-1) + if (not tll): l.pop(); continue + tl += tll + if (tl[-1][-1].token == '\\'): continue + #if (len(tl) >= 2 and tl[-2][-1].token == '\\'): tl[-1] = tl[-2][:-1]+tl.pop() # TODO FIXME?: [['a', '+', '\\'], 'b'] --> [['a', '+', 'b']] + if (tl[0][-1].token == '{' and tl[-1][-1].token != '}'): continue + ast = build_ast(tl, interactive=True) + validate_ast(ast, ns) + optimize_ast(ast, ns) + execute_node(ast.code, ns) + except KeyboardInterrupt: + buf = readline.get_line_buffer() + print(f"\r\033[2m^C{'> '+buf if (buf) else ' '}\033[0m") + except EOFError: + print(end='\r\033[K') + break + except (SlSyntaxError, SlValidationError) as ex: + ex.line = l[ex.lineno-1] + print(ex) + tl.clear() + l.clear() + finally: readline.write_history_file(histfile) + +if (__name__ == '__main__'): repl() + +# by Sdore, 2020 diff --git a/sl/Slang/Slang.sl b/sl/Slang/Slang.sl new file mode 100644 index 0000000..3341fe5 --- /dev/null +++ b/sl/Slang/Slang.sl @@ -0,0 +1,17 @@ +# Slang + +import lexer:* + +str src = "main {print('hello');}" + +main { + stdio.println("Source: {"+src+"}\n") + + list tl = parse_string(src) + stdio.println(tl) + stdio.println("Tokens:") + stdio.println(tl) + stdio.println("\n") +} + +# by Sdore, 2020 diff --git a/sl/Slang/lexer.sl b/sl/Slang/lexer.sl new file mode 100644 index 0000000..4d0659f --- /dev/null +++ b/sl/Slang/lexer.sl @@ -0,0 +1,74 @@ +# Slang lexer + +import tokens:* + +tuple read_token(str src, int lineno, int offset, int lineoff) { + tuple c = lstripcount(src#|[offset:]|#, whitespace) + int l = c[0] + src = c[1] + str line = src + offset += l + if not src or src[0] in '\n;' { + return (offset, None) + } + int length = 0 + int ii = -1 + for i in Token.types { + ii += 1 + auto r = globals['find_'+i.casefold()](src) or 0 + if r isof int and r <= 0 { + length = max(length, -r) + continue + } + tuple c + if r isof tuple { + c = r + } + else { + c = (r, src[:r]) + } + int n = c[0] + str s = c[1] + return (offset+n, Token(ii, s, lineno=lineno, offset=offset+lineoff)) + } #else raise SlSyntaxError("Invalid token", line, lineno=lineno, offset=offset+lineoff, length=length) + return (0, Token()) +} + +tuple parse_expr(str src, int lineno = 1, int lineoff = 0) { + list r = [Token] + int offset + while 1 { + offset, tok = read_token(src, lineno=lineno, offset=offset, lineoff=lineoff) + if tok is None { + break + } + r.append(tok) + } + return (offset, r) +} + +list parse_string(str src) { + src = src.rstrip() + list tl = [Token] + int lines = src.count('\n') + int lineoff = 0 + while src { + tuple c = parse_expr(src, lineno=lines-src.count('\n')+1, lineoff=lineoff) + int offset = c[0] + list r = c[1] + lineoff += offset + if offset < src.len { + if src[offset] == '\n' { + lineoff = 0 + } + else { + lineoff += 1 + } + } + src = src[offset+1:] + tl.append(r) + } + return tl +} + +# by Sdore, 2020 diff --git a/sl/Slang/tokens.sl b/sl/Slang/tokens.sl new file mode 100644 index 0000000..36e5abe --- /dev/null +++ b/sl/Slang/tokens.sl @@ -0,0 +1,49 @@ +# Slang tokens + +str whitespace = ' \t\r\v\f' + +tuple lstripcount(str s, str chars) { + int ii = -1 + char i + for i in s { + ii += 1 + if i not in chars { + break + } + } + else { + ii = 0 + } + return (ii, s#|[ii:]|#) +} + +class Token { + int type + str token + int lineno + int offset + + tuple types# = ('SPECIAL', 'OPERATOR', 'LITERAL', 'KEYWORD', 'IDENTIFIER') # order is also resolution order + + constr(int type, str token, int lineno, int offset) { + #.type, .token, .lineno, .offset = type, token, lineno, offset + } + + #|repr { + return "" + }|# + + #|eq { + #return super() == x or .token == x + }|# + + #|property typename { + #return .types[.type] + } + + property length { + #return .token.length + }|# +} + +# by Sdore, 2020 diff --git a/slang.lang b/slang.lang new file mode 100644 index 0000000..fa2a74d --- /dev/null +++ b/slang.lang @@ -0,0 +1,288 @@ + + + + + text/x-slang;application/x-slang + *.sl + # + #| + |# + + + +