Migration: Remove translator and directly interpret Python AST (#51)

.gitignore

@@ -107,4 +107,7 @@ dist
 .idea
 
 # TypeScript build files
-build/
+build/
+
+# Ignore personal test files
+*.local.test.ts

src/conductor/runner/types/PyEvaluator.ts

@@ -2,12 +2,11 @@
 // https://github.com/source-academy/conductor
 // Original author(s): Source Academy Team
 
-import { runInContext } from "../../../";
+import { PyRunInContext } from "../../../";
 import { Context } from "../../../cse-machine/context";
 import { BasicEvaluator } from "../BasicEvaluator";
 import { IRunnerPlugin } from "./IRunnerPlugin";
 import { IOptions } from "../../../";
-import { Finished } from "../../../types";
 
 const defaultContext = new Context();
 const defaultOptions: IOptions = {
@@ -28,12 +27,14 @@ export class PyEvaluator extends BasicEvaluator {
 
     async evaluateChunk(chunk: string): Promise<void> {
         try {
-            const result = await runInContext(
+            const result = await PyRunInContext(
                 chunk,       // Code
                 this.context,
                 this.options
             );
-            this.conductor.sendOutput(`${(result as Finished).representation.toString((result as Finished).value)}`);
+            if ('status' in result && result.status === 'finished') {
+                this.conductor.sendOutput(`${result.representation.toString(result.value)}`);
+            }
         } catch (error) {
             this.conductor.sendOutput(`Error: ${error instanceof Error ? error.message : error}`);
         }

src/cse-machine/py_closure.ts

@@ -0,0 +1,72 @@
+import { StmtNS, ExprNS } from '../ast-types'
+import { PyEnvironment, uniqueId } from './py_environment'
+import { PyContext } from './py_context'
+import { StatementSequence } from './py_types'
+import { PyControlItem } from './py_control'
+
+/**
+ * Represents a python closure, the class is a runtime representation of a function.
+ * Bundles the function's code (AST node) with environment in which its defined.
+ * When Closure is called, a new environment will be created whose parent is the 'Environment' captured
+ */
+export class PyClosure {
+  public readonly id: string
+  /** AST node for function, either a 'def' or 'lambda' */
+  public node: StmtNS.FunctionDef | ExprNS.Lambda;
+  /** Environment captures at time of function's definition, key for lexical scoping */
+  public environment: PyEnvironment;
+  public context: PyContext;
+  public readonly predefined: boolean;
+  public originalNode?: StmtNS.FunctionDef | ExprNS.Lambda;
+  /** Stores local variables for scope check */
+  public localVariables: Set<string>;
+
+  constructor(
+    node: StmtNS.FunctionDef | ExprNS.Lambda,
+    environment: PyEnvironment,
+    context: PyContext,
+    predefined: boolean = false,
+    localVariables: Set<string> = new Set()
+) {
+      this.id = uniqueId(context);
+      this.node = node;
+      this.environment = environment;
+      this.context = context;
+      this.predefined = predefined;
+      this.originalNode = node;
+      this.localVariables = localVariables;
+  }
+
+  /** 
+   * Creates closure for FunctionDef
+   */
+  static makeFromFunctionDef (
+    node: StmtNS.FunctionDef,
+    environment: PyEnvironment,
+    context: PyContext,
+    localVariables: Set<string>
+  ): PyClosure {
+    const closure = new PyClosure(node, environment, context, false, localVariables);
+    return closure;
+  }
+
+  /**
+   * Creates closure for Lambda
+   */
+  static makeFromLambda(
+       node: ExprNS.Lambda,
+       environment: PyEnvironment,
+       context: PyContext,
+       localVariables: Set<string>
+     ): PyClosure {
+       const closure = new PyClosure(node, environment, context, false, localVariables);
+       return closure;
+     }
+   }
+
+/**
+* Type guard to check if a control item is a StatementSequence.
+*/
+export const isStatementSequence = (node: PyControlItem): node is StatementSequence => {
+  return (node as StatementSequence).type === 'StatementSequence';
+};

src/cse-machine/py_context.ts

@@ -0,0 +1,170 @@
+import { Stash, Value } from './stash';
+import { PyControl, PyControlItem } from './py_control';
+import { createSimpleEnvironment, createProgramEnvironment, PyEnvironment } from './py_environment';
+import { CseError } from './error';
+import { Heap } from './heap';
+import { PyNode } from './py_types';
+import { NativeStorage } from '../types';
+import { StmtNS } from '../ast-types';
+
+export class PyContext {
+  public control: PyControl;
+  public stash: Stash;
+  public output: string = '';
+  //public environment: Environment;
+  public errors: CseError[] = [];
+
+  runtime: {
+    break: boolean
+    debuggerOn: boolean
+    isRunning: boolean
+    environmentTree: EnvTree
+    environments: PyEnvironment[]
+    nodes: PyNode[]
+    control: PyControl | null
+    stash: Stash | null
+    objectCount: number
+    envStepsTotal: number
+    breakpointSteps: number[]
+    changepointSteps: number[]
+  }
+
+  /**
+   * Used for storing the native context and other values
+   */
+  nativeStorage: NativeStorage
+
+  constructor(program?: StmtNS.Stmt, context?: PyContext) {
+    this.control = new PyControl(program);
+    this.stash = new Stash();
+    this.runtime = this.createEmptyRuntime();
+    //this.environment = createProgramEnvironment(context || this, false);
+    if (this.runtime.environments.length === 0) {
+      const globalEnvironment = this.createGlobalEnvironment()
+      this.runtime.environments.push(globalEnvironment)
+      this.runtime.environmentTree.insert(globalEnvironment)
+    }
+    this.nativeStorage = {
+      builtins: new Map<string, Value>(),
+      previousProgramsIdentifiers: new Set<string>(),
+      operators: new Map<string, (...operands: Value[]) => Value>(),
+      maxExecTime: 1000,
+      //evaller: null,
+      loadedModules: {},
+      loadedModuleTypes: {}
+    }
+  }
+
+  createGlobalEnvironment = (): PyEnvironment => ({
+    tail: null,
+    name: 'global',
+    head: {},
+    heap: new Heap(),
+    id: '-1'
+  })
+
+  createEmptyRuntime = () => ({
+    break: false,
+    debuggerOn: true,
+    isRunning: false,
+    environmentTree: new EnvTree(),
+    environments: [],
+    value: undefined,
+    nodes: [],
+    control: null,
+    stash: null,
+    objectCount: 0,
+    envSteps: -1,
+    envStepsTotal: 0,
+    breakpointSteps: [],
+    changepointSteps: []
+  })
+
+  public reset(program?: StmtNS.Stmt): void {
+    this.control = new PyControl(program);
+    this.stash = new Stash();
+    //this.environment = createProgramEnvironment(this, false);
+    this.errors = []; 
+  }
+
+  public copy(): PyContext {
+    const newContext = new PyContext();
+    newContext.control = this.control.copy();
+    newContext.stash = this.stash.copy();
+    //newContext.environments = this.copyEnvironment(this.environments);
+    return newContext;
+  }
+
+  private copyEnvironment(env: PyEnvironment): PyEnvironment {
+    const newTail = env.tail ? this.copyEnvironment(env.tail) : null;
+    const newEnv: PyEnvironment = {
+      id: env.id, 
+      name: env.name,
+      tail: newTail,
+      head: { ...env.head },
+      heap: new Heap(),
+      callExpression: env.callExpression, 
+      thisContext: env.thisContext
+    };
+    return newEnv;
+  }
+}
+
+export class EnvTree {
+  private _root: EnvTreeNode | null = null
+  private map = new Map<PyEnvironment, EnvTreeNode>()
+
+  get root(): EnvTreeNode | null {
+    return this._root
+  }
+
+  public insert(environment: PyEnvironment): void {
+    const tailEnvironment = environment.tail
+    if (tailEnvironment === null) {
+      if (this._root === null) {
+        this._root = new EnvTreeNode(environment, null)
+        this.map.set(environment, this._root)
+      }
+    } else {
+      const parentNode = this.map.get(tailEnvironment)
+      if (parentNode) {
+        const childNode = new EnvTreeNode(environment, parentNode)
+        parentNode.addChild(childNode)
+        this.map.set(environment, childNode)
+      }
+    }
+  }
+
+  public getTreeNode(environment: PyEnvironment): EnvTreeNode | undefined {
+    return this.map.get(environment)
+  }
+}
+
+export class EnvTreeNode {
+  private _children: EnvTreeNode[] = []
+
+  constructor(readonly environment: PyEnvironment, public parent: EnvTreeNode | null) {}
+
+  get children(): EnvTreeNode[] {
+    return this._children
+  }
+
+  public resetChildren(newChildren: EnvTreeNode[]): void {
+    this.clearChildren()
+    this.addChildren(newChildren)
+    newChildren.forEach(c => (c.parent = this))
+  }
+
+  private clearChildren(): void {
+    this._children = []
+  }
+
+  private addChildren(newChildren: EnvTreeNode[]): void {
+    this._children.push(...newChildren)
+  }
+
+  public addChild(newChild: EnvTreeNode): EnvTreeNode {
+    this._children.push(newChild)
+    return newChild
+  }
+}

src/cse-machine/py_control.ts

@@ -0,0 +1,55 @@
+import { Stack } from "./stack";
+import { PyNode, Instr } from "./py_types";
+import { StmtNS } from "../ast-types";
+import { isEnvDependent } from './utils'; // TODO
+import { StatementSequence } from "./py_types";
+
+export type PyControlItem = (PyNode | Instr) & {
+  isEnvDependent?: boolean;
+  skipEnv?: boolean;
+};
+
+
+export class PyControl extends Stack<PyControlItem> {
+  private numEnvDependentItems: number;
+  public constructor(program?: StmtNS.Stmt) {
+    super()
+    this.numEnvDependentItems = 0
+    // Load program into control stack
+    program ? this.push(program) : null
+  }
+
+  public canAvoidEnvInstr(): boolean {
+    return this.numEnvDependentItems === 0
+  }
+
+  // For testing purposes
+  public getNumEnvDependentItems(): number {
+    return this.numEnvDependentItems
+  }
+
+// TODO in the future
+//   public pop(): PyControlItem | undefined {
+//       const item = super.pop();
+//       if (item !== undefined && isEnvDependent(item)) {
+//         this.numEnvDependentItems--;
+//       }
+//       return item;
+//     }
+//   public push(...items: PyControlItem[]): void {
+//     items.forEach((item: PyControlItem) => {
+//     // We keep this logic for future use with the stepper.
+//     if (isEnvDependent(item)) {
+//         this.numEnvDependentItems++;
+//     }
+//     });
+//   super.push(...items);
+//   }
+
+  public copy(): PyControl {
+    const newControl = new PyControl();
+    const stackCopy = super.getStack();
+    newControl.push(...stackCopy);
+    return newControl;
+  }
+}