Remove remove_globals. Work in progress

Author: markshannon

Include/internal/pycore_optimizer.h

@@ -172,6 +172,7 @@ typedef enum _JitSymType {
     JIT_SYM_KNOWN_CLASS_TAG = 6,
     JIT_SYM_KNOWN_VALUE_TAG = 7,
     JIT_SYM_TUPLE_TAG = 8,
+    JIT_SYM_FUNCTION_TAG = 9,
 } JitSymType;
 
 typedef struct _jit_opt_known_class {
@@ -198,24 +199,34 @@ typedef struct _jit_opt_tuple {
     uint16_t items[MAX_SYMBOLIC_TUPLE_SIZE];
 } JitOptTuple;
 
+typedef struct _jit_opt_function {
+    uint8_t tag;
+    uint16_t arg_count;
+    uint32_t version;
+    PyCodeObject *code;
+} JitOptFunction;
+
 typedef union _jit_opt_symbol {
     uint8_t tag;
     JitOptKnownClass cls;
     JitOptKnownValue value;
     JitOptKnownVersion version;
     JitOptTuple tuple;
+    JitOptFunction function;
 } JitOptSymbol;
 
 
-
 struct _Py_UOpsAbstractFrame {
     // Max stacklen
     int stack_len;
     int locals_len;
-
+    PyFunctionObject *function;
     JitOptSymbol **stack_pointer;
     JitOptSymbol **stack;
     JitOptSymbol **locals;
+    bool function_checked;
+    bool builtins_watched;
+    bool globals_watched;
 };
 
 typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame;
@@ -234,6 +245,8 @@ typedef struct _JitOptContext {
     _Py_UOpsAbstractFrame *frame;
     _Py_UOpsAbstractFrame frames[MAX_ABSTRACT_FRAME_DEPTH];
     int curr_frame_depth;
+    PyInterpreterState *interp;
+    _PyBloomFilter *dependencies;
 
     // Arena for the symbolic types.
     ty_arena t_arena;
@@ -260,6 +273,8 @@ extern void _Py_uop_sym_set_null(JitOptContext *ctx, JitOptSymbol *sym);
 extern void _Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptSymbol *sym);
 extern void _Py_uop_sym_set_type(JitOptContext *ctx, JitOptSymbol *sym, PyTypeObject *typ);
 extern bool _Py_uop_sym_set_type_version(JitOptContext *ctx, JitOptSymbol *sym, unsigned int version);
+extern void _Py_uop_sym_set_function_version(JitOptContext *ctx, JitOptSymbol *sym, uint32_t version);
+extern uint32_t _Py_uop_sym_get_function_version(JitOptSymbol *sym);
 extern void _Py_uop_sym_set_const(JitOptContext *ctx, JitOptSymbol *sym, PyObject *const_val);
 extern bool _Py_uop_sym_is_bottom(JitOptSymbol *sym);
 extern int _Py_uop_sym_truthiness(JitOptSymbol *sym);

Python/optimizer_analysis.c

@@ -1,4 +1,4 @@
-#ifdef _Py_TIER2
+// #ifdef _Py_TIER2
 
 /*
  * This file contains the support code for CPython's uops optimizer.
@@ -124,36 +124,122 @@ convert_global_to_const(_PyUOpInstruction *inst, PyObject *obj, bool pop)
 }
 
 static int
-incorrect_keys(_PyUOpInstruction *inst, PyObject *obj)
+incorrect_keys(uint32_t version, PyObject *obj)
 {
     if (!PyDict_CheckExact(obj)) {
         return 1;
     }
     PyDictObject *dict = (PyDictObject *)obj;
-    if (dict->ma_keys->dk_version != inst->operand0) {
+    if (dict->ma_keys->dk_version != version) {
         return 1;
     }
     return 0;
 }
 
-static int
-check_next_uop(_PyUOpInstruction *buffer, int size, int pc, uint16_t expected)
+int
+optimize_guard_globals_version(
+    _PyUOpInstruction *instr,
+    uint32_t version,
+    JitOptContext *ctx
+)
 {
-    if (pc + 1 >= size) {
-        DPRINTF(1, "Cannot rewrite %s at pc %d: buffer too small\n",
-                _PyOpcode_uop_name[buffer[pc].opcode], pc);
-        return 0;
+    if (ctx->frame->function == NULL) {
+        return -1;
     }
-    uint16_t next_opcode = buffer[pc + 1].opcode;
-    if (next_opcode != expected) {
-        DPRINTF(1,
-                "Cannot rewrite %s at pc %d: unexpected next opcode %s, "
-                "expected %s\n",
-                _PyOpcode_uop_name[buffer[pc].opcode], pc,
-                _PyOpcode_uop_name[next_opcode], _PyOpcode_uop_name[expected]);
-        return 0;
+    PyDictObject *globals = (PyDictObject *)ctx->frame->function->func_globals;
+    if (!PyDict_CheckExact(globals)) {
+        ctx->done = true;
+        return -1;
+    }
+    if (globals->ma_keys->dk_version != version) {
+        OPT_STAT_INC(remove_globals_incorrect_keys);
+        ctx->done = true;
+        return -1;
+    }
+    uint64_t watched_mutations = get_mutations(globals);
+    if (watched_mutations >= _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) {
+        return -1;
+    }
+    if (!ctx->frame->globals_watched) {
+        PyDict_Watch(GLOBALS_WATCHER_ID, (PyObject *)globals);
+        ctx->frame->globals_watched = true;
+    }
+    instr->opcode = _NOP;
+    instr->oparg = 0;
+    return 0;
+}
+
+JitOptSymbol *
+optimize_load_global_module(
+    _PyUOpInstruction *instr,
+    uint32_t version,
+    JitOptContext *ctx
+)
+{
+    if (ctx->frame->function == NULL) {
+        goto no_opt;
     }
-    return 1;
+    PyObject *globals = ctx->frame->function->func_globals;
+    if (incorrect_keys(version, globals)) {
+        OPT_STAT_INC(remove_globals_incorrect_keys);
+        ctx->done = true;
+        goto no_opt;
+    }
+    if (get_mutations(globals) >= _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS) {
+        goto no_opt;
+    }
+    if (!ctx->frame->globals_watched) {
+        PyDict_Watch(GLOBALS_WATCHER_ID, globals);
+        _Py_BloomFilter_Add(ctx->dependencies, globals);
+        ctx->frame->globals_watched = true;
+    }
+    if (!ctx->frame->function_checked && instr[-1].opcode == _NOP) {
+        instr[-1].opcode = _CHECK_FUNCTION;
+        instr[-1].operand0 = func_version;
+        ctx->frame->function_checked = true;
+    }
+    if (ctx->frame->function_checked) {
+        PyObject *cnst = convert_global_to_const(instr, globals, false);
+        if (cnst != NULL) {
+            return _Py_uop_sym_new_const(ctx, cnst);
+        }
+    }
+no_opt:
+    return _Py_uop_sym_new_not_null(ctx);
+}
+
+
+JitOptSymbol *
+optimize_load_global_builtins(
+    _PyUOpInstruction *instr,
+    uint32_t version,
+    JitOptContext *ctx
+)
+{
+    if (ctx->frame->function == NULL) {
+        goto no_opt;
+    }
+    PyObject *builtins = ctx->frame->function->func_builtins;
+    if (incorrect_keys(version, builtins)) {
+        OPT_STAT_INC(remove_globals_incorrect_keys);
+        ctx->done = true;
+        goto no_opt;
+    }
+    if (ctx->interp->rare_events.builtin_dict >= _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) {
+        goto no_opt;
+    }
+    if (!ctx->frame->builtins_watched) {
+        PyDict_Watch(BUILTINS_WATCHER_ID, builtins);
+        ctx->frame->builtins_watched = true;
+    }
+    if (ctx->frame->function_checked) {
+        PyObject *cnst = convert_global_to_const(instr, builtins, false);
+        if (cnst != NULL) {
+            return _Py_uop_sym_new_const(ctx, cnst);
+        }
+    }
+no_opt:
+    return _Py_uop_sym_new_not_null(ctx);
 }
 
 /* Returns 1 if successfully optimized
@@ -200,7 +286,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer,
         int opcode = inst->opcode;
         switch(opcode) {
             case _GUARD_GLOBALS_VERSION:
-                if (incorrect_keys(inst, globals)) {
+                if (incorrect_keys(inst->operand0, globals)) {
                     OPT_STAT_INC(remove_globals_incorrect_keys);
                     return 0;
                 }
@@ -222,7 +308,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer,
                 }
                 break;
             case _LOAD_GLOBAL_BUILTINS:
-                if (incorrect_keys(inst, builtins)) {
+                if (incorrect_keys(inst->operand0, builtins)) {
                     OPT_STAT_INC(remove_globals_incorrect_keys);
                     return 0;
                 }
@@ -238,7 +324,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer,
                 }
                 break;
             case _LOAD_GLOBAL_MODULE:
-                if (incorrect_keys(inst, globals)) {
+                if (incorrect_keys(inst->operand0, globals)) {
                     OPT_STAT_INC(remove_globals_incorrect_keys);
                     return 0;
                 }
@@ -350,6 +436,8 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer,
 #define sym_get_type _Py_uop_sym_get_type
 #define sym_matches_type _Py_uop_sym_matches_type
 #define sym_matches_type_version _Py_uop_sym_matches_type_version
+#define sym_get_function_version _Py_uop_sym_get_function_version
+#define sym_set_function_version _Py_uop_sym_set_function_version
 #define sym_set_null(SYM) _Py_uop_sym_set_null(ctx, SYM)
 #define sym_set_non_null(SYM) _Py_uop_sym_set_non_null(ctx, SYM)
 #define sym_set_type(SYM, TYPE) _Py_uop_sym_set_type(ctx, SYM, TYPE)
@@ -447,14 +535,16 @@ get_code_with_logging(_PyUOpInstruction *op)
 /* 1 for success, 0 for not ready, cannot error at the moment. */
 static int
 optimize_uops(
-    PyCodeObject *co,
+    _PyInterpreterFrame *real_frame,
     _PyUOpInstruction *trace,
     int trace_len,
     int curr_stacklen,
     _PyBloomFilter *dependencies
 )
 {
-
+    PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(real_frame->f_funcobj);
+    PyCodeObject *co = _PyFrame_GetCode(real_frame);
+    assert(PyFunction_Check(func));
     JitOptContext context;
     JitOptContext *ctx = &context;
     uint32_t opcode = UINT16_MAX;
@@ -468,11 +558,14 @@ optimize_uops(
     if (frame == NULL) {
         return -1;
     }
+    frame->function = (PyFunctionObject *)func;
     ctx->curr_frame_depth++;
     ctx->frame = frame;
     ctx->done = false;
     ctx->out_of_space = false;
     ctx->contradiction = false;
+    ctx->dependencies = dependencies;
+    ctx->interp = _PyInterpreterState_GET();
 
     _PyUOpInstruction *this_instr = NULL;
     for (int i = 0; !ctx->done; i++) {
@@ -649,7 +742,7 @@ _Py_uop_analyze_and_optimize(
     }
 
     length = optimize_uops(
-        _PyFrame_GetCode(frame), buffer,
+        frame, buffer,
         length, curr_stacklen, dependencies);
 
     if (length <= 0) {

Python/optimizer_bytecodes.c

@@ -20,6 +20,8 @@ typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame;
 #define sym_new_null _Py_uop_sym_new_null
 #define sym_matches_type _Py_uop_sym_matches_type
 #define sym_matches_type_version _Py_uop_sym_matches_type_version
+#define sym_get_function_version _Py_uop_sym_get_function_version
+#define sym_set_function_version _Py_uop_sym_set_function_version
 #define sym_get_type _Py_uop_sym_get_type
 #define sym_has_type _Py_uop_sym_has_type
 #define sym_set_null(SYM) _Py_uop_sym_set_null(ctx, SYM)
@@ -596,12 +598,18 @@ dummy_func(void) {
     }
 
     op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) {
-        if (sym_is_const(callable) && sym_matches_type(callable, &PyFunction_Type)) {
-            assert(PyFunction_Check(sym_get_const(callable)));
-            REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0, func_version);
-            this_instr->operand1 = (uintptr_t)sym_get_const(callable);
+        assert(func_version != 0);
+        if (sym_get_function_version(callable) == func_version) {
+            REPLACE_OP(this_instr, _NOP, 0, 0);
+        }
+        else {
+            if (sym_is_const(callable) && sym_matches_type(callable, &PyFunction_Type)) {
+                assert(PyFunction_Check(sym_get_const(callable)));
+                REPLACE_OP(this_instr, _CHECK_FUNCTION_VERSION_INLINE, 0, func_version);
+                this_instr->operand1 = (uintptr_t)sym_get_const(callable);
+            }
+            sym_set_function_version(ctx, callable, func_version);
         }
-        sym_set_type(callable, &PyFunction_Type);
     }
 
     op(_CHECK_FUNCTION_EXACT_ARGS, (callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) {
@@ -762,13 +770,21 @@ dummy_func(void) {
         ctx->frame = new_frame;
         ctx->curr_frame_depth++;
         stack_pointer = new_frame->stack_pointer;
-        co = get_code(this_instr);
-        if (co == NULL) {
+        uint64_t operand = this_instr->operand0;
+        if (operand == 0) {
             // should be about to _EXIT_TRACE anyway
             ctx->done = true;
             break;
         }
-
+        if (operand & 1) {
+            co = (PyCodeObject *)(operand & ~1);
+        }
+        else {
+            PyFunctionObject *func = (PyFunctionObject *)operand;
+            assert(PyFunction_Check(func));
+            co = (PyCodeObject *)func->func_code;
+            new_frame->function = func;
+        }
         /* Stack space handling */
         int framesize = co->co_framesize;
         assert(framesize > 0);
@@ -889,6 +905,20 @@ dummy_func(void) {
         }
     }
 
+    op(_GUARD_GLOBALS_VERSION, (version/1 --)) {
+        optimize_guard_globals_version(this_instr, version, ctx);
+    }
+
+    op(_LOAD_GLOBAL_MODULE, (version/1, unused/1, index/1 -- res)) {
+        (void)index;
+        res = optimize_load_global_module(this_instr, version, ctx);
+    }
+
+    op(_LOAD_GLOBAL_BUILTINS, (version/1, index/1 -- res)) {
+        (void)index;
+        res = optimize_load_global_builtins(this_instr, version, ctx);
+    }
+
 
 // END BYTECODES //