Коварный code type itgm #9
TRANSCRIPT
Коварный CodeType, или от segfault'а
к работающему коду
Андрей Захаревич Дмитрий Алимов
С чего все началось # Plain functions if inspect.isroutine(obj): if inspect.ismethod(obj): has_self = True argspec = inspect.getargspec(obj)
# Class constructors elif inspect.isclass(obj): try: argspec = inspect.getargspec(obj.__new__) except (AttributeError, TypeError): argspec = inspect.getargspec(obj.__init__) has_self = True
# obj objects elif hasattr(obj, '__call__'): method = obj.__call__ argspec = inspect.getargspec(method) has_self = inspect.ismethod(method)
return argspec.args[1:] if has_self else args
# Plain functionsif inspect.isroutine(obj): if inspect.ismethod(obj): has_self = True argspec = inspect.getargspec(obj)
# Class constructorselif inspect.isclass(obj): try: argspec = inspect.getargspec(obj.__new__) except (AttributeError, TypeError): argspec = inspect.getargspec(obj.__init__) has_self = True
# obj objectselif hasattr(obj, '__call__'): method = obj.__call__ argspec = inspect.getargspec(method) has_self = inspect.ismethod(method)
return argspec.args[1:] if has_self else args
Проблема
● Моя функция принимает переменное число аргументов (*args)
● Число передаваемых аргументов я узнаю в рантайме
● Объект функции в это время уже создан
Что же делать?
Метапрограммирование!
Ленивый вариант
compile('def func(a, b): f(a, b)', 'func.py', 'exec')
Не спортивно
Идем на stackoverflowimport types def create_function(name, args): def y(): pass y_code = types.CodeType(args, \ y.func_code.co_nlocals, \ y.func_code.co_stacksize, \ y.func_code.co_flags, \ y.func_code.co_code, \ y.func_code.co_consts, \ y.func_code.co_names, \ y.func_code.co_varnames, \ y.func_code.co_filename, \ name, \ y.func_code.co_firstlineno, \ y.func_code.co_lnotab) return types.FunctionType(y_code, y.func_globals, name) myfunc = create_function('myfunc', 3)myfunc(1,2,3,4)# TypeError: myfunc() takes exactly 3 arguments (4 given)
import types
def create_function(name, args): def y(): pass
y_code = types.CodeType(args, y.func_code.co_nlocals, y.func_code.co_stacksize, y.func_code.co_flags, y.func_code.co_code, y.func_code.co_consts, y.func_code.co_names, y.func_code.co_varnames, y.func_code.co_filename, name, y.func_code.co_firstlineno, y.func_code.co_lnotab)
return types.FunctionType(y_code, y.func_globals, name)
myfunc = create_function('myfunc', 3)myfunc(1, 2, 3, 4)# TypeError: myfunc() takes exactly 3 arguments (4 given)
Кажется должно работатьimport types def create_function(name, nargs): def y(*args): print args y_code = types.CodeType(nargs, \ y.func_code.co_nlocals, \ y.func_code.co_stacksize, \ y.func_code.co_flags, \ y.func_code.co_code, \ y.func_code.co_consts, \ y.func_code.co_names, \ y.func_code.co_varnames, \ y.func_code.co_filename, \ name, \ y.func_code.co_firstlineno, \ y.func_code.co_lnotab) return types.FunctionType(y_code, y.func_globals, name) myfunc = create_function('myfunc', 3)myfunc(1,2,3)
def create_function(name, nargs):
def y(*args): print args
y_code = types.CodeType(nargs, y.func_code.co_nlocals, # Параметры )
return types.FunctionType(y_code, y.func_globals, name)
myfunc = create_function('myfunc', 3)myfunc(1, 2, 3)
Но нет
[1] 8514 segmentation fault python func.py
Еще попыткаimport types
class Meta(type):
def __new__(mcls, name, bases, attrs):
nargs = 3
varnames = tuple(['self'] + [str(i) for i in range(0, nargs)])
ret_code = types.CodeType(nargs + 1, \
attrs['ret_func'].func_code.co_nlocals, \
attrs['ret_func'].func_code.co_stacksize, \
attrs['ret_func'].func_code.co_flags, \
attrs['ret_func'].func_code.co_code, \
attrs['ret_func'].func_code.co_consts, \
attrs['ret_func'].func_code.co_names, \
varnames, \
attrs['ret_func'].func_code.co_filename, \
'call', \
attrs['ret_func'].func_code.co_firstlineno, \
attrs['ret_func'].func_code.co_lnotab)
attrs['call'] = types.FunctionType(ret_code,
attrs['ret_func'].func_globals, 'call')
return super(Meta, mcls).__new__(mcls, name, bases, attrs)
class CustomCall(object):
__metaclass__ = Meta
def __call__(self, *args):
self.call(*args)
def ret_func(self):
print self.call.__name__
obj = CustomCall()
obj.call(1, 2, 3)
# Если закомментировать, то все
упадет
obj(1, 2, 3)
class Meta(type): def __new__(mcls, name, bases, attrs): nargs = 3 varnames = tuple(['self'] + [str(i) for i in range(nargs)]) ret_code = types.CodeType( nargs + 1, attrs['ret_func'].func_code.co_nlocals, attrs['ret_func'].func_code.co_stacksize, attrs['ret_func'].func_code.co_flags, attrs['ret_func'].func_code.co_code, attrs['ret_func'].func_code.co_consts, attrs['ret_func'].func_code.co_names, varnames, attrs['ret_func'].func_code.co_filename, 'call', attrs['ret_func'].func_code.co_firstlineno, attrs['ret_func'].func_code.co_lnotab) attrs['call'] = types.FunctionType( ret_code, attrs['ret_func'].func_globals, 'call') return super(Meta, mcls).__new__(mcls, name, bases, attrs)
class CustomCall(object): __metaclass__ = Meta
def __call__(self, *args): self.call(*args)
def ret_func(self): print self.call.__name__
obj = CustomCall()
obj.call(1, 2, 3)# Если закомментировать, то все упадетobj(1, 2, 3)
Еще попыткаimport types
class Meta(type):
def __new__(mcls, name, bases, attrs):
nargs = 3
varnames = tuple(['self'] + [str(i) for i in range(0, nargs)])
ret_code = types.CodeType(nargs + 1, \
attrs['ret_func'].func_code.co_nlocals, \
attrs['ret_func'].func_code.co_stacksize, \
attrs['ret_func'].func_code.co_flags, \
attrs['ret_func'].func_code.co_code, \
attrs['ret_func'].func_code.co_consts, \
attrs['ret_func'].func_code.co_names, \
varnames, \
attrs['ret_func'].func_code.co_filename, \
'call', \
attrs['ret_func'].func_code.co_firstlineno, \
attrs['ret_func'].func_code.co_lnotab)
attrs['call'] = types.FunctionType(ret_code,
attrs['ret_func'].func_globals, 'call')
return super(Meta, mcls).__new__(mcls, name, bases, attrs)
class CustomCall(object):
__metaclass__ = Meta
def __call__(self, *args):
self.call(*args)
def ret_func(self):
print self.call.__name__
obj = CustomCall()
obj.call(1, 2, 3)
# Если закомментировать, то все
упадет
obj(1, 2, 3)
class Meta(type): def __new__(mcls, name, bases, attrs): nargs = 3 varnames = tuple(['self'] + [str(i) for i in range(nargs)]) ret_code = types.CodeType( nargs + 1, attrs['ret_func'].func_code.co_nlocals, attrs['ret_func'].func_code.co_stacksize, attrs['ret_func'].func_code.co_flags, attrs['ret_func'].func_code.co_code, attrs['ret_func'].func_code.co_consts, attrs['ret_func'].func_code.co_names, varnames, attrs['ret_func'].func_code.co_filename, 'call', attrs['ret_func'].func_code.co_firstlineno, attrs['ret_func'].func_code.co_lnotab) attrs['call'] = types.FunctionType( ret_code, attrs['ret_func'].func_globals, 'call') return super(Meta, mcls).__new__(mcls, name, bases, attrs)
class CustomCall(object): __metaclass__ = Meta
def __call__(self, *args): self.call(*args)
def ret_func(self): print self.call.__name__
obj = CustomCall()
obj.call(1, 2, 3)# Если закомментировать, то все упадетobj(1, 2, 3)
Заглянем в Disassembler
5D09E429 |. 8B7424 10 MOV ESI,DWORD PTR SS:[ESP+10] ; ESI = frame
5D09E42D |. 8B7D 14 MOV EDI,DWORD PTR SS:[EBP+14] ; EDI = *args
5D09E430 |. 8B5C24 18 MOV EBX,DWORD PTR SS:[ESP+18] ; EBX = argcount
5D09E434 |. 81C6 38010000 ADD ESI,0x138 ; ESI += 0x138 (f_localsplus)
5D09E43A |. 2BFE SUB EDI,ESI ; EDI -= ESI
5D09E43C |. 8D6424 00 LEA ESP,[ESP] ; NOP
5D09E440 |> 8B0437 /MOV EAX,DWORD PTR DS:[ESI+EDI] ; EAX = *(ESI + EDI)
5D09E443 |. FF00 |INC DWORD PTR DS:[EAX] ; Py_INCREF(*EAX)
5D09E445 |. 8B0E |MOV ECX,DWORD PTR DS:[ESI] ; ECX = *ESI
5D09E447 |. 8906 |MOV DWORD PTR DS:[ESI],EAX ; *ESI = EAX
5D09E449 |. 85C9 |TEST ECX,ECX ; test ECX:
5D09E44B |.- 74 11 |JZ SHORT 5D09E45E ; if 0: goto 5D09E45E
5D09E44D |. 8301 FF |ADD DWORD PTR DS:[ECX],-1 ; else: Py_XDECREF(*ECX)
5D09E450 |.- 75 0C |JNZ SHORT 5D09E45E ; if not 0: goto 5D09E45E
5D09E452 |. 8B41 04 |MOV EAX,DWORD PTR DS:[ECX+4] ; ECX->ob_type
5D09E455 |. 51 |PUSH ECX ; save fastlocals[i] to stack
5D09E456 |. 8B48 18 |MOV ECX,DWORD PTR DS:[EAX+18] ; ECX = ob_type->tp_dealloc
5D09E459 |. FFD1 |CALL ECX ; call ob_type->tp_dealloc()
5D09E45B |. 83C4 04 |ADD ESP,4 ; restore ESP
5D09E45E |> 83C6 04 |ADD ESI,4 ; ESI += 4
5D09E461 |. 83EB 01 |SUB EBX,1 ; EBX -= 1
5D09E464 |.- 75 DA \JNZ SHORT 5D09E440 ;
01eb3030 00000001 ob_refcnt
01eb3034 5886d830 python27!PyFrame_Type
01eb3038 00000002 ob_size
01eb303c 023b6b30 f_back
01eb3040 023a6b18 f_code
01eb3044 01e790c0 f_builtins
01eb3048 01e8aa50 f_globals
01eb304c 00000000 f_locals
01eb3050 01eb316c f_valuestack
01eb3054 01eb316c f_stacktop
01eb3058 00000000 f_trace
01eb305c 00000000 f_exc_type
01eb3060 00000000 f_exc_value
01eb3064 00000000 f_exc_traceback
01eb3068 01f72cf0 f_tstate
01eb306c ffffffff f_lasti
01eb3070 0000002f f_lineno
01eb3074 00000000 f_iblock
01eb3078 baadf00d f_blockstack[20] = {b_type,
01eb307c baadf00d b_handler
01eb3080 baadf00d b_level}
...
01eb3168 023bf550 f_localsplus // frame + 0x138
01eb316c 01f7d8a0
01eb3170 baadf00d
01eb3174 baadf00d
Что в памяти
PyObject *
PyEval_EvalCodeEx(PyCodeObject *co, PyObject *globals, PyObject *locals,
PyObject **args, int argcount, PyObject **kws, int kwcount,
PyObject **defs, int defcount, PyObject *closure) {
...
n = co->co_argcount;
for (i = 0; i < n; i++) {
x = args[i];
Py_INCREF(x);
SETLOCAL(i, x);
}
...
https://github.com/python/cpython/blob/2.7/Python/ceval.c
#define Py_INCREF(op) ( \
_Py_INC_REFTOTAL _Py_REF_DEBUG_COMMA \
((PyObject*)(op))->ob_refcnt++)
#define GETLOCAL(i) (fastlocals[i])
#define SETLOCAL(i, value) do { PyObject *tmp = GETLOCAL(i); \
GETLOCAL(i) = value; \
Py_XDECREF(tmp); } while (0)
#define Py_XDECREF(op) do { if ((op) == NULL); else Py_DECREF(op); } while (0)
#define Py_DECREF(op) \
do { \
if (_Py_DEC_REFTOTAL _Py_REF_DEBUG_COMMA \
--((PyObject*)(op))->ob_refcnt != 0) \
_Py_CHECK_REFCNT(op) \
else \
_Py_Dealloc((PyObject *)(op)); \
} while (0)
#define _Py_Dealloc(op) ( \
_Py_INC_TPFREES(op) _Py_COUNT_ALLOCS_COMMA \
(*Py_TYPE(op)->tp_dealloc)((PyObject *)(op)))
#endif /* !Py_TRACE_REFS */
https://github.com/python/cpython/blob/2.7/Include/object.h
typedef struct _frame {
PyObject_VAR_HEAD
struct _frame *f_back; /* previous frame, or NULL */
PyCodeObject *f_code; /* code segment */
PyObject *f_builtins; /* builtin symbol table (PyDictObject) */
PyObject *f_globals; /* global symbol table (PyDictObject) */
PyObject *f_locals; /* local symbol table (any mapping) */
PyObject **f_valuestack; /* points after the last local */
PyObject **f_stacktop;
PyObject *f_trace; /* Trace function */
PyObject *f_exc_type, *f_exc_value, *f_exc_traceback;
PyThreadState *f_tstate;
int f_lasti; /* Last instruction if called */
int f_lineno; /* Current line number */
int f_iblock; /* index in f_blockstack */
PyTryBlock f_blockstack[CO_MAXBLOCKS]; /* for try and loop blocks */
PyObject *f_localsplus[1]; /* locals+stack, dynamically sized */
} PyFrameObject;
https://github.com/python/cpython/blob/2.7/Include/frameobject.h
ncells = PyTuple_GET_SIZE(code->co_cellvars);
nfrees = PyTuple_GET_SIZE(code->co_freevars);
extras = code->co_stacksize + code->co_nlocals + ncells + nfrees;
if (free_list == NULL) {
f = PyObject_GC_NewVar(PyFrameObject, &PyFrame_Type, extras);
...
https://github.com/python/cpython/blob/2.7/Objects/frameobject.c
co_cellvars – is a tuple with the names of local variables referenced by nested functions;
co_freevars – names of variables in the function, defined in an enclosing function scope;
Память для f_localsplus
Проблема найдена
ret_code = types.CodeType(
nargs + 1,
attrs['ret_func'].func_code.co_nlocals,
attrs['ret_func'].func_code.co_stacksize,
attrs['ret_func'].func_code.co_flags,
attrs['ret_func'].func_code.co_code,
attrs['ret_func'].func_code.co_consts,
attrs['ret_func'].func_code.co_names,
varnames,
attrs['ret_func'].func_code.co_filename,
'call',
attrs['ret_func'].func_code.co_firstlineno,
attrs['ret_func'].func_code.co_lnotab,
(), # freevars
() # cellvars
)
ret_code = types.CodeType( nargs + 1, attrs['ret_func'].func_code.co_nlocals, attrs['ret_func'].func_code.co_stacksize, attrs['ret_func'].func_code.co_flags, attrs['ret_func'].func_code.co_code, attrs['ret_func'].func_code.co_consts, attrs['ret_func'].func_code.co_names, varnames, attrs['ret_func'].func_code.co_filename, 'call', attrs['ret_func'].func_code.co_firstlineno, attrs['ret_func'].func_code.co_lnotab, (), # freevars () # cellvars)
http://markup.su/highlighter/ Python - LAZY
Попробуем применитьdef create_func(fun, nargs):
def wrapper(fun):
args = map(itemgetter(1), sorted(((var_name, arg) for (var_name, arg) in locals().items() if var_name != 'fun'),
key=itemgetter(0)))
fun(*args)
varnames = tuple(['fun'] + [str(i) for i in range(0, nargs)])
ret_code = types.CodeType(nargs + 1, \
wrapper.func_code.co_nlocals + nargs, \
wrapper.func_code.co_stacksize, \
wrapper.func_code.co_flags, \
wrapper.func_code.co_code, \
wrapper.func_code.co_consts, \
wrapper.func_code.co_names, \
varnames, \
wrapper.func_code.co_filename, \
wrapper.func_code.co_name, \
wrapper.func_code.co_firstlineno, \
wrapper.func_code.co_lnotab)
return partial(types.FunctionType(ret_code, wrapper.func_globals, wrapper.func_code.co_name), fun)
def test_func(*args):
print args
fun = create_func(test_func, 3)
fun(1,2,3)
def create_func(fun, nargs): def wrapper(): args = map(itemgetter(1), sorted(((var_name, arg) for (var_name, arg) in locals().items() if var_name != 'fun'), key=itemgetter(0))) fun(*args)
varnames = tuple(str(i) for i in range(0, nargs)) ret_code = types.CodeType(nargs, wrapper.func_code.co_nlocals + nargs, # Параметры ) return types.FunctionType(ret_code, wrapper.func_globals, wrapper.func_code.co_name)
def test_func(*args): print args
fun = create_func(test_func, 3)fun(1, 2, 3)
Как-то не очень
Traceback (most recent call last):
File "func.py", line 38, in <module>
fun(1,2,3)
File "func.py", line 16, in wrapper
fun(*args)
SystemError: Objects/cellobject.c:24: bad argument to internal function
Заработало!def create_func(fun, nargs):
def wrapper(fun):
args = map(itemgetter(1), sorted(((var_name, arg) for (var_name, arg) in locals().items() if var_name != 'fun'),
key=itemgetter(0)))
fun(*args)
varnames = tuple(['fun'] + [str(i) for i in range(0, nargs)])
ret_code = types.CodeType(nargs + 1, \
wrapper.func_code.co_nlocals + nargs, \
wrapper.func_code.co_stacksize, \
wrapper.func_code.co_flags, \
wrapper.func_code.co_code, \
wrapper.func_code.co_consts, \
wrapper.func_code.co_names, \
varnames, \
wrapper.func_code.co_filename, \
wrapper.func_code.co_name, \
wrapper.func_code.co_firstlineno, \
wrapper.func_code.co_lnotab)
return partial(types.FunctionType(ret_code, wrapper.func_globals, wrapper.func_code.co_name), fun)
def test_func(*args):
print args
fun = create_func(test_func, 3)
fun(1,2,3)
def create_func(fun, nargs): def wrapper(fun): args = map(itemgetter(1), sorted(((var_name, arg) for (var_name, arg) in locals().items() if var_name != 'fun'), key=itemgetter(0))) fun(*args)
varnames = tuple(['fun'] + [str(i) for i in range(0, nargs)]) ret_code = types.CodeType(nargs + 1, wrapper.func_code.co_nlocals + nargs, # Параметры ) return partial(types.FunctionType(ret_code, wrapper.func_globals, wrapper.func_code.co_name), fun)
def test_func(*args): print args
fun = create_func(test_func, 3)fun(1, 2, 3)
Что-то потеряли
>>> dir(wrapper.func_code)
['__class__', '__cmp__', '__delattr__', '__doc__', '__eq__', '__format__', '__ge__', '__getattribute__', '__gt__', '__hash__', '__init__', '__le__', '__lt__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__', 'co_argcount', 'co_cellvars', 'co_code', 'co_consts', 'co_filename', 'co_firstlineno', 'co_flags', 'co_freevars', 'co_lnotab', 'co_name', 'co_names', 'co_nlocals', 'co_stacksize', 'co_varnames']
def create_func(fun, nargs):
def wrapper():
args = map(itemgetter(1), sorted(((var_name, arg) for (var_name, arg) in locals().items() if var_name != 'fun'),
key=itemgetter(0)))
fun(*args)
varnames = tuple([str(i) for i in range(0, nargs)])
ret_code = types.CodeType(nargs, \
wrapper.func_code.co_nlocals + nargs, \
wrapper.func_code.co_stacksize, \
wrapper.func_code.co_flags, \
wrapper.func_code.co_code, \
wrapper.func_code.co_consts, \
wrapper.func_code.co_names, \
varnames, \
wrapper.func_code.co_filename, \
wrapper.func_code.co_name, \
wrapper.func_code.co_firstlineno, \
wrapper.func_code.co_lnotab,
wrapper.func_code.co_freevars)
return types.FunctionType(ret_code, wrapper.func_globals, wrapper.func_code.co_name, None, wrapper.func_closure)
def test_func(*args):
print args
fun = create_func(test_func, 3)
fun(1,2,3)
Добавим параметр
ret_code = types.CodeType(nargs, wrapper.func_code.co_nlocals + nargs, # Параметры wrapper.func_code.co_freevars)return types.FunctionType(ret_code, wrapper.func_globals, wrapper.func_code.co_name)
>>>Traceback (most recent call last): File "func.py", line 39, in <module> fun = create_func(test_func, 3) File "func.py", line 33, in create_func return types.FunctionType(ret_code, wrapper.func_globals, wrapper.func_code.co_name)TypeError: arg 5 (closure) must be tuple
def create_func(fun, nargs): def wrapper(): args = map(itemgetter(1), sorted(((var_name, arg) for (var_name, arg) in locals().items() if var_name != 'fun'), key=itemgetter(0))) fun(*args)
varnames = tuple([str(i) for i in range(0, nargs)]) ret_code = types.CodeType(nargs, wrapper.func_code.co_nlocals + nargs, # Параметры wrapper.func_code.co_freevars) return types.FunctionType(ret_code, wrapper.func_globals, wrapper.func_code.co_name, None, wrapper.func_closure)
def test_func(*args): print args
fun = create_func(test_func, 3)fun(1, 2, 3)
def create_func(fun, nargs):
def wrapper():
args = map(itemgetter(1), sorted(((var_name, arg) for (var_name, arg) in locals().items() if var_name != 'fun'),
key=itemgetter(0)))
fun(*args)
varnames = tuple([str(i) for i in range(0, nargs)])
ret_code = types.CodeType(nargs, \
wrapper.func_code.co_nlocals + nargs, \
wrapper.func_code.co_stacksize, \
wrapper.func_code.co_flags, \
wrapper.func_code.co_code, \
wrapper.func_code.co_consts, \
wrapper.func_code.co_names, \
varnames, \
wrapper.func_code.co_filename, \
wrapper.func_code.co_name, \
wrapper.func_code.co_firstlineno, \
wrapper.func_code.co_lnotab,
wrapper.func_code.co_freevars)
return types.FunctionType(ret_code, wrapper.func_globals, wrapper.func_code.co_name, None, wrapper.func_closure)
def test_func(*args):
print args
fun = create_func(test_func, 3)
fun(1,2,3)
И наконец, замыкания работают!
… почти
def wrapper():
print locals()
args = map(
itemgetter(1),
sorted(
((var_name, arg) for (var_name, arg) in locals().items() if var_name != 'fun'),
key=itemgetter(0)
)
)
print locals()
fun(*args)
{'1': 2, '0': 1, '2': 3, 'fun': <function test_func at 0x10cd62398>}
{'1': 2, '0': [1, 2, 3], '2': 3, 'fun': <function test_func at 0x10cd62398>}
from operator import itemgetter
def wrapper(): print locals() args = map(itemgetter(1), sorted(((var_name, arg) for (var_name, arg) in locals().items() if var_name != 'fun'), key=itemgetter(0)))
print locals() fun(*args)
{'1': 2, '0': 1, '2': 3, 'fun': <function test_func at 0x10cd62398>}
{'1': 2, '0': [1, 2, 3], '2': 3, 'fun': <function test_func at 0x10cd62398>}
Спасибо за внимание!
Вопросы?
SPb Python Interest Group
https://telegram.me/spbpython