Black Magic for Good Fairies

Artem Malyshev

@proofit404

Python for Humans

they told me...

ORLY?

Q

Output

tailf /tmp/q
0.0s test: prefix + q(sep or '')=' '
0.0s test: 'xxx'
0.0s test: 'xxx '
0.0s foo('xxx')
0.0s -> 'xxx!!!'

Usage

import q
write(prefix + q(sep or ''))
write(q/prefix + (sep or ''))
write(q|prefix + (sep or ''))

@q
def foo(bar): ...

Usage

import q
write(prefix + q(sep or ''))
write(q/prefix + (sep or ''))
write(q|prefix + (sep or ''))

@q
def foo(bar): ...

Replace module during its import

class Q(object):
    import inspect
    def __truediv__(self, arg):
        info = self.inspect.getframeinfo(...
        self.show(info.function, [arg])
        return arg

sys.modules['q'] = Q()

Replace module during its import

class Q(object):
    import inspect
    def __truediv__(self, arg):
        info = self.inspect.getframeinfo(...
        self.show(info.function, [arg])
        return arg

sys.modules['q'] = Q()

Logging

usage

def my_func():
    logger.info('We are here')

output

INFO:my_func:We are here

Search who calls you

def get_caller():
    f = sys._getframe()
    while hasattr(f, 'f_code'):
        filename = f.f_code.co_filename
        if filename == __file__:
            f = f.f_back
            continue
        return f.f_code.co_name

Search who calls you

def get_caller():
    f = sys._getframe()
    while hasattr(f, 'f_code'):
        filename = f.f_code.co_filename
        if filename == __file__:
            f = f.f_back
            continue
        return f.f_code.co_name

Django

Model exceptions

try:
    Entry.objects.get(id='foo')
except Entry.DoesNotExist:
    pass  # Caught it

Model exceptions

try:
    Entry.objects.get(id='foo')
except Post.DoesNotExist:
    pass  # Lost it

Inheritance

class Entry(models.Model):
    field1 = models.CharField()
    field2 = models.IntegerField()

class factories

Entry = ModelBase('Entry', (models.Model,), {
    'field1': models.CharField(),
    'field2': models.IntegerField(),
})

Metaclasses

class ModelBase(type):
    """
    Metaclass for all models.
    """
    def __new__(cls, name, bases, attrs):
        new_class = type(cls, name, bases, {
            '__module__': module,
        })
        ...
        return new_class

Introduce model exceptions

if not abstract:
    new_class.add_to_class(
        'DoesNotExist',
        subclass_exception(
            str('DoesNotExist'),
            (ObjectDoesNotExist,),
            module,
            attached_to=new_class))

Sly

Lexer

from sly import Lexer

class CalcLexer(Lexer):
    @_(r'\d+')
    def NUMBER(self, t):
        t.value = int(t.value)
        return t

Normal class

Traceback (most recent call last):
  File "name_error_sly.py", line 6, in <module>
    class CalcLexer(Lexer):
  File "name_error_sly.py", line 7, in CalcLexer
    @_(r'\d+')
NameError: name '_' is not defined

Extend class scope

class LexerType(type):
    @staticmethod
    def __prepare__(cls, *args, **kwargs):
        def _(pattern): ...
        d = {'_': _}
        return d

class Lexer(metaclass=LexerType):
    pass

Ruby

String interpolation

apples = 4
puts "I have #{apples} apples"

Output

I have 4 apples

Python can do that

# coding: interpy
apples = 4
print('I have #{apples} apples')

Output

I have 4 apples

React

HTML inside JS? Why not

render: function () {
  return (
    <div className="comment">
      <h2 className="commentAuthor">
        {this.props.author}
      </h2>
      {this.props.children}
    </div>
  );
}

Yep, in Python too

# coding: pyxl
def render(self):
    return (
        <div class="comment">
            <h2 class="commentAuthor">
                {self.author.name}
            </h2>
            {self.children()}
        </div>)

# coding: pyxl

set file preprocessor

def search_function(encoding):
    return codecs.CodecInfo(
        name = 'pyxl',
        ...
    )

codecs.register(search_function)

pytest

Test definition

def test_addition():
    x = 2 * 2
    y = 5
    assert x == y

unittest output

Traceback (most recent call last):
  File "test.py", line 89, in test_addition
    assert x == y
AssertionError

pytest output

> x == y
E assert 8 == 9
E  +  where 8 = <bound method ...
E  +    where <bound method ...
E  +      where <test_dependencies ...

File system hint

ls -lA tests/__pycache__/
total 304
test_dependencies.cpython-26-PYTEST.pyc
test_dependencies.cpython-27-PYTEST.pyc
test_dependencies.cpython-34-PYTEST.pyc
test_dependencies.cpython-35-PYTEST.pyc

Import hook

class AssertionRewritingHook(object):
    def find_module(self, name, path=None):
        fn = imp.find_module(name)
        cache_name = fn[:-3] + 'PYTEST.pyc'
        co = _read_pyc(fn)
        co = _rewrite_test(fn, co)
        _make_rewritten_pyc(co, cache_name)

sys.meta_path.append(AssertionRewritingHook())

Import hook

class AssertionRewritingHook(object):
    def find_module(self, name, path=None):
        fn = imp.find_module(name)
        cache_name = fn[:-3] + 'PYTEST.pyc'
        co = _read_pyc(fn)
        co = _rewrite_test(fn, co)
        _make_rewritten_pyc(co, cache_name)

sys.meta_path.append(AssertionRewritingHook())

ast rewrite

import ast

class Int2Str(ast.NodeTransformer):

    def visit_Num(self, node):
        return ast.copy_location(
            ast.Str(s=str(node.n)),
            node,
        )

how it works

exp = 'c = [1, 2, 3]'
exec(exp)
print(c)
# [1, 2, 3]
ast = ast.parse(exp)
new_exp = Int2Str().visit(ast)
exec(compile(new_exp))
print(c)
# ['1', '2', '3']

jinja2

html in the python traceback

Traceback (most recent call last):
  File "jinja2/environment.py", line 989
    return handle_exception(exc_info, True)
  File "jinja2/environment.py", line 754
    reraise(exc_type, exc_value, tb)
  File "jinja2/_compat.py", line 37
    raise value.with_traceback(tb)
  File "templates/index.html", line 5
    {{ test() }}
UndefinedError: 'test' is undefined

original error

try:
    hello # <- NameError here
except Exception as e:
    tb = make_tb()
    raise e.with_traceback(tb)

fake error location

def make_tb():
    exc_type, exc_value, tb = sys.exc_info()
    filename = 'xxx.html'
    src = '\n' * 22 + 'raise exc_value'
    code = compile(src, filename, 'exec')
    try:
        exec(code, {'exc_value': exc_type()})
    except:
        return sys.exc_info()[2].tb_next

fake error location

def make_tb():
    exc_type, exc_value, tb = sys.exc_info()
    filename = 'xxx.html'
    src = '\n' * 22 + 'raise exc_value'
    code = compile(src, filename, 'exec')
    try:
        exec(code, {'exc_value': exc_type()})
    except:
        return sys.exc_info()[2].tb_next

Faked

Traceback (most recent call last):
  File "tb_rewrite.py", line 25, in <module>
    raise e.with_traceback(tb)
  File "xxx.html", line 22, in html
    {{ hello }}
NameError: name 'hello' is not defined

skip internal frames

try:
    tb_set_next = _init_ugly_crap()
except:
    pass
del _init_ugly_crap

we have to deal with this crap...

import ctypes

class _PyObject(ctypes.Structure):
    pass

class _Traceback(_PyObject):
    pass

_Traceback._fields_ = [
    ('tb_next', ctypes.POINTER(_Traceback)),
    ('tb_frame', ctypes.POINTER(_PyObject)),
]

Let's patch immutable object

def tb_set_next(tb, next):

    obj = _Traceback.from_address(id(tb))
    old = _Traceback.from_address(id(tb.tb_next))
    old.ob_refcnt -= 1
    next = _Traceback.from_address(id(next))
    next.ob_refcnt += 1
    obj.tb_next = ctypes.pointer(next)

It's magic.

Isn't it?

Questions?