pyconru 2016. Осторожно, dsl!
TRANSCRIPT
Осторожно, DSL!
Цыганов Иван Positive Technologies
server { location / { proxy_pass http://localhost:8080/; }
location ~ \.(gif|jpg|png)$ { root /data/images; }}
version: '2'services: db: image: postgres web: build: . command: python manage.py runserver 0.0.0.0:8000 volumes: - .:/code ports: - "8000:8000" depends_on: - db
DOCKER COMPOSE
LOGROTATE
compress
"/var/log/httpd/access.log" /var/log/httpd/error.log { rotate 5 size=100k sharedscripts nocompress postrotate /sbin/killall -HUP httpd endscript}
/var/log/messages { rotate 5 weekly postrotate /sbin/killall -HUP syslogd endscript}
compress
"/var/log/httpd/access.log" /var/log/httpd/error.log { rotate 5 size=100k sharedscripts nocompress postrotate /sbin/killall -HUP httpd endscript}
config = { 'compress': True, ('/var/log/nginx/access.log', '/var/log/nginx/error.log'): { 'rotate': 5, 'size': 102400, 'compress': False, 'sharedscripts': True, 'postrotate': [ '/sbin/killall -HUP syslogd' ] }}
compress
"/var/log/httpd/access.log" /var/log/httpd/error.log { rotate 5 size=100k sharedscripts nocompress postrotate /sbin/killall -HUP httpd endscript}
config = { 'compress': True, ('/var/log/nginx/access.log', '/var/log/nginx/error.log'): { 'rotate': 5, 'size': 102400, 'compress': False, 'sharedscripts': True, 'postrotate': [ '/sbin/killall -HUP syslogd' ] }}
compress
"/var/log/httpd/access.log" /var/log/httpd/error.log { rotate 5 size=100k sharedscripts nocompress postrotate /sbin/killall -HUP httpd endscript}
size=100k
'size': 102400,
config = { 'compress': True, ('/var/log/nginx/access.log', '/var/log/nginx/error.log'): { 'rotate': 5, 'size': 102400, 'postrotate': [ '/sbin/killall -HUP syslogd' ] }}
'size': '100 KB',
config = { 'compress': True, ('/var/log/nginx/access.log', '/var/log/nginx/error.log'): { 'rotate': 5, 'size': 102400, 'postrotate': [ '/sbin/killall -HUP syslogd' ] }}
'size': '100 KB',
def get_size(self, value): size_value, _, size_unit = value.partition(' ') return self.units_to_bytes(size_value, size_unit)
config = { 'compress': True, ('/var/log/nginx/access.log', '/var/log/nginx/error.log'): { 'rotate': 5, 'size': 102400, 'postrotate': [ '/sbin/killall -HUP syslogd' ] }}
'size': '1MB + 100KB',
config = { 'compress': True, ('/var/log/nginx/access.log', '/var/log/nginx/error.log'): { 'rotate': 5, 'size': 102400, 'postrotate': [ '/sbin/killall -HUP syslogd' ] }}
'size': '1MB + 100KB',
parse_size = re.compile( pattern=r''' (?P<Value>\d+)\s* (?P<ValueUnit>KB|MB|GB)?\s* ((?P<Operator>[-+/*])\s*)? ( (?P<Delta>\d+)\s* (?P<DeltaUnit>KB|MB|GB)? )? ''', flags=re.IGNORECASE | re.VERBOSE).search
config = { 'compress': True, ('/var/log/nginx/access.log', '/var/log/nginx/error.log'): { 'rotate': 5, 'size': 102400, 'postrotate': [ '/sbin/killall -HUP syslogd' ] }}
'size': ‘100MB - 2 * (1MB + 100KB)’,
Some people, when confronted with a problem, think "I know, I’ll use regular expressions." Now they have two problems.
— Jamie Zawinski
Осторожно, DSL!
– Мартин Фаулер
Предметно-ориентированный язык — это язык программирования с ограниченными выразительными возможностями, ориентированный на некую конкретную предметную область
✤ SQL✤ REGEXP✤ TeX/LaTeX
Виды DSL
DSL
ВнутренниеВнешние✤ PonyORM✤ WTForm✤ Django models
Что будет дальше?
✤ Внутренние DSL
✤ Внешние DSL
✤ Инструменты для создания анализаторов
Внутренние DSL
Все возможности базового языка
Привычный синтаксис
Ограничен базовым языком
'size': 100MB - 2 * (1MB - 100KB)
GB = lambda x: x*2**30 MB = lambda x: x*2**20 KB = lambda x: x*2**10
GB = 2**30 MB = 2**20 KB = 2**10
config = { (‘/var/log/nginx/access.log',): { 'size': 100*MB - 2 * (1*MB - 100*KB) }, (‘/var/log/nginx/error.log',): { 'size': 100*MB - 2 * (1*MB - 100*KB) } }
'size': MB(100) - 2*(MB(1) + KB(100))
'size': 100*MB - 2 * (1*MB - 100*KB)
Внешние DSL
Сами выбираем синтаксис
Необходимо разрабатывать анализаторы
compress: truerules:- files: [/var/log/nginx/access.log, /var/log/nginx/error.log] rotate: 5 size: 100MB - 2 * (1MB + 100KB) postrotate: [/sbin/killall -HUP syslogd]
config = { 'compress': True, ('/var/log/nginx/access.log', '/var/log/nginx/error.log'): { 'rotate': 5, 'size': 100*MB - 2 * (1*MB - 100*KB) 'postrotate': [ '/sbin/killall -HUP syslogd' ] }}
Python Lex-Yacc (PLY)
ply.lex
ply.yacc
ASTRunCode
source
ply.lex
Type ValuePLUS + MINUS - MUL *DIV /UNIT GB|MB|KB|BDIGIT \d+LPAREN (RPAREN )
ply.lex 2 * (1KB + 1KB)
DIGIT = 2
MUL = *
LPAREN = (
DIGIT = 1
UNIT = KB
PLUS = +
DIGIT = 1
UNIT = KB
RPAREN = )
ply.lex 1KB1KB-MB
DIGIT = 1
DIGIT = 1
UNIT = KB
MINUS = -
UNIT = MB
UNIT = KB
ply.yacc
expression : expression PLUS expression | expression MINUS expression | expression MUL expression | expression DIV expression
expression : LPAREN expression RPAREN
expression : DIGIT UNIT
expression : DIGIT
ply.yacc
def p_expression(self, p): ''' expression : expression PLUS expression | expression MINUS expression | expression MUL expression | expression DIV expression ''' if p[2] == '+': p[0] = p[1] + p[3] if p[2] == '-': p[0] = p[1] - p[3] if p[2] == '*': p[0] = p[1] * p[3] if p[2] == '/': p[0] = p[1] / p[3]
ply.yacc
precedence = ( ('left', 'PLUS', 'MINUS'), ('left', 'MUL', 'DIV'), )
def p_expression(self, p): ''' expression : expression PLUS expression | expression MINUS expression | expression MUL expression | expression DIV expression ''' if p[2] == '+': p[0] = p[1] + p[3] if p[2] == '-': p[0] = p[1] - p[3] if p[2] == '*': p[0] = p[1] * p[3] if p[2] == '/': p[0] = p[1] / p[3]
PLY
Гибкость
Отладка
Обработка ошибок
Понятный код библиотеки
Высокий порог входа
Многословный
funcparserlib
def tokenize(str): specs = [ ('Spaces', (r'[ \s\t\r\n]+',)), ('PLUS', (r'\+',)), ('MINUS', (r'-',)), ('MUL', (r'\*',)), ('DIV', (r'/',)), ('UNIT', (r'GB|MB|KB|B',)), ('DIGIT', (r'\d+',)), ('LPAREN', (r'\(',)), ('RPAREN', (r'\)',)), ] return list(filter( lambda t: t.type != 'Spaces', (t for t in make_tokenizer(specs)(str)) ))
funcparserlib
number = value_of('DIGIT') >> intunit = number + value_of('UNIT') >> to_bytesoperand = unit | numbermakeop = lambda s, f: skip_token(s) >> const(f)add = makeop('PLUS', operator.add)sub = makeop('MINUS', operator.sub)mul = makeop('MUL', operator.mul)div = makeop('DIV', operator.floordiv) mul_op = mul | divadd_op = add | sub
funcparserlib
primary = with_forward_decls( lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN')) ) term = primary + many(mul_op + primary) >> eval_exprexpr = term + many(add_op + term) >> eval_exprparser = maybe(expr)
funcparserlib
primary = with_forward_decls( lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN')) ) term = primary + many(mul_op + primary) >> eval_exprexpr = term + many(add_op + term) >> eval_exprparser = maybe(expr)
funcparserlib
2+2*2
primary = with_forward_decls( lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN')) ) term = primary + many(mul_op + primary) >> eval_exprexpr = term + many(add_op + term) >> eval_exprparser = maybe(expr)
funcparserlib
2+2*2
maybe(expr)
primary = with_forward_decls( lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN')) ) term = primary + many(mul_op + primary) >> eval_exprexpr = term + many(add_op + term) >> eval_exprparser = maybe(expr)
funcparserlib
2+2*2
term
primary = with_forward_decls( lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN')) ) term = primary + many(mul_op + primary) >> eval_exprexpr = term + many(add_op + term) >> eval_exprparser = maybe(expr)
funcparserlib
2+2*2
primary
primary = with_forward_decls( lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN')) ) term = primary + many(mul_op + primary) >> eval_exprexpr = term + many(add_op + term) >> eval_exprparser = maybe(expr)
funcparserlib
2+2*22
operand
primary = with_forward_decls( lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN')) ) term = primary + many(mul_op + primary) >> eval_exprexpr = term + many(add_op + term) >> eval_exprparser = maybe(expr)
funcparserlib
2+2*22
many(mul_op + primary)
primary = with_forward_decls( lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN')) ) term = primary + many(mul_op + primary) >> eval_exprexpr = term + many(add_op + term) >> eval_exprparser = maybe(expr)
funcparserlib
many(add_op + term)
2+2*22+
primary = with_forward_decls( lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN')) ) term = primary + many(mul_op + primary) >> eval_exprexpr = term + many(add_op + term) >> eval_exprparser = maybe(expr)
funcparserlib
primary
2+2*22+
primary = with_forward_decls( lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN')) ) term = primary + many(mul_op + primary) >> eval_exprexpr = term + many(add_op + term) >> eval_exprparser = maybe(expr)
funcparserlib
operand
2+2*22+2
primary = with_forward_decls( lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN')) ) term = primary + many(mul_op + primary) >> eval_exprexpr = term + many(add_op + term) >> eval_exprparser = maybe(expr)
funcparserlib
2+2*22+2*
many(mul_op + primary)
primary = with_forward_decls( lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN')) ) term = primary + many(mul_op + primary) >> eval_exprexpr = term + many(add_op + term) >> eval_exprparser = maybe(expr)
funcparserlib
operand
2+2*22+2*2
primary = with_forward_decls( lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN')) ) term = primary + many(mul_op + primary) >> eval_exprexpr = term + many(add_op + term) >> eval_exprparser = maybe(expr)
funcparserlib
eval_expr
2+2*22+2*2
2+4
primary = with_forward_decls( lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN')) ) term = primary + many(mul_op + primary) >> eval_exprexpr = term + many(add_op + term) >> eval_exprparser = maybe(expr)
funcparserlib
eval_expr
6
primary = with_forward_decls( lambda: operand | (skip_token('LPAREN') + expr + skip_token('RPAREN')) ) term = primary + many(mul_op + primary) >> eval_exprexpr = term + many(add_op + term) >> eval_exprparser = maybe(expr)
funcparserlib
maybe(expr)
funcparserlib
Компактный
Гибкий
Для любителей функционального программирования :)
Многое приходится делать руками
Для любителей функционального программирования :)
pyparsing
pyparsing
plusop = oneOf('+ -') multop = oneOf('* /') digit = Word(nums)unit = digit + oneOf('GB MB KB’) operand = unit | digitparser = Forward()primary = operand | Literal('(') + parser + Literal(')') term = (primary + ZeroOrMore(multop + primary)).setParseAction(eval_expr)expr = (term + ZeroOrMore(plusop + term)).setParseAction(eval_expr)parser << Optional(expr)
pyparsing
plusop = oneOf('+ -') multop = oneOf('* /')
digit = Word(nums)unit = digit + oneOf('GB MB KB’)
operand = unit | digit parser = operatorPrecedence( operand, [ (multop, 2, opAssoc.LEFT, calculate), (plusop, 2, opAssoc.LEFT, calculate) ])
pyparsing
Понятный код
Базовые компоненты
Свои компоненты
Документация
Отладка
А что насчет быстродействия?
Скорость
Скорость
А что же пользователи?
Сообщения об ошибках
Traceback (most recent call last): File "size_parser/on_ply.py", line 100, in parse return parser.parse(string) File "size_parser/on_ply.py", line 94, in parse p = self._parser.parse(data, debug=self._debug) File ".env/site-packages/ply/yacc.py", line 331, in parse return self.parseopt_notrack(input, lexer, debug, tracking, tokenfunc) File ".env/site-packages/ply/yacc.py", line 1049, in parseopt_notrack lookahead = get_token() # Get the next token File ".env/site-packages/ply/lex.py", line 396, in token raise LexError("Illegal character '%s' at index %d" % (lexdata[lexpos], lexpos), lexdata[lexpos:]) ply.lex.LexError: Illegal character '_' at index 5
Traceback (most recent call last): File "size_parser/on_ply.py", line 100, in parse return parser.parse(string) File "size_parser/on_ply.py", line 94, in parse p = self._parser.parse(data, debug=self._debug) File ".env/site-packages/ply/yacc.py", line 331, in parse return self.parseopt_notrack(input, lexer, debug, tracking, tokenfunc) File ".env/site-packages/ply/yacc.py", line 1049, in parseopt_notrack lookahead = get_token() # Get the next token File ".env/site-packages/ply/lex.py", line 396, in token raise LexError("Illegal character '%s' at index %d" % (lexdata[lexpos], lexpos), lexdata[lexpos:]) ply.lex.LexError: Illegal character '_' at index 5
Сообщения об ошибках
ply
def t_error(self, t): raise ParserException(t, self.source)def p_error(self, p): raise ParserException(p, self.source)
>>> parse(‘1MB+_GB-100KB’)Unexpected "_GB-100KB" at position 4:1MB+_GB-100KB ^^^
Так что же выбрать?
Что же выбрать?
pyparsing✤ Хочу легко все описать✤ Быстродействие не главное
Что же выбрать?
funcparserlib
pyparsing✤ Хочу легко все описать✤ Быстродействие не главное
✤ Люблю функциональное программирование
✤ Быстродействие не главное
Что же выбрать?
PLY
funcparserlib
pyparsing
✤ Хочу как в учебнике✤ Скорость работы - главное!
✤ Хочу легко все описать✤ Быстродействие не главное
✤ Люблю функциональное программирование
✤ Быстродействие не главное
И что в итоге?
✤ Для простых задач попробуйте:
✤ Средства самого языка
✤ Регулярные выражения
✤ Если задача сложная:
✤ Внутренние DSL
✤ Yaml, Json, XML, …
✤ Внешние DSL
Спасибо за внимание!Вопросы?
http://tsyganov-ivan.com/