python,scons: Move grammar.py and code_formatter.py into build_tools.
These are only used in a build, and so don't need to be built into gem5. grammar.py is used by slicc and the fast model project file parser, and code_formatter.py is only used by SConscripts. Change-Id: Id43e62459d69f07fdb2ed125548a83e38bbb7590 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/49396 Tested-by: kokoro <noreply+kokoro@google.com> Reviewed-by: Hoa Nguyen <hoanguyen@ucdavis.edu> Maintainer: Gabe Black <gabe.black@gmail.com>
This commit is contained in:
Gabe Black
346
build_tools/code_formatter.py
Normal file
346
build_tools/code_formatter.py
Normal file
@@ -0,0 +1,346 @@
|
||||
# Copyright (c) 2006-2009 Nathan Binkert <nate@binkert.org>
|
||||
# All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are
|
||||
# met: redistributions of source code must retain the above copyright
|
||||
# notice, this list of conditions and the following disclaimer;
|
||||
# redistributions in binary form must reproduce the above copyright
|
||||
# notice, this list of conditions and the following disclaimer in the
|
||||
# documentation and/or other materials provided with the distribution;
|
||||
# neither the name of the copyright holders nor the names of its
|
||||
# contributors may be used to endorse or promote products derived from
|
||||
# this software without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
try:
|
||||
import builtins
|
||||
except ImportError:
|
||||
# Python 2 fallback
|
||||
import __builtin__ as builtins
|
||||
import inspect
|
||||
import os
|
||||
import re
|
||||
|
||||
class lookup(object):
|
||||
def __init__(self, formatter, frame, *args, **kwargs):
|
||||
self.frame = frame
|
||||
self.formatter = formatter
|
||||
self.dict = self.formatter._dict
|
||||
self.args = args
|
||||
self.kwargs = kwargs
|
||||
self.locals = {}
|
||||
|
||||
def __setitem__(self, item, val):
|
||||
self.locals[item] = val
|
||||
|
||||
def __getitem__(self, item):
|
||||
if item in self.locals:
|
||||
return self.locals[item]
|
||||
|
||||
if item in self.kwargs:
|
||||
return self.kwargs[item]
|
||||
|
||||
if item == '__file__':
|
||||
return self.frame.f_code.co_filename
|
||||
|
||||
if item == '__line__':
|
||||
return self.frame.f_lineno
|
||||
|
||||
if self.formatter.locals and item in self.frame.f_locals:
|
||||
return self.frame.f_locals[item]
|
||||
|
||||
if item in self.dict:
|
||||
return self.dict[item]
|
||||
|
||||
if self.formatter.globals and item in self.frame.f_globals:
|
||||
return self.frame.f_globals[item]
|
||||
|
||||
if item in builtins.__dict__:
|
||||
return builtins.__dict__[item]
|
||||
|
||||
try:
|
||||
item = int(item)
|
||||
return self.args[item]
|
||||
except ValueError:
|
||||
pass
|
||||
raise IndexError("Could not find '%s'" % item)
|
||||
|
||||
class code_formatter_meta(type):
|
||||
pattern = r"""
|
||||
(?:
|
||||
%(delim)s(?P<escaped>%(delim)s) | # escaped delimiter
|
||||
^(?P<indent>[ ]*)%(delim)s(?P<lone>%(ident)s)$ | # lone identifier
|
||||
%(delim)s(?P<ident>%(ident)s) | # identifier
|
||||
%(delim)s%(lb)s(?P<b_ident>%(ident)s)%(rb)s | # braced identifier
|
||||
%(delim)s(?P<pos>%(pos)s) | # positional parameter
|
||||
%(delim)s%(lb)s(?P<b_pos>%(pos)s)%(rb)s | # braced positional
|
||||
%(delim)s%(ldb)s(?P<eval>.*?)%(rdb)s | # double braced expression
|
||||
%(delim)s(?P<invalid>) # ill-formed delimiter exprs
|
||||
)
|
||||
"""
|
||||
def __init__(cls, name, bases, dct):
|
||||
super(code_formatter_meta, cls).__init__(name, bases, dct)
|
||||
if 'pattern' in dct:
|
||||
pat = cls.pattern
|
||||
else:
|
||||
# tuple expansion to ensure strings are proper length
|
||||
lb,rb = cls.braced
|
||||
lb1,lb2,rb2,rb1 = cls.double_braced
|
||||
pat = code_formatter_meta.pattern % {
|
||||
'delim' : re.escape(cls.delim),
|
||||
'ident' : cls.ident,
|
||||
'pos' : cls.pos,
|
||||
'lb' : re.escape(lb),
|
||||
'rb' : re.escape(rb),
|
||||
'ldb' : re.escape(lb1+lb2),
|
||||
'rdb' : re.escape(rb2+rb1),
|
||||
}
|
||||
cls.pattern = re.compile(pat, re.VERBOSE | re.DOTALL | re.MULTILINE)
|
||||
|
||||
class code_formatter(object, metaclass=code_formatter_meta):
|
||||
delim = r'$'
|
||||
ident = r'[_A-z]\w*'
|
||||
pos = r'[0-9]+'
|
||||
braced = r'{}'
|
||||
double_braced = r'{{}}'
|
||||
|
||||
globals = True
|
||||
locals = True
|
||||
fix_newlines = True
|
||||
def __init__(self, *args, **kwargs):
|
||||
self._data = []
|
||||
self._dict = {}
|
||||
self._indent_level = 0
|
||||
self._indent_spaces = 4
|
||||
self.globals = kwargs.pop('globals', type(self).globals)
|
||||
self.locals = kwargs.pop('locals', type(self).locals)
|
||||
self._fix_newlines = \
|
||||
kwargs.pop('fix_newlines', type(self).fix_newlines)
|
||||
|
||||
if args:
|
||||
self.__call__(args)
|
||||
|
||||
def indent(self, count=1):
|
||||
self._indent_level += self._indent_spaces * count
|
||||
|
||||
def dedent(self, count=1):
|
||||
assert self._indent_level >= (self._indent_spaces * count)
|
||||
self._indent_level -= self._indent_spaces * count
|
||||
|
||||
def fix(self, status):
|
||||
previous = self._fix_newlines
|
||||
self._fix_newlines = status
|
||||
return previous
|
||||
|
||||
def nofix(self):
|
||||
previous = self._fix_newlines
|
||||
self._fix_newlines = False
|
||||
return previous
|
||||
|
||||
def clear():
|
||||
self._data = []
|
||||
|
||||
def write(self, *args):
|
||||
f = open(os.path.join(*args), "w")
|
||||
name, extension = os.path.splitext(f.name)
|
||||
|
||||
# Add a comment to inform which file generated the generated file
|
||||
# to make it easier to backtrack and modify generated code
|
||||
frame = inspect.currentframe().f_back
|
||||
if re.match('\.(cc|hh|c|h)', extension) is not None:
|
||||
f.write(f'''/**
|
||||
* DO NOT EDIT THIS FILE!
|
||||
* File automatically generated by
|
||||
* {frame.f_code.co_filename}:{frame.f_lineno}
|
||||
*/
|
||||
|
||||
''')
|
||||
elif re.match('\.py', extension) is not None:
|
||||
f.write(f'''#
|
||||
# DO NOT EDIT THIS FILE!
|
||||
# File automatically generated by
|
||||
# {frame.f_code.co_filename}:{frame.f_lineno}
|
||||
#
|
||||
|
||||
''')
|
||||
elif re.match('\.html', extension) is not None:
|
||||
f.write(f'''<!--
|
||||
DO NOT EDIT THIS FILE!
|
||||
File automatically generated by
|
||||
{frame.f_code.co_filename}:{frame.f_lineno}
|
||||
-->
|
||||
|
||||
''')
|
||||
|
||||
for data in self._data:
|
||||
f.write(data)
|
||||
f.close()
|
||||
|
||||
def __str__(self):
|
||||
data = ''.join(self._data)
|
||||
self._data = [ data ]
|
||||
return data
|
||||
|
||||
def __getitem__(self, item):
|
||||
return self._dict[item]
|
||||
|
||||
def __setitem__(self, item, value):
|
||||
self._dict[item] = value
|
||||
|
||||
def __delitem__(self, item):
|
||||
del self._dict[item]
|
||||
|
||||
def __contains__(self, item):
|
||||
return item in self._dict
|
||||
|
||||
def __iadd__(self, data):
|
||||
self.append(data)
|
||||
|
||||
def append(self, data):
|
||||
if isinstance(data, code_formatter):
|
||||
self._data.extend(data._data)
|
||||
else:
|
||||
self._append(str(data))
|
||||
|
||||
def _append(self, data):
|
||||
if not self._fix_newlines:
|
||||
self._data.append(data)
|
||||
return
|
||||
|
||||
initial_newline = not self._data or self._data[-1] == '\n'
|
||||
for line in data.splitlines():
|
||||
if line:
|
||||
if self._indent_level:
|
||||
self._data.append(' ' * self._indent_level)
|
||||
self._data.append(line)
|
||||
|
||||
if line or not initial_newline:
|
||||
self._data.append('\n')
|
||||
|
||||
initial_newline = False
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
if not args:
|
||||
self._data.append('\n')
|
||||
return
|
||||
|
||||
format = args[0]
|
||||
args = args[1:]
|
||||
|
||||
frame = inspect.currentframe().f_back
|
||||
|
||||
l = lookup(self, frame, *args, **kwargs)
|
||||
def convert(match):
|
||||
ident = match.group('lone')
|
||||
# check for a lone identifier
|
||||
if ident:
|
||||
indent = match.group('indent') # must be spaces
|
||||
lone = '%s' % (l[ident], )
|
||||
|
||||
def indent_lines(gen):
|
||||
for line in gen:
|
||||
yield indent
|
||||
yield line
|
||||
return ''.join(indent_lines(lone.splitlines(True)))
|
||||
|
||||
# check for an identifier, braced or not
|
||||
ident = match.group('ident') or match.group('b_ident')
|
||||
if ident is not None:
|
||||
return '%s' % (l[ident], )
|
||||
|
||||
# check for a positional parameter, braced or not
|
||||
pos = match.group('pos') or match.group('b_pos')
|
||||
if pos is not None:
|
||||
pos = int(pos)
|
||||
if pos > len(args):
|
||||
raise ValueError \
|
||||
('Positional parameter #%d not found in pattern' % pos,
|
||||
code_formatter.pattern)
|
||||
return '%s' % (args[int(pos)], )
|
||||
|
||||
# check for a double braced expression
|
||||
eval_expr = match.group('eval')
|
||||
if eval_expr is not None:
|
||||
result = eval(eval_expr, {}, l)
|
||||
return '%s' % (result, )
|
||||
|
||||
# check for an escaped delimiter
|
||||
if match.group('escaped') is not None:
|
||||
return '$'
|
||||
|
||||
# At this point, we have to match invalid
|
||||
if match.group('invalid') is None:
|
||||
# didn't match invalid!
|
||||
raise ValueError('Unrecognized named group in pattern',
|
||||
code_formatter.pattern)
|
||||
|
||||
i = match.start('invalid')
|
||||
if i == 0:
|
||||
colno = 1
|
||||
lineno = 1
|
||||
else:
|
||||
lines = format[:i].splitlines(True)
|
||||
colno = i - sum(len(z) for z in lines)
|
||||
lineno = len(lines)
|
||||
|
||||
raise ValueError('Invalid format string: line %d, col %d' %
|
||||
(lineno, colno))
|
||||
|
||||
d = code_formatter.pattern.sub(convert, format)
|
||||
self._append(d)
|
||||
|
||||
__all__ = [ "code_formatter" ]
|
||||
|
||||
if __name__ == '__main__':
|
||||
from .code_formatter import code_formatter
|
||||
f = code_formatter()
|
||||
|
||||
class Foo(dict):
|
||||
def __init__(self, **kwargs):
|
||||
self.update(kwargs)
|
||||
def __getattr__(self, attr):
|
||||
return self[attr]
|
||||
|
||||
x = "this is a test"
|
||||
l = [ [Foo(x=[Foo(y=9)])] ]
|
||||
|
||||
y = code_formatter()
|
||||
y('''
|
||||
{
|
||||
this_is_a_test();
|
||||
}
|
||||
''')
|
||||
f(' $y')
|
||||
f('''$__file__:$__line__
|
||||
{''')
|
||||
f("${{', '.join(str(x) for x in range(4))}}")
|
||||
f('${x}')
|
||||
f('$x')
|
||||
f.indent()
|
||||
for i in range(5):
|
||||
f('$x')
|
||||
f('$i')
|
||||
f('$0', "zero")
|
||||
f('$1 $0', "zero", "one")
|
||||
f('${0}', "he went")
|
||||
f('${0}asdf', "he went")
|
||||
f.dedent()
|
||||
|
||||
f('''
|
||||
${{l[0][0]["x"][0].y}}
|
||||
}
|
||||
''', 1, 9)
|
||||
|
||||
print(f, end=' ')
|
||||
137
build_tools/grammar.py
Normal file
137
build_tools/grammar.py
Normal file
@@ -0,0 +1,137 @@
|
||||
# Copyright (c) 2006-2011 Nathan Binkert <nate@binkert.org>
|
||||
# All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are
|
||||
# met: redistributions of source code must retain the above copyright
|
||||
# notice, this list of conditions and the following disclaimer;
|
||||
# redistributions in binary form must reproduce the above copyright
|
||||
# notice, this list of conditions and the following disclaimer in the
|
||||
# documentation and/or other materials provided with the distribution;
|
||||
# neither the name of the copyright holders nor the names of its
|
||||
# contributors may be used to endorse or promote products derived from
|
||||
# this software without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
import os
|
||||
|
||||
import ply.lex
|
||||
import ply.yacc
|
||||
|
||||
class ParseError(Exception):
|
||||
def __init__(self, message, token=None):
|
||||
Exception.__init__(self, message)
|
||||
self.token = token
|
||||
|
||||
class Grammar(object):
|
||||
def setupLexerFactory(self, **kwargs):
|
||||
if 'module' in kwargs:
|
||||
raise AttributeError("module is an illegal attribute")
|
||||
self.lex_kwargs = kwargs
|
||||
|
||||
def setupParserFactory(self, **kwargs):
|
||||
if 'module' in kwargs:
|
||||
raise AttributeError("module is an illegal attribute")
|
||||
|
||||
if 'output' in kwargs:
|
||||
dir,tab = os.path.split(output)
|
||||
if not tab.endswith('.py'):
|
||||
raise AttributeError('The output file must end with .py')
|
||||
kwargs['outputdir'] = dir
|
||||
kwargs['tabmodule'] = tab[:-3]
|
||||
|
||||
self.yacc_kwargs = kwargs
|
||||
|
||||
def __getattr__(self, attr):
|
||||
if attr == 'lexers':
|
||||
self.lexers = []
|
||||
return self.lexers
|
||||
|
||||
if attr == 'lex_kwargs':
|
||||
self.setupLexerFactory()
|
||||
return self.lex_kwargs
|
||||
|
||||
if attr == 'yacc_kwargs':
|
||||
self.setupParserFactory()
|
||||
return self.yacc_kwargs
|
||||
|
||||
if attr == 'lex':
|
||||
self.lex = ply.lex.lex(module=self, **self.lex_kwargs)
|
||||
return self.lex
|
||||
|
||||
if attr == 'yacc':
|
||||
self.yacc = ply.yacc.yacc(module=self, **self.yacc_kwargs)
|
||||
return self.yacc
|
||||
|
||||
if attr == 'current_lexer':
|
||||
if not self.lexers:
|
||||
return None
|
||||
return self.lexers[-1][0]
|
||||
|
||||
if attr == 'current_source':
|
||||
if not self.lexers:
|
||||
return '<none>'
|
||||
return self.lexers[-1][1]
|
||||
|
||||
if attr == 'current_line':
|
||||
if not self.lexers:
|
||||
return -1
|
||||
return self.current_lexer.lineno
|
||||
|
||||
raise AttributeError(
|
||||
"'%s' object has no attribute '%s'" % (type(self), attr))
|
||||
|
||||
def parse_string(self, data, source='<string>', debug=None, tracking=0):
|
||||
if not isinstance(data, str):
|
||||
raise AttributeError(
|
||||
"argument must be a string, was '%s'" % type(f))
|
||||
|
||||
lexer = self.lex.clone()
|
||||
lexer.input(data)
|
||||
self.lexers.append((lexer, source))
|
||||
|
||||
lrtab = ply.yacc.LRTable()
|
||||
lrtab.lr_productions = self.yacc.productions
|
||||
lrtab.lr_action = self.yacc.action
|
||||
lrtab.lr_goto = self.yacc.goto
|
||||
|
||||
parser = ply.yacc.LRParser(lrtab, self.yacc.errorfunc)
|
||||
result = parser.parse(lexer=lexer, debug=debug, tracking=tracking)
|
||||
self.lexers.pop()
|
||||
return result
|
||||
|
||||
def parse_file(self, f, **kwargs):
|
||||
if isinstance(f, str):
|
||||
source = f
|
||||
f = open(f, 'r')
|
||||
elif isinstance(f, file):
|
||||
source = f.name
|
||||
else:
|
||||
raise AttributeError(
|
||||
"argument must be either a string or file, was '%s'" % type(f))
|
||||
|
||||
return self.parse_string(f.read(), source, **kwargs)
|
||||
|
||||
def p_error(self, t):
|
||||
if t:
|
||||
msg = "Syntax error at %s:%d:%d\n>>%s<<" % \
|
||||
(self.current_source, t.lineno, t.lexpos + 1, t.value)
|
||||
else:
|
||||
msg = "Syntax error at end of %s" % (self.current_source, )
|
||||
raise ParseError(msg, t)
|
||||
|
||||
def t_error(self, t):
|
||||
msg = "Illegal character %s @ %d:%d" % \
|
||||
(repr(t.value[0]), t.lineno, t.lexpos)
|
||||
raise ParseError(msg, t)
|
||||
Reference in New Issue
Block a user