diff options
| -rw-r--r-- | LICENSE.txt | 2 | ||||
| -rw-r--r-- | PKG-INFO | 12 | ||||
| -rw-r--r-- | README.rst | 7 | ||||
| -rwxr-xr-x | bin/jp | 70 | ||||
| -rw-r--r-- | debian/changelog | 6 | ||||
| -rw-r--r-- | jmespath.egg-info/PKG-INFO | 12 | ||||
| -rw-r--r-- | jmespath.egg-info/SOURCES.txt | 2 | ||||
| -rw-r--r-- | jmespath.egg-info/requires.txt | 1 | ||||
| -rw-r--r-- | jmespath/__init__.py | 10 | ||||
| -rw-r--r-- | jmespath/ast.py | 727 | ||||
| -rw-r--r-- | jmespath/compat.py | 33 | ||||
| -rw-r--r-- | jmespath/exceptions.py | 106 | ||||
| -rw-r--r-- | jmespath/lexer.py | 191 | ||||
| -rw-r--r-- | jmespath/parser.py | 605 | ||||
| -rw-r--r-- | setup.py | 28 | ||||
| -rw-r--r-- | tests/__init__.py | 24 | ||||
| -rw-r--r-- | tests/test_ast.py | 351 | ||||
| -rw-r--r-- | tests/test_compliance.py | 64 | ||||
| -rw-r--r-- | tests/test_parser.py | 74 |
19 files changed, 1738 insertions, 587 deletions
diff --git a/LICENSE.txt b/LICENSE.txt index 07d9e8c..aa68928 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,3 +1,5 @@ +Copyright (c) 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved + Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: jmespath -Version: 0.2.1 +Version: 0.4.1 Summary: JSON Matching Expressions Home-page: https://github.com/boto/jmespath Author: James Saryerwinnie @@ -9,7 +9,7 @@ License: UNKNOWN Description: JMESPath ======== - JMESPath (pronounced "jaymz path") allows you to declaratively specify how to + JMESPath (pronounced ``\ˈjāmz path\``) allows you to declaratively specify how to extract elements from a JSON document. For example, given this document:: @@ -33,7 +33,7 @@ Description: JMESPath The expression: ``foo.bar[*].name`` will return ["one", "two"]. Negative indexing is also supported (-1 refers to the last element in the list). Given the data above, the expression - ``foo.bar[-1].name`` will return ["two"]. + ``foo.bar[-1].name`` will return "two". The ``*`` can also be used for hash types:: @@ -90,11 +90,8 @@ Description: JMESPath You can also use the ``jmespath.parser.Parser`` class directly if you want more control. - - .. _RFC4234: http://tools.ietf.org/html/rfc4234 - Platform: UNKNOWN -Classifier: Development Status :: 3 - Alpha +Classifier: Development Status :: 5 - Production/Stable Classifier: Intended Audience :: Developers Classifier: Natural Language :: English Classifier: License :: OSI Approved :: MIT License @@ -102,5 +99,4 @@ Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 2.6 Classifier: Programming Language :: Python :: 2.7 Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.2 Classifier: Programming Language :: Python :: 3.3 @@ -1,7 +1,7 @@ JMESPath ======== -JMESPath (pronounced "jaymz path") allows you to declaratively specify how to +JMESPath (pronounced ``\ˈjāmz path\``) allows you to declaratively specify how to extract elements from a JSON document. For example, given this document:: @@ -25,7 +25,7 @@ syntax:: The expression: ``foo.bar[*].name`` will return ["one", "two"]. Negative indexing is also supported (-1 refers to the last element in the list). Given the data above, the expression -``foo.bar[-1].name`` will return ["two"]. +``foo.bar[-1].name`` will return "two". The ``*`` can also be used for hash types:: @@ -81,6 +81,3 @@ and reuse them to perform repeated searches:: You can also use the ``jmespath.parser.Parser`` class directly if you want more control. - - -.. _RFC4234: http://tools.ietf.org/html/rfc4234 @@ -1,17 +1,63 @@ #!/usr/bin/env python -import jmespath import sys import json +import argparse + +import jmespath +from jmespath import exceptions +from jmespath.compat import OrderedDict + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('expression') + parser.add_argument('-o', '--ordered', action='store_true', + help='Preserve the order of hash keys, which ' + 'are normally unordered.') + parser.add_argument('-f', '--filename', + help=('The filename containing the input data. ' + 'If a filename is not given then data is ' + 'read from stdin.')) + parser.add_argument('--ast', action='store_true', + help=('Pretty print the AST, do not search the data.')) + args = parser.parse_args() + expression = args.expression + if args.ast: + # Only print the AST + expression = jmespath.compile(args.expression) + sys.stdout.write(str(expression)) + sys.stdout.write('\n') + return 0 + if args.filename: + with open(args.filename, 'r') as f: + data = json.load(f) + else: + data = sys.stdin.read() + if args.ordered: + data = json.loads(data, object_pairs_hook=OrderedDict) + else: + data = json.loads(data) + try: + sys.stdout.write(json.dumps( + jmespath.search(expression, data), indent=4)) + sys.stdout.write('\n') + except exceptions.ArityError as e: + sys.stderr.write("invalid-arity: %s\n" % e) + return 1 + except exceptions.JMESPathTypeError as e: + sys.stderr.write("invalid-type: %s\n" % e) + return 1 + except exceptions.UnknownFunctionError as e: + sys.stderr.write("unknown-function: %s\n" % e) + return 1 + except exceptions.LexerError as e: + sys.stderr.write("syntax-error: %s\n" % e) + return 1 + except exceptions.ParseError as e: + sys.stderr.write("syntax-error: %s\n" % e) + return 1 + -if not len(sys.argv) >= 2: - sys.stderr.write('usage: jp <expression> <filepath>\n\n') - sys.exit(1) -expression = sys.argv[1] -if len(sys.argv) == 3: - with open(sys.argv[2], 'r') as f: - data = json.load(f) -else: - data = json.load(sys.stdin) -sys.stdout.write(json.dumps(jmespath.search(expression, data), indent=4)) -sys.stdout.write('\n') +if __name__ == '__main__': + sys.exit(main()) diff --git a/debian/changelog b/debian/changelog index d8d7924..8c87835 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +python-jmespath (0.4.1-1) unstable; urgency=medium + + * New upstream release + + -- TANIGUCHI Takaki <takaki@debian.org> Mon, 23 Jun 2014 00:35:03 +0900 + python-jmespath (0.2.1-1) unstable; urgency=low * Initial release (Closes: #734131) diff --git a/jmespath.egg-info/PKG-INFO b/jmespath.egg-info/PKG-INFO index bc21be4..7bfe79a 100644 --- a/jmespath.egg-info/PKG-INFO +++ b/jmespath.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: jmespath -Version: 0.2.1 +Version: 0.4.1 Summary: JSON Matching Expressions Home-page: https://github.com/boto/jmespath Author: James Saryerwinnie @@ -9,7 +9,7 @@ License: UNKNOWN Description: JMESPath ======== - JMESPath (pronounced "jaymz path") allows you to declaratively specify how to + JMESPath (pronounced ``\ˈjāmz path\``) allows you to declaratively specify how to extract elements from a JSON document. For example, given this document:: @@ -33,7 +33,7 @@ Description: JMESPath The expression: ``foo.bar[*].name`` will return ["one", "two"]. Negative indexing is also supported (-1 refers to the last element in the list). Given the data above, the expression - ``foo.bar[-1].name`` will return ["two"]. + ``foo.bar[-1].name`` will return "two". The ``*`` can also be used for hash types:: @@ -90,11 +90,8 @@ Description: JMESPath You can also use the ``jmespath.parser.Parser`` class directly if you want more control. - - .. _RFC4234: http://tools.ietf.org/html/rfc4234 - Platform: UNKNOWN -Classifier: Development Status :: 3 - Alpha +Classifier: Development Status :: 5 - Production/Stable Classifier: Intended Audience :: Developers Classifier: Natural Language :: English Classifier: License :: OSI Approved :: MIT License @@ -102,5 +99,4 @@ Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 2.6 Classifier: Programming Language :: Python :: 2.7 Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.2 Classifier: Programming Language :: Python :: 3.3 diff --git a/jmespath.egg-info/SOURCES.txt b/jmespath.egg-info/SOURCES.txt index 37b19d4..b32cbc1 100644 --- a/jmespath.egg-info/SOURCES.txt +++ b/jmespath.egg-info/SOURCES.txt @@ -6,12 +6,12 @@ bin/jp jmespath/__init__.py jmespath/ast.py jmespath/compat.py +jmespath/exceptions.py jmespath/lexer.py jmespath/parser.py jmespath.egg-info/PKG-INFO jmespath.egg-info/SOURCES.txt jmespath.egg-info/dependency_links.txt -jmespath.egg-info/requires.txt jmespath.egg-info/top_level.txt tests/__init__.py tests/test_ast.py diff --git a/jmespath.egg-info/requires.txt b/jmespath.egg-info/requires.txt deleted file mode 100644 index 7163743..0000000 --- a/jmespath.egg-info/requires.txt +++ /dev/null @@ -1 +0,0 @@ -ply==3.4
\ No newline at end of file diff --git a/jmespath/__init__.py b/jmespath/__init__.py index a0d870a..9847342 100644 --- a/jmespath/__init__.py +++ b/jmespath/__init__.py @@ -1,11 +1,11 @@ from jmespath import parser -__version__ = '0.2.1' +__version__ = '0.4.1' -def compile(expression, debug=False): - return parser.Parser(debug=debug).parse(expression) +def compile(expression): + return parser.Parser().parse(expression) -def search(expression, data, debug=False): - return parser.Parser(debug=debug).parse(expression).search(data) +def search(expression, data): + return parser.Parser().parse(expression).search(data) diff --git a/jmespath/ast.py b/jmespath/ast.py index 3cafe83..64d4ed3 100644 --- a/jmespath/ast.py +++ b/jmespath/ast.py @@ -1,29 +1,74 @@ +import operator +import math +import json + +from jmespath.compat import with_repr_method +from jmespath.compat import string_type as STRING_TYPE +from jmespath.compat import zip_longest +from jmespath.exceptions import JMESPathTypeError, UnknownFunctionError + + +NUMBER_TYPE = (float, int) +_VARIADIC = object() + + +# python types -> jmespath types +TYPES_MAP = { + 'bool': 'boolean', + 'list': 'array', + 'dict': 'object', + 'NoneType': 'null', + 'unicode': 'string', + 'str': 'string', + 'float': 'number', + 'int': 'number', + 'OrderedDict': 'object', + '_Projection': 'array', + '_Expression': 'expref', +} + + +# jmespath types -> python types +REVERSE_TYPES_MAP = { + 'boolean': ('bool',), + 'array': ('list', '_Projection'), + 'object': ('dict', 'OrderedDict',), + 'null': ('None',), + 'string': ('unicode', 'str'), + 'number': ('float', 'int'), + 'expref': ('_Expression',), +} + + +class _Arg(object): + __slots__ = ('types',) + + def __init__(self, types=None): + self.types = types + + +@with_repr_method class AST(object): - VALUE_METHODS = [] + + def __init__(self): + self.children = [] def search(self, value): pass - def _get_value_method(self, value): - # This will find the appropriate getter method - # based on the passed in value. - for method_name in self.VALUE_METHODS: - method = getattr(value, method_name, None) - if method is not None: - return method - def pretty_print(self, indent=''): return super(AST, self).__repr__() def __repr__(self): return self.pretty_print() - def __eq__(self, other): - return (isinstance(other, self.__class__) - and self.__dict__ == other.__dict__) - def __ne__(self, other): - return not self.__eq__(other) +class Identity(AST): + def search(self, value): + return value + + def pretty_print(self, indent=''): + return "%sIdentity()" % indent class SubExpression(AST): @@ -36,72 +81,78 @@ class SubExpression(AST): """ def __init__(self, parent, child): - self.parent = parent - self.child = child + self.children = [parent, child] def search(self, value): # To evaluate a subexpression we first evaluate the parent object # and then feed the match of the parent node into the child node. - sub_value = self.parent.search(value) - found = self.child.search(sub_value) + sub_value = self.children[0].search(value) + found = self.children[1].search(sub_value) return found def pretty_print(self, indent=''): sub_indent = indent + ' ' * 4 - return "%sSubExpression(\n%s%s,\n%s%s)" % ( - indent, - sub_indent, self.parent.pretty_print(sub_indent), - sub_indent, self.child.pretty_print(sub_indent)) + return "%s%s(\n%s%s,\n%s%s)" % ( + indent, self.__class__.__name__, + sub_indent, self.children[0].pretty_print(sub_indent), + sub_indent, self.children[1].pretty_print(sub_indent)) + + +# This is used just to differentiate between +# subexpressions and indexexpressions (wildcards can hang +# off of an indexexpression). +class IndexExpression(SubExpression): + pass class Field(AST): - VALUE_METHODS = ['get'] def __init__(self, name): self.name = name + self.children = [] def pretty_print(self, indent=''): return "%sField(%s)" % (indent, self.name) def search(self, value): - method = self._get_value_method(value) - if method is not None: - return method(self.name) + if value is not None: + try: + return value.get(self.name) + except AttributeError: + return None class BaseMultiField(AST): def __init__(self, nodes): - self.nodes = nodes + self.children = list(nodes) def search(self, value): if value is None: return None - method = self._get_value_method(value) - if method is not None: - return method(self.nodes) - else: - return self._multi_get(value) + return self._multi_get(value) + + def _multi_get(self, value): + # Subclasses must define this method. + raise NotImplementedError("_multi_get") def pretty_print(self, indent=''): - return "%s%s(%s)" % (indent, self.__class__.__name__, self.nodes) + return "%s%s(%s)" % (indent, self.__class__.__name__, self.children) class MultiFieldDict(BaseMultiField): - VALUE_METHODS = ['multi_get'] def _multi_get(self, value): collected = {} - for node in self.nodes: + for node in self.children: collected[node.key_name] = node.search(value) return collected class MultiFieldList(BaseMultiField): - VALUE_METHODS = ['multi_get_list'] def _multi_get(self, value): collected = [] - for node in self.nodes: + for node in self.children: collected.append(node.search(value)) return collected @@ -109,20 +160,20 @@ class MultiFieldList(BaseMultiField): class KeyValPair(AST): def __init__(self, key_name, node): self.key_name = key_name - self.node = node + self.children = [node] def search(self, value): - return self.node.search(value) + return self.children[0].search(value) def pretty_print(self, indent=''): - return "%sKeyValPair(key_name=%s, node=%s)" % (indent, self.key_name, - self.node) + return "%sKeyValPair(key_name=%s, node=%s)" % ( + indent, self.key_name, self.children[0]) class Index(AST): - VALUE_METHODS = ['get_index', '__getitem__'] def __init__(self, index): + super(Index, self).__init__() self.index = index def pretty_print(self, indent=''): @@ -133,142 +184,528 @@ class Index(AST): # want to support that. if not isinstance(value, list): return None - method = self._get_value_method(value) - if method is not None: - try: - return method(self.index) - except IndexError: - pass + try: + return value[self.index] + except IndexError: + return None -class WildcardIndex(AST): - """Represents a wildcard index. +class ORExpression(AST): + def __init__(self, first, remaining): + self.children = [first, remaining] - For example:: + def search(self, value): + matched = self.children[0].search(value) + if self._is_false(matched): + matched = self.children[1].search(value) + return matched - foo[*] -> SubExpression(Field(foo), WildcardIndex()) + def _is_false(self, value): + # This looks weird, but we're explicitly using equality checks + # because the truth/false values are different between + # python and jmespath. + return (value == '' or value == [] or value == {} or value is None or + value == False) + + def pretty_print(self, indent=''): + return "%sORExpression(%s, %s)" % (indent, self.children[0], + self.children[1]) + + +class FilterExpression(AST): + + def __init__(self, expression): + self.children = [expression] - """ def search(self, value): if not isinstance(value, list): return None - return _Projection(value) + result = [] + for element in value: + if self.children[0].search(element): + result.append(element) + return result def pretty_print(self, indent=''): - return "%sIndex(*)" % indent + return '%sFilterExpression(%s)' % (indent, self.children[0]) -class WildcardValues(AST): - """Represents a wildcard on the values of a JSON object. +class Literal(AST): - For example:: + def __init__(self, literal_value): + super(Literal, self).__init__() + self.literal_value = literal_value - foo.* -> SubExpression(Field(foo), WildcardValues()) + def search(self, value): + return self.literal_value - """ + def pretty_print(self, indent=''): + return '%sLiteral(%s)' % (indent, self.literal_value) + + +class Comparator(AST): + # Subclasses must define the operation function. + operation = None + + def __init__(self, first, second): + self.children = [first, second] + + def search(self, data): + return self.operation(self.children[0].search(data), + self.children[1].search(data)) + + def pretty_print(self, indent=''): + return '%s%s(%s, %s)' % (indent, self.__class__.__name__, + self.children[0], self.children[1]) + + +class OPEquals(Comparator): + def _equals(self, first, second): + if self._is_special_integer_case(first, second): + return False + else: + return first == second + + def _is_special_integer_case(self, first, second): + # We need to special case comparing 0 or 1 to + # True/False. While normally comparing any + # integer other than 0/1 to True/False will always + # return False. However 0/1 have this: + # >>> 0 == True + # False + # >>> 0 == False + # True + # >>> 1 == True + # True + # >>> 1 == False + # False + # + # Also need to consider that: + # >>> 0 in [True, False] + # True + if first is 0 or first is 1: + return second is True or second is False + elif second is 0 or second is 1: + return first is True or first is False + + operation = _equals + + +class OPNotEquals(OPEquals): + def _not_equals(self, first, second): + return not super(OPNotEquals, self)._equals(first, second) + + operation = _not_equals + + +class OPLessThan(Comparator): + operation = operator.lt + + +class OPLessThanEquals(Comparator): + operation = operator.le + + +class OPGreaterThan(Comparator): + operation = operator.gt + + +class OPGreaterThanEquals(Comparator): + operation = operator.ge + + +class CurrentNode(AST): def search(self, value): + return value + + +class FunctionExpression(AST): + + def __init__(self, name, args): + self.name = name + # The .children attribute is to support homogeneous + # children nodes, but .args is a better name for all the + # code that uses the children, so we support both. + self.children = args + self.args = args try: - return _Projection(value.values()) + self.function = getattr(self, '_func_%s' % name) except AttributeError: + raise UnknownFunctionError("Unknown function: %s" % self.name) + self.arity = self.function.arity + self.variadic = self.function.variadic + self.function = self._resolve_arguments_wrapper(self.function) + + def pretty_print(self, indent=''): + return "%sFunctionExpression(name=%s, args=%s)" % ( + indent, self.name, self.args) + + def search(self, value): + return self.function(value) + + def _resolve_arguments_wrapper(self, function): + def _call_with_resolved_args(value): + # Before calling the function, we have two things to do: + # 1. Resolve the arguments (evaluate the arg expressions + # against the passed in input. + # 2. Type check the arguments + resolved_args = [] + for arg_expression, arg_spec in zip_longest( + self.args, function.argspec, + fillvalue=function.argspec[-1]): + # 1. Resolve the arguments. + current = arg_expression.search(value) + # 2. Type check (provided we have type information). + if arg_spec.types is not None: + _type_check(arg_spec.types, current) + resolved_args.append(current) + return function(*resolved_args) + + def _get_allowed_pytypes(types): + allowed_types = [] + allowed_subtypes = [] + for t in types: + type_ = t.split('-', 1) + if len(type_) == 2: + type_, subtype = type_ + allowed_subtypes.append(REVERSE_TYPES_MAP[subtype]) + else: + type_ = type_[0] + allowed_types.extend(REVERSE_TYPES_MAP[type_]) + return allowed_types, allowed_subtypes + + def _type_check(types, current): + # Type checking involves checking the top level type, + # and in the case of arrays, potentially checking the types + # of each element. + allowed_types, allowed_subtypes = _get_allowed_pytypes(types) + # We're not using isinstance() on purpose. + # The type model for jmespath does not map + # 1-1 with python types (booleans are considered + # integers in python for example). + actual_typename = type(current).__name__ + if actual_typename not in allowed_types: + raise JMESPathTypeError(self.name, current, + TYPES_MAP.get(actual_typename, + 'unknown'), + types) + # If we're dealing with a list type, we can have + # additional restrictions on the type of the list + # elements (for example a function can require a + # list of numbers or a list of strings). + # Arrays are the only types that can have subtypes. + if allowed_subtypes: + _subtype_check(current, allowed_subtypes, types) + + def _subtype_check(current, allowed_subtypes, types): + if len(allowed_subtypes) == 1: + # The easy case, we know up front what type + # we need to validate. + allowed_subtypes = allowed_subtypes[0] + for element in current: + actual_typename = type(element).__name__ + if actual_typename not in allowed_subtypes: + raise JMESPathTypeError(self.name, element, + actual_typename, + types) + elif len(allowed_subtypes) > 1 and current: + # Dynamic type validation. Based on the first + # type we see, we validate that the remaining types + # match. + first = type(current[0]).__name__ + for subtypes in allowed_subtypes: + if first in subtypes: + allowed = subtypes + break + else: + raise JMESPathTypeError(self.name, current[0], + first, types) + for element in current: + actual_typename = type(element).__name__ + if actual_typename not in allowed: + raise JMESPathTypeError(self.name, element, + actual_typename, + types) + + return _call_with_resolved_args + + def signature(*arguments, **kwargs): + def _record_arity(func): + func.arity = len(arguments) + func.variadic = kwargs.get('variadic', False) + func.argspec = arguments + return func + return _record_arity + + @signature(_Arg(), variadic=True) + def _func_not_null(self, *arguments): + for argument in arguments: + if argument is not None: + return argument + + @signature(_Arg(types=['number'])) + def _func_abs(self, arg): + return abs(arg) + + @signature(_Arg(types=['array-number'])) + def _func_avg(self, arg): + return sum(arg) / float(len(arg)) + + @signature(_Arg()) + def _func_to_string(self, arg): + if isinstance(arg, STRING_TYPE): + return arg + else: + return json.dumps(arg) + + @signature(_Arg()) + def _func_to_number(self, arg): + if isinstance(arg, (list, dict, bool)): + return None + elif arg is None: + return None + elif isinstance(arg, (int, float)): + return arg + else: + try: + if '.' in arg: + return float(arg) + else: + return int(arg) + except ValueError: + return None + + @signature(_Arg(types=['array', 'string']), _Arg()) + def _func_contains(self, subject, search): + return search in subject + + @signature(_Arg(types=['string', 'array', 'object'])) + def _func_length(self, arg): + return len(arg) + + @signature(_Arg(types=['number'])) + def _func_ceil(self, arg): + return math.ceil(arg) + + @signature(_Arg(types=['number'])) + def _func_floor(self, arg): + return math.floor(arg) + + @signature(_Arg(types=['string']), _Arg(types=['array-string'])) + def _func_join(self, separator, array): + return separator.join(array) + + @signature(_Arg(types=['array-number'])) + def _func_max(self, arg): + if arg: + return max(arg) + else: + return None + + @signature(_Arg(types=['array-number'])) + def _func_min(self, arg): + if arg: + return min(arg) + else: return None + @signature(_Arg(types=['array-string', 'array-number'])) + def _func_sort(self, arg): + return list(sorted(arg)) + + @signature(_Arg(types=['array-number'])) + def _func_sum(self, arg): + return sum(arg) + + @signature(_Arg(types=['object'])) + def _func_keys(self, arg): + # To be consistent with .values() + # should we also return the indices of a list? + return list(arg.keys()) + + @signature(_Arg(types=['object'])) + def _func_values(self, arg): + return list(arg.values()) + + @signature(_Arg()) + def _func_type(self, arg): + if isinstance(arg, STRING_TYPE): + return "string" + elif isinstance(arg, bool): + return "boolean" + elif isinstance(arg, list): + return "array" + elif isinstance(arg, dict): + return "object" + elif isinstance(arg, (float, int)): + return "number" + elif arg is None: + return "null" + + def _create_key_func(self, expression, allowed_types): + py_types = [] + for type_ in allowed_types: + py_types.extend(REVERSE_TYPES_MAP[type_]) + def keyfunc(x): + result = expression.search(x) + type_name = type(result).__name__ + if type_name not in py_types: + raise JMESPathTypeError(self.name, + result, + type_name, + allowed_types) + return result + return keyfunc + + @signature(_Arg(types=['array']), _Arg(types=['expref'])) + def _func_sort_by(self, array, expref): + # sort_by allows for the expref to be either a number of + # a string, so we have some special logic to handle this. + # We evaluate the first array element and verify that it's + # either a string of a number. We then create a key function + # that validates that type, which requires that remaining array + # elements resolve to the same type as the first element. + if not array: + return array + required_type = TYPES_MAP.get( + type(expref.search(array[0])).__name__) + if required_type not in ['number', 'string']: + raise JMESPathTypeError(self.name, + array[0], + required_type, + ['string', 'number']) + keyfunc = self._create_key_func(expref, [required_type]) + return list(sorted(array, key=keyfunc)) + + @signature(_Arg(types=['array']), _Arg(types=['expref'])) + def _func_max_by(self, array, expref): + keyfunc = self._create_key_func(expref, ['number']) + return max(array, key=keyfunc) + + @signature(_Arg(types=['array']), _Arg(types=['expref'])) + def _func_min_by(self, array, expref): + keyfunc = self._create_key_func(expref, ['number']) + return min(array, key=keyfunc) + + +class ExpressionReference(AST): + def __init__(self, expression): + self.children = [expression] + + def search(self, value): + return _Expression(self.children[0]) + + +class _Expression(AST): + def __init__(self, expression): + self.expression = expression + + def search(self, value): + return self.expression.search(value) + + +class Pipe(AST): + def __init__(self, parent, child): + self.children = [parent, child] + + def search(self, value): + left = self.children[0].search(value) + return self.children[1].search(left) + def pretty_print(self, indent=''): - return "%sWildcardValues()" % indent + sub_indent = indent + ' ' * 4 + return "%s%s(\n%s%s,\n%s%s)" % ( + indent, self.__class__.__name__, + sub_indent, self.children[0].pretty_print(sub_indent), + sub_indent, self.children[1].pretty_print(sub_indent)) + +class Projection(AST): + def __init__(self, left, right): + self.children = [left, right] -class ListElements(AST): def search(self, value): - if isinstance(value, list): - # reduce inner list elements into - # a single list. - merged_list = [] - for element in value: - if isinstance(element, list): - merged_list.extend(element) - else: - merged_list.append(element) - return _Projection(merged_list) + base = self._evaluate_left_child(value) + if base is None: + return None else: + collected = self._evaluate_right_child(base) + return collected + + def _evaluate_left_child(self, value): + base = self.children[0].search(value) + if isinstance(base, list): + return base + else: + # Invalid type, so we return None. return None + def _evaluate_right_child(self, value): + collected = [] + for element in value: + current = self.children[1].search(element) + if current is not None: + collected.append(current) + return collected + def pretty_print(self, indent=''): - return "%sListElements()" % indent + sub_indent = indent + ' ' * 4 + return "%s%s(\n%s%s,\n%s%s)" % ( + indent, self.__class__.__name__, + sub_indent, self.children[0].pretty_print(sub_indent), + sub_indent, self.children[1].pretty_print(sub_indent)) -class _Projection(list): - def __init__(self, elements): - self.extend(elements) +class ValueProjection(Projection): + def _evaluate_left_child(self, value): + base_hash = self.children[0].search(value) + try: + return base_hash.values() + except AttributeError: + return None - def get(self, value): - results = self.__class__([]) - for element in self: - try: - result = element.get(value) - except AttributeError: - continue - if result is not None: - if isinstance(result, list): - result = self.__class__(result) - results.append(result) - return results - - def get_index(self, index): - matches = [] - for el in self: - if not isinstance(el, list): - continue - try: - matches.append(el[index]) - except (IndexError, TypeError): - pass - return self.__class__(matches) - - def multi_get(self, nodes): - results = self.__class__([]) - for element in self: - if isinstance(element, self.__class__): - result = element.multi_get(nodes) - else: - result = {} - for node in nodes: - result[node.key_name] = node.search(element) - results.append(result) - return results - - def multi_get_list(self, nodes): - results = self.__class__([]) - for element in self: - if isinstance(element, self.__class__): - result = element.multi_get_list(nodes) - else: - result = [] - for node in nodes: - result.append(node.search(element)) - results.append(result) - return results - - def values(self): - results = self.__class__([]) - for element in self: - try: - current = self.__class__(element.values()) - results.append(current) - except AttributeError: - continue - return results +class FilterProjection(Projection): + # A filter projection is a left projection that + # filter elements against an expression before allowing + # them to be right evaluated. + def __init__(self, left, right, comparator): + self.children = [left, right, comparator] -class ORExpression(AST): - def __init__(self, first, remaining): - self.first = first - self.remaining = remaining + def _evaluate_right_child(self, value): + result = [] + for element in value: + if self.children[2].search(element): + result.append(element) + return super(FilterProjection, self)._evaluate_right_child(result) - def search(self, value): - matched = self.first.search(value) - if matched is None: - matched = self.remaining.search(value) - return matched + def pretty_print(self, indent=''): + sub_indent = indent + ' ' * 4 + return "%s%s(\n%s%s,\n%s%s,\n%s%s)" % ( + indent, self.__class__.__name__, + sub_indent, self.children[0].pretty_print(sub_indent), + sub_indent, self.children[2].pretty_print(sub_indent), + sub_indent, self.children[1].pretty_print(sub_indent), + ) + + +class Flatten(AST): + def __init__(self, element): + self.children = [element] def pretty_print(self, indent=''): - return "%sORExpression(%s, %s)" % (indent, self.first, - self.remaining) + return "%s%s(%s)" % ( + indent, self.__class__.__name__, + self.children[0].pretty_print(indent).lstrip()) + + def search(self, value): + original = self.children[0].search(value) + if not isinstance(original, list): + return None + merged_list = [] + for element in original: + if isinstance(element, list): + merged_list.extend(element) + else: + merged_list.append(element) + return merged_list diff --git a/jmespath/compat.py b/jmespath/compat.py index 9f81717..910b652 100644 --- a/jmespath/compat.py +++ b/jmespath/compat.py @@ -3,6 +3,11 @@ import sys PY2 = sys.version_info[0] == 2 if PY2: + text_type = unicode + string_type = basestring + from itertools import izip_longest as zip_longest + LR_TABLE = 'jmespath._lrtable' + def with_str_method(cls): """Class decorator that handles __str__ compat between py2 and py3.""" # In python2, the __str__ should be __unicode__ @@ -12,7 +17,35 @@ if PY2: return self.__unicode__().encode('utf-8') cls.__str__ = __str__ return cls + def with_repr_method(cls): + """Class decorator that handle __repr__ with py2 and py3.""" + # This is almost the same thing as with_str_method *except* + # it uses the unicode_escape encoding. This also means we need to be + # careful encoding the input multiple times, so we only encode + # if we get a unicode type. + original_repr_method = cls.__repr__ + def __repr__(self): + original_repr = original_repr_method(self) + if isinstance(original_repr, text_type): + original_repr = original_repr.encode('unicode_escape') + return original_repr + cls.__repr__ = __repr__ + return cls else: + LR_TABLE = 'jmespath._lrtable3' + text_type = str + string_type = str + from itertools import zip_longest def with_str_method(cls): # In python3, we don't need to do anything, we return a str type. return cls + def with_repr_method(cls): + return cls + + +if sys.version_info[:2] == (2, 6): + from ordereddict import OrderedDict + import simplejson as json +else: + from collections import OrderedDict + import json diff --git a/jmespath/exceptions.py b/jmespath/exceptions.py new file mode 100644 index 0000000..b8cb678 --- /dev/null +++ b/jmespath/exceptions.py @@ -0,0 +1,106 @@ +from jmespath.compat import with_str_method + + +class JMESPathError(ValueError): + pass + + +@with_str_method +class ParseError(JMESPathError): + _ERROR_MESSAGE = 'Invalid jmespath expression' + def __init__(self, lex_position, token_value, token_type, + msg=_ERROR_MESSAGE): + super(ParseError, self).__init__(lex_position, token_value, token_type) + self.lex_position = lex_position + self.token_value = token_value + self.token_type = token_type.upper() + self.msg = msg + # Whatever catches the ParseError can fill in the full expression + self.expression = None + + def __str__(self): + # self.lex_position +1 to account for the starting double quote char. + underline = ' ' * (self.lex_position + 1) + '^' + return ( + '%s: Parse error at column %s near ' + 'token "%s" (%s) for expression:\n"%s"\n%s' % ( + self.msg, self.lex_position, self.token_value, self.token_type, + self.expression, underline)) + + +@with_str_method +class IncompleteExpressionError(ParseError): + def set_expression(self, expression): + self.expression = expression + self.lex_position = len(expression) + self.token_type = None + self.token_value = None + + def __str__(self): + # self.lex_position +1 to account for the starting double quote char. + underline = ' ' * (self.lex_position + 1) + '^' + return ( + 'Invalid jmespath expression: Incomplete expression:\n' + '"%s"\n%s' % (self.expression, underline)) + + +@with_str_method +class LexerError(ParseError): + def __init__(self, lexer_position, lexer_value, message, expression=None): + self.lexer_position = lexer_position + self.lexer_value = lexer_value + self.message = message + super(LexerError, self).__init__(lexer_position, + lexer_value, + message) + # Whatever catches LexerError can set this. + self.expression = expression + + def __str__(self): + underline = ' ' * self.lexer_position + '^' + return 'Bad jmespath expression: %s:\n%s\n%s' % ( + self.message, self.expression, underline) + + +@with_str_method +class ArityError(ParseError): + def __init__(self, function_node): + self.expected_arity = function_node.arity + self.actual_arity = len(function_node.args) + self.function_name = function_node.name + self.expression = None + + def __str__(self): + return ("Expected %s arguments for function %s, " + "received %s" % (self.expected_arity, + self.function_name, + self.actual_arity)) + + +@with_str_method +class VariadictArityError(ArityError): + def __str__(self): + return ("Expected at least %s arguments for function %s, " + "received %s" % (self.expected_arity, + self.function_name, + self.actual_arity)) + + +@with_str_method +class JMESPathTypeError(JMESPathError): + def __init__(self, function_name, current_value, actual_type, + expected_types): + self.function_name = function_name + self.current_value = current_value + self.actual_type = actual_type + self.expected_types = expected_types + + def __str__(self): + return ('In function %s(), invalid type for value: %s, ' + 'expected one of: %s, received: "%s"' % ( + self.function_name, self.current_value, + self.expected_types, self.actual_type)) + + +class UnknownFunctionError(JMESPathError): + pass diff --git a/jmespath/lexer.py b/jmespath/lexer.py index 4a98caf..7e0b003 100644 --- a/jmespath/lexer.py +++ b/jmespath/lexer.py @@ -1,75 +1,136 @@ import re from json import loads -from jmespath.compat import with_str_method +from jmespath.exceptions import LexerError -@with_str_method -class LexerError(ValueError): - def __init__(self, lexer_position, lexer_value, message): - self.lexer_position = lexer_position - self.lexer_value = lexer_value - self.message = message - super(LexerError, self).__init__(lexer_position, - lexer_value, - message) - # Whatever catches LexerError can set this. - self.expression = None - def __str__(self): - underline = ' ' * self.lexer_position + '^' - return 'Bad jmespath expression: %s:\n%s\n%s' % ( - self.message, self.expression, underline) +class Lexer(object): + TOKENS = ( + r'(?P<number>-?\d+)|' + r'(?P<unquoted_identifier>([a-zA-Z_][a-zA-Z_0-9]*))|' + r'(?P<quoted_identifier>("(?:\\\\|\\"|[^"])*"))|' + r'(?P<literal>(`(?:\\\\|\\`|[^`])*`))|' + r'(?P<filter>\[\?)|' + r'(?P<or>\|\|)|' + r'(?P<pipe>\|)|' + r'(?P<ne>!=)|' + r'(?P<rbrace>\})|' + r'(?P<eq>==)|' + r'(?P<dot>\.)|' + r'(?P<star>\*)|' + r'(?P<gte>>=)|' + r'(?P<lparen>\()|' + r'(?P<lbrace>\{)|' + r'(?P<lte><=)|' + r'(?P<flatten>\[\])|' + r'(?P<rbracket>\])|' + r'(?P<lbracket>\[)|' + r'(?P<rparen>\))|' + r'(?P<comma>,)|' + r'(?P<colon>:)|' + r'(?P<lt><)|' + r'(?P<expref>&)|' + r'(?P<gt>>)|' + r'(?P<current>@)|' + r'(?P<skip>[ \t]+)' + ) + def __init__(self): + self.master_regex = re.compile(self.TOKENS) + def tokenize(self, expression): + previous_column = 0 + for match in self.master_regex.finditer(expression): + value = match.group() + start = match.start() + end = match.end() + if match.lastgroup == 'skip': + # Ignore whitespace. + previous_column = end + continue + if start != previous_column: + bad_value = expression[previous_column:start] + # Try to give a good error message. + if bad_value == '"': + raise LexerError( + lexer_position=previous_column, + lexer_value=value, + message='Starting quote is missing the ending quote', + expression=expression) + raise LexerError(lexer_position=previous_column, + lexer_value=value, + message='Unknown character', + expression=expression) + previous_column = end + token_type = match.lastgroup + handler = getattr(self, '_token_%s' % token_type.lower(), None) + if handler is not None: + value = handler(value, start, end) + yield {'type': token_type, 'value': value, 'start': start, 'end': end} + # At the end of the loop make sure we've consumed all the input. + # If we haven't then we have unidentified characters. + if end != len(expression): + msg = "Unknown characters at the end of the expression" + raise LexerError(lexer_position=end, + lexer_value='', + message=msg, expression=expression) + else: + yield {'type': 'eof', 'value': '', + 'start': len(expression), 'end': len(expression)} -class LexerDefinition(object): - reflags = re.DOTALL - reserved = {} - tokens = ( - 'STAR', - 'DOT', - 'LBRACKET', - 'RBRACKET', - 'LBRACE', - 'RBRACE', - 'OR', - 'NUMBER', - 'IDENTIFIER', - 'COMMA', - 'COLON', - ) + tuple(reserved.values()) + def _token_number(self, value, start, end): + return int(value) - t_STAR = r'\*' - t_DOT = r'\.' - t_LBRACKET = r'\[' - t_RBRACKET = r'\]' - t_LBRACE = r'\{' - t_RBRACE = r'\}' - t_OR = r'\|\|' - t_COMMA = r',' - t_COLON = r':' - t_ignore = ' ' + def _token_quoted_identifier(self, value, start, end): + try: + return loads(value) + except ValueError as e: + error_message = str(e).split(':')[0] + raise LexerError(lexer_position=start, + lexer_value=value, + message=error_message) - def t_NUMBER(self, t): - r'-?\d+' - t.value = int(t.value) - return t + def _token_literal(self, value, start, end): + actual_value = value[1:-1] + actual_value = actual_value.replace('\\`', '`').lstrip() + # First, if it looks like JSON then we parse it as + # JSON and any json parsing errors propogate as lexing + # errors. + if self._looks_like_json(actual_value): + try: + return loads(actual_value) + except ValueError: + raise LexerError(lexer_position=start, + lexer_value=value, + message="Bad token %s" % value) + else: + potential_value = '"%s"' % actual_value + try: + # There's a shortcut syntax where string literals + # don't have to be quoted. This is only true if the + # string doesn't start with chars that could start a valid + # JSON value. + return loads(potential_value) + except ValueError: + raise LexerError(lexer_position=start, + lexer_value=value, + message="Bad token %s" % value) - def t_IDENTIFIER(self, t): - r'(([a-zA-Z_][a-zA-Z_0-9]*)|("(?:\\"|[^"])*"))' - t.type = self.reserved.get(t.value, 'IDENTIFIER') - i = 0 - if t.value[0] == '"' and t.value[-1] == '"': - t.value = loads(t.value) - return t - return t - - def t_error(self, t): - # Try to be helpful in the case where they have a missing - # quote char. - if t.value.startswith('"'): - raise LexerError( - lexer_position=t.lexpos, - lexer_value=t.value, - message=("Bad token '%s': starting quote is missing " - "the ending quote" % t.value)) - raise ValueError("Illegal token value '%s'" % t.value) + def _looks_like_json(self, value): + # Figure out if the string "value" starts with something + # that looks like json. + if not value: + return False + elif value[0] in ['"', '{', '[']: + return True + elif value in ['true', 'false', 'null']: + return True + elif value[0] in ['-', '0', '1', '2', '3', '4', '5', + '6', '7', '8', '9']: + # Then this is JSON, return True. + try: + loads(value) + return True + except ValueError: + return False + else: + return False diff --git a/jmespath/parser.py b/jmespath/parser.py index 8f37a53..389c3c4 100644 --- a/jmespath/parser.py +++ b/jmespath/parser.py @@ -1,197 +1,452 @@ -import random +"""Top down operator precedence parser. -import ply.yacc -import ply.lex +This is an implementation of Vaughan R. Pratt's +"Top Down Operator Precedence" parser. +(http://dl.acm.org/citation.cfm?doid=512927.512931). + +These are some additional resources that help explain the +general idea behind a Pratt parser: + +* http://effbot.org/zone/simple-top-down-parsing.htm +* http://javascript.crockford.com/tdop/tdop.html +* http://journal.stuffwithstuff.com/2011/03/19/pratt-parsers-expression-parsing-made-easy/ + +A few notes on the implementation. + +* All the nud/led tokens are on the Parser class itself, and are dispatched + using getattr(). This keeps all the parsing logic contained to a single class. +* We use two passes through the data. One to create a list of token, + then one pass through the tokens to create the AST. While the lexer actually + yields tokens, we convert it to a list so we can easily implement two tokens + of lookahead. A previous implementation used a fixed circular buffer, but it + was significantly slower. Also, the average jmespath expression typically + does not have a large amount of token so this is not an issue. And + interestingly enough, creating a token list first is actually faster than + consuming from the token iterator one token at a time. + +""" +import random -from jmespath import ast from jmespath import lexer -from jmespath.compat import with_str_method - - -@with_str_method -class ParseError(ValueError): - def __init__(self, lex_position, token_value, token_type): - super(ParseError, self).__init__(lex_position, token_value, token_type) - self.lex_position = lex_position - self.token_value = token_value - self.token_type = token_type - # Whatever catches the ParseError can fill in the full expression - self.expression = None - - def __str__(self): - # self.lex_position +1 to account for the starting double quote char. - underline = ' ' * (self.lex_position + 1) + '^' - return ( - 'Invalid jmespath expression: Parse error at column %s near ' - 'token "%s" (%s) for expression:\n"%s"\n%s' % ( - self.lex_position, self.token_value, self.token_type, - self.expression, underline)) - - -class IncompleteExpressionError(ParseError): - def set_expression(self, expression): - self.expression = expression - self.lex_position = len(expression) - self.token_type = None - self.token_value = None - - def __str__(self): - # self.lex_position +1 to account for the starting double quote char. - underline = ' ' * (self.lex_position + 1) + '^' - return ( - 'Invalid jmespath expression: Incomplete expression:\n' - '"%s"\n%s' % (self.expression, underline)) - - - -class Grammar(object): - precedence = ( - ('right', 'DOT', 'LBRACKET'), - ) - - def p_jmespath_subexpression(self, p): - """ expression : expression DOT expression - | STAR - """ - if len(p) == 2: - # Then this is the STAR rule. - p[0] = ast.WildcardValues() - else: - # This is the expression DOT expression rule. - p[0] = ast.SubExpression(p[1], p[3]) - - def p_jmespath_index(self, p): - """expression : expression bracket-spec - | bracket-spec - """ - if len(p) == 3: - p[0] = ast.SubExpression(p[1], p[2]) - elif len(p) == 2: - # Otherwise this is just a bracket-spec, which is valid as a root - # level node (e.g. [2]) so we just assign the root node to the - # bracket-spec. - p[0] = p[1] - - def p_jmespath_bracket_specifier(self, p): - """bracket-spec : LBRACKET STAR RBRACKET - | LBRACKET NUMBER RBRACKET - | LBRACKET RBRACKET - """ - if len(p) == 3: - p[0] = ast.ListElements() - elif p[2] == '*': - p[0] = ast.WildcardIndex() - else: - p[0] = ast.Index(p[2]) - - def p_jmespath_identifier(self, p): - """expression : IDENTIFIER - | NUMBER - """ - p[0] = ast.Field(p[1]) - - def p_jmespath_multiselect(self, p): - """expression : LBRACE keyval-exprs RBRACE - """ - p[0] = ast.MultiFieldDict(p[2]) - - def p_jmespath_multiselect_list(self, p): - """expression : LBRACKET expressions RBRACKET - """ - p[0] = ast.MultiFieldList(p[2]) - - def p_jmespath_keyval_exprs(self, p): - """keyval-exprs : keyval-exprs COMMA keyval-expr - | keyval-expr - """ - if len(p) == 2: - p[0] = [p[1]] - elif len(p) == 4: - p[1].append(p[3]) - p[0] = p[1] - - def p_jmespath_keyval_expr(self, p): - """keyval-expr : IDENTIFIER COLON expression - """ - p[0] = ast.KeyValPair(p[1], p[3]) - - def p_jmespath_multiple_expressions(self, p): - """expressions : expressions COMMA expression - | expression - """ - if len(p) == 2: - p[0] = [p[1]] - elif len(p) == 4: - p[1].append(p[3]) - p[0] = p[1] - - def p_jmespath_or_expression(self, p): - """expression : expression OR expression""" - p[0] = ast.ORExpression(p[1], p[3]) - - def p_error(self, t): - if t is not None: - raise ParseError(t.lexpos, t.value, t.type) - else: - raise IncompleteExpressionError(None, None, None) +from jmespath.compat import with_repr_method +from jmespath import ast +from jmespath import exceptions class Parser(object): - # The _max_size most recent expressions are cached in - # _cache dict. - _cache = {} - _max_size = 64 - - def __init__(self, lexer_definition=None, grammar=None, - debug=False): - if lexer_definition is None: - lexer_definition = lexer.LexerDefinition - if grammar is None: - grammar = Grammar - self._lexer_definition = lexer_definition - self._grammar = grammar - self.tokens = self._lexer_definition.tokens - self._debug = debug + BINDING_POWER = { + 'eof': 0, + 'unquoted_identifier': 0, + 'quoted_identifier': 0, + 'rbracket': 0, + 'rparen': 0, + 'comma': 0, + 'rbrace': 0, + 'number': 0, + 'current': 0, + 'expref': 0, + 'pipe': 1, + 'eq': 2, + 'gt': 2, + 'lt': 2, + 'gte': 2, + 'lte': 2, + 'ne': 2, + 'or': 5, + 'flatten': 6, + 'star': 20, + 'dot': 40, + 'lbrace': 50, + 'filter': 50, + 'lbracket': 55, + 'lparen': 60, + } + # The _MAX_SIZE most recent expressions are cached in + # _CACHE dict. + _CACHE = {} + _MAX_SIZE = 128 + + def __init__(self, lookahead=2): + self.tokenizer = None + self._tokens = [None] * lookahead + self._buffer_size = lookahead + self._index = 0 def parse(self, expression): - cached = self._cache.get(expression) + cached = self._CACHE.get(expression) if cached is not None: return cached - lexer = ply.lex.lex(module=self._lexer_definition(), - debug=self._debug, - reflags=self._lexer_definition.reflags) - grammar = self._grammar() - grammar.tokens = self._lexer_definition.tokens - parser = ply.yacc.yacc(module=grammar, debug=self._debug, - write_tables=False) - parsed = self._parse_expression(parser=parser, expression=expression, - lexer_obj=lexer) - self._cache[expression] = parsed - if len(self._cache) > self._max_size: + parsed_result = self._do_parse(expression) + self._CACHE[expression] = parsed_result + if len(self._CACHE) > self._MAX_SIZE: self._free_cache_entries() - return parsed + return parsed_result - def _parse_expression(self, parser, expression, lexer_obj): + def _do_parse(self, expression): try: - parsed = parser.parse(input=expression, lexer=lexer_obj) - return parsed - except lexer.LexerError as e: + return self._parse(expression) + except exceptions.LexerError as e: e.expression = expression - raise e - except IncompleteExpressionError as e: + raise + except exceptions.IncompleteExpressionError as e: e.set_expression(expression) - raise e - except ParseError as e: + raise + except exceptions.ParseError as e: e.expression = expression - raise e + raise + + def _parse(self, expression): + self.tokenizer = lexer.Lexer().tokenize(expression) + self._tokens = list(self.tokenizer) + self._index = 0 + parsed = self._expression(binding_power=0) + if not self._current_token() == 'eof': + t = self._lookahead_token(0) + raise exceptions.ParseError(t['start'], t['value'], t['type'], + "Unexpected token: %s" % t['value']) + return ParsedResult(expression, parsed) + + def _expression(self, binding_power=0): + left_token = self._lookahead_token(0) + self._advance() + nud_function = getattr( + self, '_token_nud_%s' % left_token['type'], + self._error_nud_token) + left = nud_function(left_token) + current_token = self._current_token() + while binding_power < self.BINDING_POWER[current_token]: + led = getattr(self, '_token_led_%s' % current_token, None) + if led is None: + error_token = self._lookahead_token(0) + self._error_led_token(error_token) + else: + self._advance() + left = led(left) + current_token = self._current_token() + return left + + def _token_nud_literal(self, token): + return ast.Literal(token['value']) + + def _token_nud_unquoted_identifier(self, token): + return ast.Field(token['value']) + + def _token_nud_quoted_identifier(self, token): + field = ast.Field(token['value']) + # You can't have a quoted identifier as a function + # name. + if self._current_token() == 'lparen': + t = self._lookahead_token(0) + raise exceptions.ParseError( + 0, t['value'], t['type'], + 'Quoted identifier not allowed for function names.') + return field + + def _token_nud_star(self, token): + left = ast.Identity() + if self._current_token() == 'rbracket': + right = ast.Identity() + else: + right = self._parse_projection_rhs(self.BINDING_POWER['star']) + return ast.ValueProjection(left, right) + + def _token_nud_filter(self, token): + return self._token_led_filter(ast.Identity()) + + def _token_nud_lbrace(self, token): + return self._parse_multi_select_hash() + + def _token_nud_flatten(self, token): + left = ast.Flatten(ast.Identity()) + right = self._parse_projection_rhs( + self.BINDING_POWER['flatten']) + return ast.Projection(left, right) + + def _token_nud_lbracket(self, token): + if self._current_token() == 'number': + node = ast.Index(self._lookahead_token(0)['value']) + self._advance() + self._match('rbracket') + return node + elif self._current_token() == 'star' and self._lookahead(1) == 'rbracket': + self._advance() + self._advance() + right = self._parse_projection_rhs(self.BINDING_POWER['star']) + return ast.Projection(ast.Identity(), right) + else: + return self._parse_multi_select_list() + + def _token_nud_expref(self, token): + expression = self._expression(self.BINDING_POWER['expref']) + return ast.ExpressionReference(expression) + + def _token_led_dot(self, left): + if not self._current_token() == 'star': + right = self._parse_dot_rhs(self.BINDING_POWER['dot']) + return ast.SubExpression(left, right) + else: + # We're creating a projection. + self._advance() + right = self._parse_projection_rhs( + self.BINDING_POWER['dot']) + return ast.ValueProjection(left, right) + + def _token_led_pipe(self, left): + right = self._expression(self.BINDING_POWER['pipe']) + return ast.Pipe(left, right) + + def _token_led_or(self, left): + right = self._expression(self.BINDING_POWER['or']) + return ast.ORExpression(left, right) + + def _token_led_lparen(self, left): + name = left.name + args = [] + while not self._current_token() == 'rparen': + if self._current_token() == 'current': + expression = ast.CurrentNode() + self._advance() + else: + expression = self._expression() + if self._current_token() == 'comma': + self._match('comma') + args.append(expression) + self._match('rparen') + function_node = ast.FunctionExpression(name, args) + if function_node.variadic: + if len(function_node.args) < function_node.arity: + raise exceptions.VariadictArityError(function_node) + elif function_node.arity != len(function_node.args): + raise exceptions.ArityError(function_node) + return function_node + + def _token_led_filter(self, left): + # Filters are projections. + condition = self._expression(0) + self._match('rbracket') + if self._current_token() == 'flatten': + right = ast.Identity() + else: + right = self._parse_projection_rhs(self.BINDING_POWER['filter']) + return ast.FilterProjection(left, right, condition) + + def _token_led_eq(self, left): + return self._parse_comparator(left, 'eq') + + def _token_led_ne(self, left): + return self._parse_comparator(left, 'ne') + + def _token_led_gt(self, left): + return self._parse_comparator(left, 'gt') + + def _token_led_gte(self, left): + return self._parse_comparator(left, 'gte') + + def _token_led_lt(self, left): + return self._parse_comparator(left, 'lt') + + def _token_led_lte(self, left): + return self._parse_comparator(left, 'lte') + + def _token_led_flatten(self, left): + left = ast.Flatten(left) + right = self._parse_projection_rhs( + self.BINDING_POWER['flatten']) + return ast.Projection(left, right) + + def _token_led_lbracket(self, left): + token = self._lookahead_token(0) + if token['type'] == 'number': + self._match('number') + right = ast.Index(token['value']) + self._match('rbracket') + return ast.IndexExpression(left, right) + else: + # We have a projection + self._match('star') + self._match('rbracket') + right = self._parse_projection_rhs(self.BINDING_POWER['star']) + return ast.Projection(left, right) + + def _parse_comparator(self, left, comparator): + op_map = { + 'lt': ast.OPLessThan, + 'lte': ast.OPLessThanEquals, + 'eq': ast.OPEquals, + 'gt': ast.OPGreaterThan, + 'gte': ast.OPGreaterThanEquals, + 'ne': ast.OPNotEquals, + } + right = self._expression(self.BINDING_POWER[comparator]) + return op_map[comparator](left, right) + + def _parse_multi_select_list(self): + expressions = [] + while not self._current_token() == 'rbracket': + expression = self._expression() + expressions.append(expression) + if self._current_token() == 'comma': + self._match('comma') + self._assert_not_token('rbracket') + self._match('rbracket') + return ast.MultiFieldList(expressions) + + def _parse_multi_select_hash(self): + pairs = [] + while True: + key_token = self._lookahead_token(0) + # Before getting the token value, verify it's + # an identifier. + self._match_multiple_tokens( + token_types=['quoted_identifier', 'unquoted_identifier']) + key_name = key_token['value'] + self._match('colon') + value = self._expression(0) + node = ast.KeyValPair(key_name=key_name, node=value) + pairs.append(node) + if self._current_token() == 'comma': + self._match('comma') + elif self._current_token() == 'rbrace': + self._match('rbrace') + break + return ast.MultiFieldDict(nodes=pairs) + + def _parse_projection_rhs(self, binding_power): + # Parse the right hand side of the projection. + if self.BINDING_POWER[self._current_token()] < 10: + # BP of 10 are all the tokens that stop a projection. + right = ast.Identity() + elif self._current_token() == 'lbracket': + right = self._expression(binding_power) + elif self._current_token() == 'filter': + right = self._expression(binding_power) + elif self._current_token() == 'dot': + self._match('dot') + right = self._parse_dot_rhs(binding_power) + else: + t = self._lookahead_token(0) + lex_position = t['start'] + actual_value = t['value'] + actual_type = t['type'] + raise exceptions.ParseError(lex_position, actual_value, + actual_type, 'syntax error') + return right + + def _parse_dot_rhs(self, binding_power): + # From the grammar: + # expression '.' ( identifier / + # multi-select-list / + # multi-select-hash / + # function-expression / + # * + # In terms of tokens that means that after a '.', + # you can have: + lookahead = self._current_token() + # Common case "foo.bar", so first check for an identifier. + if lookahead in ['quoted_identifier', 'unquoted_identifier', 'star']: + return self._expression(binding_power) + elif lookahead == 'lbracket': + self._match('lbracket') + return self._parse_multi_select_list() + elif lookahead == 'lbrace': + self._match('lbrace') + return self._parse_multi_select_hash() + else: + t = self._lookahead_token(0) + allowed = ['quoted_identifier', 'unquoted_identifier', + 'lbracket', 'lbrace'] + lex_position = t['start'] + actual_value = t['value'] + actual_type = t['type'] + raise exceptions.ParseError( + lex_position, actual_value, actual_type, + "Expecting: %s, got: %s" % (allowed, + actual_type)) + + def _assert_not_token(self, *token_types): + if self._current_token() in token_types: + t = self._lookahead_token(0) + lex_position = t['start'] + actual_value = t['value'] + actual_type = t['type'] + raise exceptions.ParseError( + lex_position, actual_value, actual_type, + "Token %s not allowed to be: %s" % (actual_type, token_types)) + + def _error_nud_token(self, token): + raise exceptions.ParseError(token['start'], token['value'], + token['type'], 'Invalid token.') + + def _error_led_token(self, token): + raise exceptions.ParseError(token['start'], token['value'], + token['type'], 'Invalid token') + + def _match(self, token_type=None): + if self._current_token() == token_type: + self._advance() + else: + t = self._lookahead_token(0) + lex_position = t['start'] + actual_value = t['value'] + actual_type = t['type'] + if actual_type == 'eof': + raise exceptions.IncompleteExpressionError( + lex_position, actual_value, actual_type) + else: + message = 'Expecting: %s, got: %s' % (token_type, + actual_type) + raise exceptions.ParseError( + lex_position, actual_value, actual_type, message) + + def _match_multiple_tokens(self, token_types): + if self._current_token() not in token_types: + t = self._lookahead_token(0) + lex_position = t['start'] + actual_value = t['value'] + actual_type = t['type'] + if actual_type == 'eof': + raise exceptions.IncompleteExpressionError( + lex_position, actual_value, actual_type) + else: + message = 'Expecting: %s, got: %s' % (token_types, + actual_type) + raise exceptions.ParseError( + lex_position, actual_value, actual_type, message) + self._advance() + + def _advance(self): + self._index += 1 + + def _current_token(self): + return self._tokens[self._index]['type'] + + def _lookahead(self, number): + return self._tokens[self._index + number]['type'] + + def _lookahead_token(self, number): + return self._tokens[self._index + number] def _free_cache_entries(self): - # This logic is borrowed from the new regex library which - # uses similar eviction strategies. - for key in random.sample(self._cache.keys(), int(self._max_size / 2)): - del self._cache[key] + for key in random.sample(self._CACHE.keys(), int(self._MAX_SIZE / 2)): + del self._CACHE[key] @classmethod def purge(cls): """Clear the expression compilation cache.""" - cls._cache.clear() + cls._CACHE.clear() + + +@with_repr_method +class ParsedResult(object): + def __init__(self, expression, parsed): + self.expression = expression + self.parsed = parsed + + def search(self, value): + return self.parsed.search(value) + + def pretty_print(self, indent=''): + return self.parsed.pretty_print(indent=indent) + + def __repr__(self): + return repr(self.parsed) @@ -2,25 +2,40 @@ import os import sys +import io from setuptools import setup, find_packages +requires = [] + + +if sys.version_info[:2] == (2, 6): + # For python2.6 we have a few other dependencies. + # First we need an ordered dictionary so we use the + # 2.6 backport. + requires.append('ordereddict==1.1') + # Then we need simplejson. This is because we need + # a json version that allows us to specify we want to + # use an ordereddict instead of a normal dict for the + # JSON objects. The 2.7 json module has this. For 2.6 + # we need simplejson. + requires.append('simplejson==3.3.0') + + setup( name='jmespath', - version='0.2.1', + version='0.4.1', description='JSON Matching Expressions', - long_description=open('README.rst').read(), + long_description=io.open('README.rst', encoding='utf-8').read(), author='James Saryerwinnie', author_email='js@jamesls.com', url='https://github.com/boto/jmespath', scripts=['bin/jp'], packages=find_packages(exclude=['tests']), - install_requires=[ - 'ply==3.4', - ], + install_requires=requires, classifiers=( - 'Development Status :: 3 - Alpha', + 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'Natural Language :: English', 'License :: OSI Approved :: MIT License', @@ -28,7 +43,6 @@ setup( 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.2', 'Programming Language :: Python :: 3.3', ), ) diff --git a/tests/__init__.py b/tests/__init__.py index 609ff6c..d86946c 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,4 +1,5 @@ import sys +from jmespath import ast # The unittest module got a significant overhaul @@ -14,3 +15,26 @@ else: from collections import OrderedDict +# Helper method used to create an s-expression +# of the AST to make unit test assertions easier. +# You get a nice string diff on assert failures. +def as_s_expression(node): + parts = [] + _as_s_expression(node, parts) + return ''.join(parts) + + +def _as_s_expression(node, parts): + parts.append("(%s" % (node.__class__.__name__.lower())) + if isinstance(node, ast.Field): + parts.append(" %s" % node.name) + elif isinstance(node, ast.FunctionExpression): + parts.append(" %s" % node.name) + elif isinstance(node, ast.KeyValPair): + parts.append(" %s" % node.key_name) + for child in node.children: + parts.append(" ") + _as_s_expression(child, parts) + parts.append(")") + + diff --git a/tests/test_ast.py b/tests/test_ast.py index fc221a2..2e0aaa1 100644 --- a/tests/test_ast.py +++ b/tests/test_ast.py @@ -1,7 +1,6 @@ #!/usr/bin/env python -import unittest -from tests import OrderedDict +from tests import OrderedDict, unittest from jmespath import ast @@ -96,12 +95,6 @@ class TestAST(unittest.TestCase): {'foo': ['one', {'bar': ['zero', 'one']}]}) self.assertEqual(match, 'one') - def test_index_with_star(self): - # jmespath: foo[*] - child = ast.SubExpression(ast.Field('foo'), ast.WildcardIndex()) - match = child.search({'foo': ['one', 'two']}) - self.assertEqual(match, ['one', 'two']) - def test_associative(self): data = {'foo': {'bar': ['one']}} # jmespath: foo.bar[0] @@ -114,79 +107,15 @@ class TestAST(unittest.TestCase): self.assertEqual(first.search(data), 'one') self.assertEqual(second.search(data), 'one') - def test_wildcard_branches_on_dict_values(self): - data = {'foo': {'bar': {'get': 'one'}, 'baz': {'get': 'two'}}} - # ast for "foo.*.get" - expression = ast.SubExpression( - ast.SubExpression(ast.Field('foo'), ast.WildcardValues()), - ast.Field('get')) - match = expression.search(data) - self.assertEqual(sorted(match), ['one', 'two']) - self.assertEqual(expression.search({'foo': [{'bar': 'one'}]}), None) - - def test_wildcard_dot_wildcard(self): - _ = OrderedDict - data = _([( - "top1", _({ - "sub1": _({"foo": "one"}) - })), - ("top2", _({ - "sub1": _({"foo": "two"}) - })), - ("top3", _({ - "sub3": _({"notfoo": "notfoo"}) - })) - ]) - # ast for "*.*" - expression = ast.SubExpression( - ast.WildcardValues(), ast.WildcardValues()) - match = expression.search(data) - self.assertEqual(match, [[{'foo': 'one'}], [{'foo': 'two'}], - [{'notfoo': 'notfoo'}]]) - - def test_wildcard_with_field_node(self): - data = { - "top1": { - "sub1": {"foo": "one"} - }, - "top2": { - "sub1": {"foo": "two"} - }, - "top3": { - "sub3": {"notfoo": "notfoo"} - } - } - # ast for "*.*.foo" - expression = ast.SubExpression( - ast.WildcardValues(), ast.SubExpression(ast.WildcardValues(), - ast.Field('foo'))) - match = expression.search(data) - self.assertEqual(sorted(match), sorted([[], - ['one'], - ['two']])) - def test_wildcard_branches_with_index(self): # foo[*].bar - child = ast.SubExpression( - ast.SubExpression(ast.Field('foo'), ast.WildcardIndex()), - ast.Field('bar') - ) + child = ast.Projection( + ast.Field('foo'), ast.Field('bar')) match = child.search( {'foo': [{'bar': 'one'}, {'bar': 'two'}]}) self.assertTrue(isinstance(match, list)) self.assertEqual(match, ['one', 'two']) - def test_index_with_multi_match(self): - # foo[*].bar[0] - child = ast.SubExpression( - ast.SubExpression(ast.Field('foo'), ast.WildcardIndex()), - ast.SubExpression( - ast.Field('bar'), - ast.Index(0))) - data = {'foo': [{'bar': ['one', 'two']}, {'bar': ['three', 'four']}]} - match = child.search(data) - self.assertEqual(match, ['one', 'three']) - def test_or_expression(self): # foo or bar field_foo = ast.Field('foo') @@ -226,57 +155,19 @@ class TestAST(unittest.TestCase): subexpr.search({'foo': {'bar': 1, 'baz': 2, 'qux': 3}}), [1, 2]) - def test_multiselect_list_wildcard(self): - data = { - 'foo': { - 'ignore1': { - 'one': 1, 'two': 2, 'three': 3, - }, - 'ignore2': { - 'one': 1, 'two': 2, 'three': 3, - }, - } - } - expr = ast.SubExpression( - ast.Field("foo"), - ast.SubExpression( - ast.WildcardValues(), - ast.MultiFieldList([ast.Field("one"), ast.Field("two")]))) - self.assertEqual(expr.search(data), [[1, 2], [1, 2]]) - - def test_wildcard_values_index_not_a_list(self): - parsed = ast.SubExpression( - ast.WildcardValues(), - ast.SubExpression(ast.Field("foo"), ast.Index(0))) - data = {"a": {"foo": 1}, "b": {"foo": 1}, "c": {"bar": 1}} - self.assertEqual(parsed.search(data), []) - - def test_wildcard_values_index_does_exist(self): - parsed = ast.SubExpression( - ast.WildcardValues(), - ast.SubExpression(ast.Field("foo"), ast.Index(0))) - data = {"a": {"foo": [1]}, "b": {"foo": 1}, "c": {"bar": 1}} - self.assertEqual(parsed.search(data), [1]) - def test_flattened_wildcard(self): - parsed = ast.SubExpression( - # foo[].bar - ast.SubExpression(ast.Field("foo"), ast.ListElements()), - ast.Field("bar")) + # foo[].bar + parsed = ast.Projection(ast.Flatten(ast.Field('foo')), + ast.Field('bar')) data = {'foo': [{'bar': 1}, {'bar': 2}, {'bar': 3}]} self.assertEqual(parsed.search(data), [1, 2, 3]) def test_multiple_nested_wildcards(self): # foo[].bar[].baz - parsed = ast.SubExpression( - ast.SubExpression( - ast.Field("foo"), - ast.ListElements()), - ast.SubExpression( - ast.SubExpression( - ast.Field("bar"), - ast.ListElements()), - ast.Field("baz"))) + parsed = ast.Projection( + ast.Flatten(ast.Projection(ast.Flatten(ast.Field('foo')), + ast.Field('bar'))), + ast.Field('baz')) data = { "foo": [ {"bar": [{"baz": 1}, {"baz": 2}]}, @@ -286,15 +177,11 @@ class TestAST(unittest.TestCase): self.assertEqual(parsed.search(data), [1, 2, 3, 4]) def test_multiple_nested_wildcards_with_list_values(self): - parsed = ast.SubExpression( - ast.SubExpression( - ast.Field("foo"), - ast.ListElements()), - ast.SubExpression( - ast.SubExpression( - ast.Field("bar"), - ast.ListElements()), - ast.Field("baz"))) + # foo[].bar[].baz + parsed = ast.Projection( + ast.Flatten(ast.Projection(ast.Flatten(ast.Field('foo')), + ast.Field('bar'))), + ast.Field('baz')) data = { "foo": [ {"bar": [{"baz": [1]}, {"baz": [2]}]}, @@ -305,16 +192,216 @@ class TestAST(unittest.TestCase): def test_flattened_multiselect_list(self): # foo[].[bar,baz] - field_foo = ast.Field('foo') - parent = ast.SubExpression(field_foo, ast.ListElements()) field_bar = ast.Field('bar') field_baz = ast.Field('baz') multiselect = ast.MultiFieldList([field_bar, field_baz]) - subexpr = ast.SubExpression(parent, multiselect) + projection = ast.Projection( + ast.Flatten(ast.Field('foo')), + multiselect) self.assertEqual( - subexpr.search({'foo': [{'bar': 1, 'baz': 2, 'qux': 3}]}), + projection.search({'foo': [{'bar': 1, 'baz': 2, 'qux': 3}]}), [[1, 2]]) + def test_flattened_multiselect_with_none(self): + # foo[].[bar,baz] + field_bar = ast.Field('bar') + field_baz = ast.Field('baz') + multiselect = ast.MultiFieldList([field_bar, field_baz]) + projection = ast.Projection( + ast.Flatten(ast.Field('foo')), + multiselect) + self.assertEqual( + projection.search({'bar': [{'bar': 1, 'baz': 2, 'qux': 3}]}), + None) + + def test_operator_eq(self): + field_foo = ast.Field('foo') + field_foo2 = ast.Field('foo') + eq = ast.OPEquals(field_foo, field_foo2) + self.assertTrue(eq.search({'foo': 'bar'})) + + def test_operator_not_equals(self): + field_foo = ast.Field('foo') + field_bar = ast.Field('bar') + eq = ast.OPNotEquals(field_foo, field_bar) + self.assertTrue(eq.search({'foo': '1', 'bar': '2'})) + + def test_operator_lt(self): + field_foo = ast.Field('foo') + field_bar = ast.Field('bar') + eq = ast.OPLessThan(field_foo, field_bar) + self.assertTrue(eq.search({'foo': 1, 'bar': 2})) + + def test_filter_expression(self): + # foo[?bar==`yes`] + field_foo = ast.Field('foo') + field_bar = ast.Field('bar') + literal = ast.Literal('yes') + eq = ast.OPEquals(field_bar, literal) + filter_expression = ast.FilterExpression(eq) + full_expression = ast.SubExpression(field_foo, filter_expression) + match = full_expression.search( + {'foo': [{'bar': 'yes', 'v': 1}, + {'bar': 'no', 'v': 2}, + {'bar': 'yes', 'v': 3},]}) + self.assertEqual(match, [{'bar': 'yes', 'v': 1}, + {'bar': 'yes', 'v': 3}]) + + def test_projection_simple(self): + # foo[*].bar + field_foo = ast.Field('foo') + field_bar = ast.Field('bar') + projection = ast.Projection(field_foo, field_bar) + data = {'foo': [{'bar': 1}, {'bar': 2}, {'bar': 3}]} + self.assertEqual(projection.search(data), [1, 2, 3]) + + def test_projection_no_left(self): + # [*].bar + field_bar = ast.Field('bar') + projection = ast.Projection(ast.Identity(), field_bar) + data = [{'bar': 1}, {'bar': 2}, {'bar': 3}] + self.assertEqual(projection.search(data), [1, 2, 3]) + + def test_projection_no_right(self): + # foo[*] + field_foo = ast.Field('foo') + projection = ast.Projection(field_foo, ast.Identity()) + data = {'foo': [{'bar': 1}, {'bar': 2}, {'bar': 3}]} + self.assertEqual(projection.search(data), + [{'bar': 1}, {'bar': 2}, {'bar': 3}]) + + def test_bare_projection(self): + # [*] + projection = ast.Projection(ast.Identity(), ast.Identity()) + data = [{'bar': 1}, {'bar': 2}, {'bar': 3}] + self.assertEqual(projection.search(data), data) + + def test_base_projection_on_invalid_type(self): + # [*] + data = {'foo': [{'bar': 1}, {'bar': 2}, {'bar': 3}]} + projection = ast.Projection(ast.Identity(), ast.Identity()) + # search() should return None because the evaluated + # type is a dict, not a list. + self.assertIsNone(projection.search(data)) + + def test_multiple_projections(self): + # foo[*].bar[*].baz + field_foo = ast.Field('foo') + field_bar = ast.Field('bar') + field_baz = ast.Field('baz') + second_projection = ast.Projection(field_bar, field_baz) + first_projection = ast.Projection(field_foo, second_projection) + data = { + 'foo': [ + {'bar': [{'baz': 1}, {'baz': 2}, {'baz': 3}], + 'other': 1}, + {'bar': [{'baz': 4}, {'baz': 5}, {'baz': 6}], + 'other': 2}, + ] + } + self.assertEqual(first_projection.search(data), + [[1, 2, 3], [4, 5, 6]]) + + def test_values_projection(self): + # foo.*.bar + field_foo = ast.Field('foo') + field_bar = ast.Field('bar') + projection = ast.ValueProjection(field_foo, field_bar) + data = { + 'foo': { + 'a': {'bar': 1}, + 'b': {'bar': 2}, + 'c': {'bar': 3}, + } + + } + result = list(sorted(projection.search(data))) + self.assertEqual(result, [1, 2, 3]) + + def test_root_value_projection(self): + # * + projection = ast.ValueProjection(ast.Identity(), ast.Identity()) + data = { + 'a': 1, + 'b': 2, + 'c': 3, + } + result = list(sorted(projection.search(data))) + self.assertEqual(result, [1, 2, 3]) + + def test_no_left_node_value_projection(self): + # *.bar + field_bar = ast.Field('bar') + projection = ast.ValueProjection(ast.Identity(), field_bar) + data = { + 'a': {'bar': 1}, + 'b': {'bar': 2}, + 'c': {'bar': 3}, + } + result = list(sorted(projection.search(data))) + self.assertEqual(result, [1, 2, 3]) + + def test_no_right_node_value_projection(self): + # foo.* + field_foo = ast.Field('foo') + projection = ast.ValueProjection(field_foo, ast.Identity()) + data = { + 'foo': { + 'a': 1, + 'b': 2, + 'c': 3, + } + } + result = list(sorted(projection.search(data))) + self.assertEqual(result, [1, 2, 3]) + + def test_filter_projection(self): + # foo[?bar==`1`].baz + field_foo = ast.Field('foo') + field_bar = ast.Field('bar') + field_baz = ast.Field('baz') + literal = ast.Literal(1) + comparator = ast.OPEquals(field_bar, literal) + filter_projection = ast.FilterProjection(field_foo, field_baz, comparator) + data = { + 'foo': [{'bar': 1}, {'bar': 2}, {'bar': 1, 'baz': 3}] + } + result = filter_projection.search(data) + self.assertEqual(result, [3]) + + def test_nested_filter_projection(self): + data = { + "reservations": [ + { + "instances": [ + { + "foo": 1, + "bar": 2 + }, + { + "foo": 1, + "bar": 3 + }, + { + "foo": 1, + "bar": 2 + }, + { + "foo": 2, + "bar": 1 + } + ] + } + ] + } + projection = ast.Projection( + ast.Flatten(ast.Field('reservations')), + ast.FilterProjection( + ast.Field('instances'), + ast.Identity(), + ast.OPEquals(ast.Field('bar'), ast.Literal(1)))) + self.assertEqual(projection.search(data), [[{'bar': 1, 'foo': 2}]]) + if __name__ == '__main__': unittest.main() diff --git a/tests/test_compliance.py b/tests/test_compliance.py index 57ffbeb..27d5d45 100644 --- a/tests/test_compliance.py +++ b/tests/test_compliance.py @@ -11,15 +11,35 @@ import jmespath TEST_DIR = os.path.join( os.path.dirname(os.path.abspath(__file__)), 'compliance') +NOT_SPECIFIED = object() def test_compliance(): - for root, dirnames, filenames in os.walk(TEST_DIR): - for filename in filenames: - if filename.endswith('.json'): - full_path = os.path.join(root, filename) - for given, expression, result in _load_cases(full_path): - yield _test_expression, given, expression, result, filename + for full_path in _walk_files(): + if full_path.endswith('.json'): + for given, expression, result, error in _load_cases(full_path): + if error is NOT_SPECIFIED and result is not NOT_SPECIFIED: + yield (_test_expression, given, expression, + result, os.path.basename(full_path)) + elif result is NOT_SPECIFIED and error is not NOT_SPECIFIED: + yield (_test_error_expression, given, expression, + error, os.path.basename(full_path)) + else: + parts = (given, expression, result, error) + raise RuntimeError("Invalid test description: %s" % parts) + + +def _walk_files(): + # Check for a shortcut when running the tests interactively. + # If a JMESPATH_TEST is defined, that file is used as the + # only test to run. Useful when doing feature development. + single_file = os.environ.get('JMESPATH_TEST') + if single_file is not None: + yield os.path.abspath(single_file) + else: + for root, dirnames, filenames in os.walk(TEST_DIR): + for filename in filenames: + yield os.path.join(root, filename) def _load_cases(full_path): @@ -27,10 +47,13 @@ def _load_cases(full_path): for test_data in all_test_data: given = test_data['given'] for case in test_data['cases']: - yield given, case['expression'], case['result'] + yield (given, case['expression'], + case.get('result', NOT_SPECIFIED), + case.get('error', NOT_SPECIFIED)) def _test_expression(given, expression, expected, filename): + import jmespath.parser try: parsed = jmespath.compile(expression) except ValueError as e: @@ -40,7 +63,28 @@ def _test_expression(given, expression, expected, filename): actual = parsed.search(given) expected_repr = json.dumps(expected, indent=4) actual_repr = json.dumps(actual, indent=4) - error_msg = ("\n(%s) The expression '%s' was suppose to give: %s.\n" - "Instead it matched: %s\nparsed as:\n%s" % ( - filename, expression, expected_repr, actual_repr, parsed)) + error_msg = ("\n\n (%s) The expression '%s' was suppose to give:\n%s\n" + "Instead it matched:\n%s\nparsed as:\n%s\ngiven:\n%s" % ( + filename, expression, expected_repr, + actual_repr, parsed, json.dumps(given, indent=4))) + error_msg = error_msg.replace(r'\n', '\n') assert_equal(actual, expected, error_msg) + + +def _test_error_expression(given, expression, error, filename): + import jmespath.parser + if error not in ('syntax', 'invalid-type', + 'unknown-function', 'invalid-arity'): + raise RuntimeError("Unknown error type '%s'" % error) + try: + parsed = jmespath.compile(expression) + parsed.search(given) + except ValueError as e: + # Test passes, it raised a parse error as expected. + pass + else: + error_msg = ("\n\n (%s) The expression '%s' was suppose to be a " + "syntax error, but it successfully parsed as:\n\n%s" % ( + filename, expression, parsed)) + error_msg = error_msg.replace(r'\n', '\n') + raise AssertionError(error_msg) diff --git a/tests/test_parser.py b/tests/test_parser.py index 1bcceb0..65fcfb2 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -1,10 +1,12 @@ #!/usr/bin/env python -from tests import unittest +from tests import unittest, as_s_expression from jmespath import parser from jmespath import ast from jmespath import lexer +from jmespath import compat +from jmespath import exceptions class TestParser(unittest.TestCase): @@ -32,9 +34,9 @@ class TestParser(unittest.TestCase): def test_quoted_subexpression(self): parsed = self.parser.parse('"foo"."bar"') - self.assertIsInstance(parsed, ast.SubExpression) - self.assertEqual(parsed.parent.name, 'foo') - self.assertEqual(parsed.child.name, 'bar') + self.assertIsInstance(parsed.parsed, ast.SubExpression) + self.assertEqual(parsed.parsed.children[0].name, 'foo') + self.assertEqual(parsed.parsed.children[1].name, 'bar') def test_wildcard(self): parsed = self.parser.parse('foo[*]') @@ -65,6 +67,17 @@ class TestParser(unittest.TestCase): parsed = self.parser.parse('foo || bar') self.assertEqual(repr(parsed), 'ORExpression(Field(foo), Field(bar))') + def test_unicode_literals_escaped(self): + parsed = self.parser.parse(r'`"\u2713"`') + if compat.PY2: + self.assertEqual(repr(parsed), r'Literal(\u2713)') + else: + self.assertEqual(repr(parsed), u'Literal(\u2713)') + + def test_unicode_pretty_print(self): + parsed = self.parser.parse(r'`"\u2713"`') + self.assertEqual(parsed.pretty_print(), u'Literal(\u2713)') + def test_multiselect(self): parsed = self.parser.parse('foo.{bar: bar,baz: baz}') self.assertEqual( @@ -90,7 +103,7 @@ class TestErrorMessages(unittest.TestCase): self.parser = parser.Parser() def assert_error_message(self, expression, error_message, - exception=parser.ParseError): + exception=exceptions.ParseError): try: self.parser.parse(expression) except exception as e: @@ -105,12 +118,12 @@ class TestErrorMessages(unittest.TestCase): "ParseError not raised for bad expression: %s" % expression) def test_bad_parse(self): - with self.assertRaises(parser.ParseError): + with self.assertRaises(exceptions.ParseError): parsed = self.parser.parse('foo]baz') def test_bad_parse_error_message(self): error_message = ( - 'Invalid jmespath expression: Parse error at column 3 ' + 'Unexpected token: ]: Parse error at column 3 ' 'near token "]" (RBRACKET) for expression:\n' '"foo]baz"\n' ' ^') @@ -125,12 +138,25 @@ class TestErrorMessages(unittest.TestCase): def test_bad_lexer_values(self): error_message = ( - 'Bad jmespath expression: Bad token \'"bar\': ' - 'starting quote is missing the ending quote:\n' + 'Bad jmespath expression: ' + 'Starting quote is missing the ending quote:\n' 'foo."bar\n' ' ^') self.assert_error_message('foo."bar', error_message, - exception=lexer.LexerError) + exception=exceptions.LexerError) + + def test_bad_lexer_literal_value_with_json_object(self): + error_message = ('Bad jmespath expression: ' + 'Bad token `{{}`:\n`{{}`\n^') + self.assert_error_message('`{{}`', error_message, + exception=exceptions.LexerError) + + + def test_bad_unicode_string(self): + error_message = ('Bad jmespath expression: ' + 'Invalid \\uXXXX escape:\n"\\uAZ12"\n^') + self.assert_error_message(r'"\uAZ12"', error_message, + exception=exceptions.LexerError) class TestParserWildcards(unittest.TestCase): @@ -261,13 +287,35 @@ class TestParserCaching(unittest.TestCase): p = parser.Parser() compiled = [] compiled2 = [] - for i in range(parser.Parser._max_size + 1): + for i in range(parser.Parser._MAX_SIZE + 1): compiled.append(p.parse('foo%s' % i)) # Rerun the test and half of these entries should be from the # cache but they should still be equal to compiled. - for i in range(parser.Parser._max_size + 1): + for i in range(parser.Parser._MAX_SIZE + 1): compiled2.append(p.parse('foo%s' % i)) - self.assertEqual(compiled, compiled2) + self.assertEqual(len(compiled), len(compiled2)) + s = as_s_expression + self.assertEqual( + [s(expr.parsed) for expr in compiled], + [s(expr.parsed) for expr in compiled2]) + + def test_cache_purge(self): + p = parser.Parser() + first = p.parse('foo') + cached = p.parse('foo') + p.purge() + second = p.parse('foo') + self.assertEqual(as_s_expression(first.parsed), + as_s_expression(second.parsed)) + self.assertEqual(as_s_expression(first.parsed), + as_s_expression(cached.parsed)) + + +class TestParserAddsExpressionAttribute(unittest.TestCase): + def test_expression_available_from_parser(self): + p = parser.Parser() + parsed = p.parse('foo.bar') + self.assertEqual(parsed.expression, 'foo.bar') if __name__ == '__main__': |