Merge pull request #21 from jmespath-community/feature/let-expression

Feature/let expression

Author: springcomp

.gitignore

@@ -4,6 +4,7 @@
 bin/
 share/
 pyvenv.cfg
+.venv/
 
 # C extensions
 *.so

jmespath.test

@@ -10,6 +10,10 @@ def arithmetic(operator, left, right):
     return {'type': 'arithmetic', 'children': [left, right], 'value': operator}
 
 
+def assign(name, expr):
+    return {'type': 'assign', 'children': [expr], 'value': name}
+
+
 def comparator(name, first, second):
     return {'type': 'comparator', 'children': [first, second], 'value': name}
 
@@ -58,6 +62,10 @@ def key_val_pair(key_name, node):
     return {"type": "key_val_pair", 'children': [node], "value": key_name}
 
 
+def let_expression(bindings, expr):
+    return {'type': 'let_expression', 'children': [*bindings, expr]}
+
+
 def literal(literal_value):
     return {'type': 'literal', 'value': literal_value, 'children': []}
 
@@ -100,3 +108,7 @@ def slice(start, end, step):
 
 def value_projection(left, right):
     return {'type': 'value_projection', 'children': [left, right]}
+
+
+def variable_ref(name):
+    return {"type": "variable_ref", "children": [], "value": name}

jmespath/ast.py

@@ -133,3 +133,9 @@ def __init__(self):
 
 class UnknownFunctionError(JMESPathError):
     pass
+
+
+class UndefinedVariable(JMESPathError):
+    def __init__(self, varname):
+        self.varname = varname
+        super().__init__(f"Reference to undefined variable: {self.varname}")

jmespath/exceptions.py

@@ -85,12 +85,6 @@ def call_function(self, function_name, resolved_args, *args, **kwargs):
         signature = spec['signature']
         self._validate_arguments(resolved_args, signature, function_name)
 
-        # supply extra arguments only if the function expects them
-
-        parameters = [parameter.name for parameter in inspect.signature(function).parameters.values()]
-        if ('kwargs' in parameters):
-            return function(self, *resolved_args, *args, scopes = kwargs.get('scopes'))
-
         return function(self, *resolved_args)
 
     def _validate_arguments(self, args, signature, function_name):
@@ -540,12 +534,6 @@ def _func_max_by(self, array, expref):
         else:
             return None
 
-    @signature({'types': ['object']}, {'types': ['expref']})
-    def _func_let(self, scope, expref, *args, **kwargs):
-        if 'scopes' in kwargs:
-            kwargs.get('scopes').pushScope(scope)
-        return expref.visit(expref.expression, expref.context, *args, **kwargs)
-
     @signature({'types': ['array'], 'variadic': True})
     def _func_zip(self, *arguments):
         return list(map(list, zip(*arguments)))

jmespath/functions.py

@@ -18,7 +18,6 @@ class Lexer(object):
         ',': 'comma',
         ':': 'colon',
         '@': 'current',
-        '$': 'root',
         '(': 'lparen',
         ')': 'rparen',
         '{': 'lbrace',
@@ -115,22 +114,15 @@ def tokenize(self, expression, options=None):
             elif self._current == '!':
                 yield self._match_or_else('=', 'ne', 'not')
             elif self._current == '=':
-                if self._next() == '=':
-                    yield {'type': 'eq', 'value': '==',
-                        'start': self._position - 1, 'end': self._position}
-                    self._next()
+                yield self._match_or_else('=', 'eq', 'assign')
+            elif self._current == '$':
+                if self._peek_may_be_valid_unquoted_identifier():
+                    yield self._consume_variable()
                 else:
-                    if self._current is None:
-                        # If we're at the EOF, we never advanced
-                        # the position so we don't need to rewind
-                        # it back one location.
-                        position = self._position
-                    else:
-                        position = self._position - 1
-                    raise LexerError(
-                        lexer_position=position,
-                        lexer_value='=',
-                        message="Unknown token '='")
+                    yield {'type': 'root',
+                           'value': self._current,
+                           'start': self._position, 'end': self._position + 1}
+                    self._next()
             else:
                 raise LexerError(lexer_position=self._position,
                                  lexer_value=self._current,
@@ -145,6 +137,28 @@ def _consume_number(self):
             buff += self._current
         return buff
 
+    def _consume_variable(self):
+        start = self._position
+        buff = self._current
+        self._next()
+        if self._current not in self.START_IDENTIFIER:
+            raise LexerError(
+                lexer_position=start,
+                lexer_value=self._current,
+                message='Invalid variable starting character %s' % self._current)
+        buff += self._current
+        while self._next() in self.VALID_IDENTIFIER:
+            buff += self._current
+        return {'type': 'variable', 'value': buff,
+                'start': start, 'end': start + len(buff)}
+
+    def _peek_may_be_valid_unquoted_identifier(self):
+        if (self._position == self._length - 1):
+            return False
+        else:
+            next = self._chars[self._position + 1]
+            return next in self.START_IDENTIFIER
+
     def _peek_is_next_digit(self):
         if (self._position == self._length - 1):
             return False

jmespath/lexer.py

@@ -38,6 +38,8 @@
 class Parser(object):
     BINDING_POWER = {
         'eof': 0,
+        'variable': 0,
+        'assign': 0,
         'unquoted_identifier': 0,
         'quoted_identifier': 0,
         'literal': 0,
@@ -145,8 +147,39 @@ def _expression(self, binding_power=0):
     def _token_nud_literal(self, token):
         return ast.literal(token['value'])
 
+    def _token_nud_variable(self, token):
+        return ast.variable_ref(token['value'][1:])
+
     def _token_nud_unquoted_identifier(self, token):
-        return ast.field(token['value'])
+        if token['value'] == 'let' and \
+                self._current_token() == 'variable':
+            return self._parse_let_expression()
+        else:
+            return ast.field(token['value'])
+
+    def _parse_let_expression(self):
+        bindings = []
+        while True:
+            var_token = self._lookahead_token(0)
+            # Strip off the '$'.
+            varname = var_token['value'][1:]
+            self._advance()
+            self._match('assign')
+            assign_expr = self._expression()
+            bindings.append(ast.assign(varname, assign_expr))
+            if self._is_in_keyword(self._lookahead_token(0)):
+                self._advance()
+                break
+            else:
+                self._match('comma')
+        expr = self._expression()
+        return ast.let_expression(bindings, expr)
+
+    def _is_in_keyword(self, token):
+        return (
+            token['type'] == 'unquoted_identifier' and
+            token['value'] == 'in'
+        )
 
     def _token_nud_quoted_identifier(self, token):
         field = ast.field(token['value'])
@@ -552,7 +585,7 @@ def __init__(self, expression, parsed):
         self.parsed = parsed
 
     def search(self, value, options=None):
-        evaluator = visitor.ScopedInterpreter(options)
+        evaluator = visitor.TreeInterpreter(options)
         return evaluator.evaluate(self.parsed, value)
 
     def _render_dot_file(self):

jmespath/parser.py

@@ -0,0 +1,40 @@
+from collections import deque
+
+
+class ScopedChainDict:
+    """Dictionary that can delegate lookups to multiple dicts.
+
+    This provides a basic get/set dict interface that is
+    backed by multiple dicts.  Each dict is searched from
+    the top most (most recently pushed) scope dict until
+    a match is found.
+
+    """
+    def __init__(self, *scopes):
+        # The scopes are evaluated starting at the top of the stack (the most
+        # recently pushed scope via .push_scope()).  If we use a normal list()
+        # and push/pop scopes by adding/removing to the end of the list, we'd
+        # have to always call reversed(self._scopes) whenever we resolve a key,
+        # because the end of the list is the top of the stack.
+        # To avoid this, we're using a deque so we can append to the front of
+        # the list via .appendleft() in constant time, and iterate over scopes
+        # without having to do so with a reversed() call each time.
+        self._scopes = deque(scopes)
+
+    def __getitem__(self, key):
+        for scope in self._scopes:
+            if key in scope:
+                return scope[key]
+        raise KeyError(key)
+
+    def get(self, key, default=None):
+        try:
+            return self[key]
+        except KeyError:
+            return default
+
+    def push_scope(self, scope):
+        self._scopes.appendleft(scope)
+
+    def pop_scope(self):
+        self._scopes.popleft()