[jsinterp] Handle regexp literals and throw/catch execution (#31182)

* based on f6ca640b12, thanks pukkandan
* adds parse support for regexp flags
This commit is contained in:
dirkf 2022-08-19 11:45:04 +01:00 committed by GitHub
parent b0a60ce203
commit 538ec65ba7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 139 additions and 22 deletions

View file

@ -9,6 +9,7 @@ import unittest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import math
import re
from youtube_dl.jsinterp import JSInterpreter
undefined = JSInterpreter.undefined
@ -316,19 +317,39 @@ class TestJSInterpreter(unittest.TestCase):
function x() { return {}; }
''')
self.assertEqual(jsi.call_function('x'), {})
jsi = JSInterpreter('''
function x() { let a = {m1: 42, m2: 0 }; return [a["m1"], a.m2]; }
''')
self.assertEqual(jsi.call_function('x'), [42, 0])
jsi = JSInterpreter('''
function x() { let a; return a?.qq; }
''')
self.assertIs(jsi.call_function('x'), undefined)
jsi = JSInterpreter('''
function x() { let a = {m1: 42, m2: 0 }; return a?.qq; }
''')
self.assertIs(jsi.call_function('x'), undefined)
def test_regex(self):
jsi = JSInterpreter('''
function x() { let a=/,,[/,913,/](,)}/; }
''')
self.assertIs(jsi.call_function('x'), None)
jsi = JSInterpreter('''
function x() { let a=/,,[/,913,/](,)}/; return a; }
''')
# Pythons disagree on the type of a pattern
self.assertTrue(isinstance(jsi.call_function('x'), type(re.compile(''))))
jsi = JSInterpreter('''
function x() { let a=/,,[/,913,/](,)}/i; return a; }
''')
self.assertEqual(jsi.call_function('x').flags & re.I, re.I)
if __name__ == '__main__':
unittest.main()

View file

@ -106,6 +106,10 @@ _NSIG_TESTS = [
'https://www.youtube.com/s/player/c81bbb4a/player_ias.vflset/en_US/base.js',
'gre3EcLurNY2vqp94', 'Z9DfGxWP115WTg',
),
(
'https://www.youtube.com/s/player/1f7d5369/player_ias.vflset/en_US/base.js',
'batNX7sYqIJdkJ', 'IhOkL_zxbkOZBw',
),
]

View file

@ -7,6 +7,7 @@ import operator
import re
from .utils import (
error_to_compat_str,
ExtractorError,
js_to_json,
remove_quotes,
@ -130,7 +131,7 @@ _SC_OPERATORS = (
_OPERATOR_RE = '|'.join(map(lambda x: re.escape(x[0]), _OPERATORS + _LOG_OPERATORS))
_MATCHING_PARENS = dict(zip(*zip('()', '{}', '[]')))
_QUOTES = '\'"'
_QUOTES = '\'"/'
def _ternary(cndn, if_true=True, if_false=False):
@ -155,6 +156,12 @@ class JS_Continue(ExtractorError):
ExtractorError.__init__(self, 'Invalid continue')
class JS_Throw(ExtractorError):
def __init__(self, e):
self.error = e
ExtractorError.__init__(self, 'Uncaught exception ' + error_to_compat_str(e))
class LocalNameSpace(ChainMap):
def __getitem__(self, key):
try:
@ -172,6 +179,17 @@ class LocalNameSpace(ChainMap):
def __delitem__(self, key):
raise NotImplementedError('Deleting is not supported')
# except
def pop(self, key, *args):
try:
off = self.__getitem__(key)
super(LocalNameSpace, self).__delitem__(key)
return off
except KeyError:
if len(args) > 0:
return args[0]
raise
def __contains__(self, key):
try:
super(LocalNameSpace, self).__getitem__(key)
@ -188,9 +206,29 @@ class JSInterpreter(object):
undefined = _UNDEFINED
RE_FLAGS = {
# special knowledge: Python's re flags are bitmask values, current max 128
# invent new bitmask values well above that for literal parsing
# TODO: new pattern class to execute matches with these flags
'd': 1024, # Generate indices for substring matches
'g': 2048, # Global search
'i': re.I, # Case-insensitive search
'm': re.M, # Multi-line search
's': re.S, # Allows . to match newline characters
'u': re.U, # Treat a pattern as a sequence of unicode code points
'y': 4096, # Perform a "sticky" search that matches starting at the current position in the target string
}
_EXC_NAME = '__youtube_dl_exception__'
_OBJ_NAME = '__youtube_dl_jsinterp_obj'
OP_CHARS = None
def __init__(self, code, objects=None):
self.code, self._functions = code, {}
self._objects = {} if objects is None else objects
if type(self).OP_CHARS is None:
type(self).OP_CHARS = self.OP_CHARS = self.__op_chars()
class Exception(ExtractorError):
def __init__(self, msg, *args, **kwargs):
@ -199,32 +237,64 @@ class JSInterpreter(object):
msg = '{0} in: {1!r}'.format(msg.rstrip(), expr[:100])
super(JSInterpreter.Exception, self).__init__(msg, *args, **kwargs)
@classmethod
def __op_chars(cls):
op_chars = set(';,')
for op in cls._all_operators():
for c in op[0]:
op_chars.add(c)
return op_chars
def _named_object(self, namespace, obj):
self.__named_object_counter += 1
name = '__youtube_dl_jsinterp_obj%d' % (self.__named_object_counter, )
name = '%s%d' % (self._OBJ_NAME, self.__named_object_counter)
namespace[name] = obj
return name
@staticmethod
def _separate(expr, delim=',', max_split=None, skip_delims=None):
@classmethod
def _regex_flags(cls, expr):
flags = 0
if not expr:
return flags, expr
for idx, ch in enumerate(expr):
if ch not in cls.RE_FLAGS:
break
flags |= cls.RE_FLAGS[ch]
return flags, expr[idx:] if idx > 0 else expr
@classmethod
def _separate(cls, expr, delim=',', max_split=None, skip_delims=None):
if not expr:
return
counters = {k: 0 for k in _MATCHING_PARENS.values()}
start, splits, pos, skipping, delim_len = 0, 0, 0, 0, len(delim) - 1
in_quote, escaping = None, False
start, splits, pos, delim_len = 0, 0, 0, len(delim) - 1
in_quote, escaping, skipping = None, False, 0
after_op, in_regex_char_group, skip_re = True, False, 0
for idx, char in enumerate(expr):
if skip_re > 0:
skip_re -= 1
continue
if not in_quote:
if char in _MATCHING_PARENS:
counters[_MATCHING_PARENS[char]] += 1
elif char in counters:
counters[char] -= 1
if not escaping:
if char in _QUOTES and in_quote in (char, None):
in_quote = None if in_quote else char
else:
escaping = in_quote and char == '\\'
else:
escaping = False
if not escaping and char in _QUOTES and in_quote in (char, None):
if in_quote or after_op or char != '/':
in_quote = None if in_quote and not in_regex_char_group else char
if in_quote is None and char == '/' and delim != '/':
# regexp flags
n_idx = idx + 1
while n_idx < len(expr) and expr[n_idx] in cls.RE_FLAGS:
n_idx += 1
skip_re = n_idx - idx - 1
if skip_re > 0:
continue
elif in_quote == '/' and char in '[]':
in_regex_char_group = char == '['
escaping = not escaping and in_quote and char == '\\'
after_op = not in_quote and char in cls.OP_CHARS or (char == ' ' and after_op)
if char != delim[pos] or any(counters.values()) or in_quote:
pos = skipping = 0
@ -313,15 +383,22 @@ class JSInterpreter(object):
if should_return:
return ret, should_return
m = re.match(r'(?P<var>(?:var|const|let)\s)|return(?:\s+|$)', stmt)
m = re.match(r'(?P<var>(?:var|const|let)\s)|return(?:\s+|(?=["\'])|$)|(?P<throw>throw\s+)', stmt)
if m:
expr = stmt[len(m.group(0)):].strip()
if m.group('throw'):
raise JS_Throw(self.interpret_expression(expr, local_vars, allow_recursion))
should_return = not m.group('var')
if not expr:
return None, should_return
if expr[0] in _QUOTES:
inner, outer = self._separate(expr, expr[0], 1)
if expr[0] == '/':
flags, _ = self._regex_flags(outer)
inner, outer = inner.replace('"', r'\"'), ''
inner = re.compile(js_to_json(inner + expr[0]), flags=flags) # , strict=True))
else:
inner = json.loads(js_to_json(inner + expr[0])) # , strict=True))
if not outer:
return inner, should_return
@ -374,22 +451,37 @@ class JSInterpreter(object):
for item in self._separate(inner)])
expr = name + outer
m = re.match(r'(?P<try>try|finally)\s*|(?:(?P<catch>catch)|(?P<for>for)|(?P<switch>switch))\s*\(', expr)
m = re.match(r'''(?x)
(?P<try>try|finally)\s*|
(?P<catch>catch\s*(?P<err>\(\s*{_NAME_RE}\s*\)))|
(?P<switch>switch)\s*\(|
(?P<for>for)\s*\(|'''.format(**globals()), expr)
md = m.groupdict() if m else {}
if md.get('try'):
if expr[m.end()] == '{':
try_expr, expr = self._separate_at_paren(expr[m.end():], '}')
else:
try_expr, expr = expr[m.end() - 1:], ''
try:
ret, should_abort = self.interpret_statement(try_expr, local_vars, allow_recursion)
if should_abort:
return ret, True
except JS_Throw as e:
local_vars[self._EXC_NAME] = e.error
except Exception as e:
# XXX: This works for now, but makes debugging future issues very hard
local_vars[self._EXC_NAME] = e
ret, should_abort = self.interpret_statement(expr, local_vars, allow_recursion)
return ret, should_abort or should_return
elif md.get('catch'):
# We ignore the catch block
_, expr = self._separate_at_paren(expr, '}')
catch_expr, expr = self._separate_at_paren(expr[m.end():], '}')
if self._EXC_NAME in local_vars:
catch_vars = local_vars.new_child({m.group('err'): local_vars.pop(self._EXC_NAME)})
ret, should_abort = self.interpret_statement(catch_expr, catch_vars, allow_recursion)
if should_abort:
return ret, True
ret, should_abort = self.interpret_statement(expr, local_vars, allow_recursion)
return ret, should_abort or should_return
@ -503,7 +595,7 @@ class JSInterpreter(object):
raise self.Exception('List index %s must be integer' % (idx, ), expr=expr)
idx = int(idx)
left_val[idx] = self._operator(
m.group('op'), left_val[idx], m.group('expr'), expr, local_vars, allow_recursion)
m.group('op'), self._index(left_val, idx), m.group('expr'), expr, local_vars, allow_recursion)
return left_val[idx], should_return
elif expr.isdigit():