diff --git a/lib/prism/translation/ripper.rb b/lib/prism/translation/ripper.rb index dd443e207f..b066f3e3ac 100644 --- a/lib/prism/translation/ripper.rb +++ b/lib/prism/translation/ripper.rb @@ -22,16 +22,9 @@ module Translation # - on_comma # - on_ignored_nl # - on_ignored_sp - # - on_kw # - on_label_end - # - on_lbrace - # - on_lbracket - # - on_lparen # - on_nl # - on_operator_ambiguous - # - on_rbrace - # - on_rbracket - # - on_rparen # - on_semicolon # - on_sp # @@ -660,7 +653,13 @@ def visit_alternation_pattern_node(node) # parenthesis node that can be used to wrap patterns. private def visit_pattern_node(node) if node.is_a?(ParenthesesNode) - visit(node.body) + bounds(node.opening_loc) + on_lparen("(") + result = visit(node.body) + bounds(node.closing_loc) + on_rparen(")") + + result else visit(node) end @@ -848,6 +847,12 @@ def visit_array_node(node) # ^^^^^ def visit_array_pattern_node(node) constant = visit(node.constant) + + if node.opening_loc + bounds(node.opening_loc) + node.opening == "[" ? on_lbracket("[") : on_lparen("(") + end + requireds = visit_all(node.requireds) if node.requireds.any? rest = if (rest_node = node.rest).is_a?(SplatNode) @@ -864,6 +869,10 @@ def visit_array_pattern_node(node) posts = visit_all(node.posts) if node.posts.any? + if node.closing_loc + bounds(node.closing_loc) + node.closing == "]" ? on_rbracket("]") : on_rparen(")") + end bounds(node.location) on_aryptn(constant, requireds, rest, posts) end @@ -871,7 +880,7 @@ def visit_array_pattern_node(node) # foo(bar) # ^^^ def visit_arguments_node(node) - arguments, _, _ = visit_call_node_arguments(node, nil, false) + arguments, _ = visit_call_node_arguments(node, nil, false) arguments end @@ -1008,8 +1017,10 @@ def visit_block_local_variable_node(node) # Visit a BlockNode. def visit_block_node(node) braces = node.opening == "{" - unless braces - bounds(node.opening_loc) + bounds(node.opening_loc) + if braces + on_lbrace("{") + else on_kw("do") end @@ -1036,7 +1047,10 @@ def visit_block_node(node) raise end - unless braces + if braces + bounds(node.closing_loc) + on_rbrace("}") + else bounds(node.closing_loc) on_kw("end") end @@ -1127,12 +1141,21 @@ def visit_call_node(node) case node.name when :[] receiver = visit(node.receiver) - arguments, block, has_ripper_block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc)) + + bounds(node.opening_loc) + on_lbracket("[") + + arguments, block_node = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc)) + + bounds(node.closing_loc) + on_rbracket("]") + + block = visit(block_node) bounds(node.location) call = on_aref(receiver, arguments) - if has_ripper_block + if block_node bounds(node.location) on_method_add_block(call, block) else @@ -1141,6 +1164,9 @@ def visit_call_node(node) when :[]= receiver = visit(node.receiver) + bounds(node.opening_loc) + on_lbracket("[") + *arguments, last_argument = node.arguments.arguments arguments << node.block if !node.block.nil? @@ -1156,6 +1182,8 @@ def visit_call_node(node) end end + bounds(node.closing_loc) + on_rbracket("]") bounds(node.equal_loc) on_op("=") @@ -1177,11 +1205,27 @@ def visit_call_node(node) if node.message == "not" on_kw("not") + if node.opening_loc + bounds(node.opening_loc) + on_lparen("(") + end + receiver = - if !node.receiver.is_a?(ParenthesesNode) || !node.receiver.body.nil? + if node.receiver.is_a?(ParenthesesNode) && node.receiver.body.nil? + # The parens in `not()` just emit parens and nothing else. + bounds(node.receiver.opening_loc) + on_lparen("(") + bounds(node.receiver.closing_loc) + on_rparen(")") + nil + else visit(node.receiver) end + if node.closing_loc + bounds(node.closing_loc) + on_rparen(")") + end bounds(node.location) on_unary(:not, receiver) else @@ -1209,7 +1253,19 @@ def visit_call_node(node) if node.variable_call? on_vcall(message) else - arguments, block, has_ripper_block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc || node.location)) + if node.opening_loc + bounds(node.opening_loc) + on_lparen("(") + end + + arguments, block_node = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc || node.location)) + + if node.closing_loc + bounds(node.closing_loc) + on_rparen(")") + end + + block = visit(block_node) call = if node.opening_loc.nil? && get_arguments_and_block(node.arguments, node.block).first.any? bounds(node.location) @@ -1222,7 +1278,7 @@ def visit_call_node(node) on_method_add_arg(on_fcall(message), on_args_new) end - if has_ripper_block + if block_node bounds(node.block.location) on_method_add_block(call, block) else @@ -1255,7 +1311,19 @@ def visit_call_node(node) bounds(node.location) on_assign(on_field(receiver, call_operator, message), value) else - arguments, block, has_ripper_block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc || node.location)) + if node.opening_loc + bounds(node.opening_loc) + on_lparen("(") + end + + arguments, block_node = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc || node.location)) + + if node.closing_loc + bounds(node.closing_loc) + on_rparen(")") + end + + block = visit(block_node) call = if node.opening_loc.nil? bounds(node.location) @@ -1273,7 +1341,7 @@ def visit_call_node(node) on_method_add_arg(on_call(receiver, call_operator, message), arguments) end - if has_ripper_block + if block_node bounds(node.block.location) on_method_add_block(call, block) else @@ -1315,8 +1383,7 @@ def visit_call_node(node) on_args_add_block(args, false) end end, - visit(block), - block != nil, + block, ] end @@ -1799,6 +1866,11 @@ def visit_def_node(node) bounds(node.name_loc) name = visit_token(node.name_loc.slice) + if node.lparen_loc + bounds(node.lparen_loc) + on_lparen("(") + end + parameters = if node.parameters.nil? bounds(node.location) @@ -1808,6 +1880,8 @@ def visit_def_node(node) end if !node.lparen_loc.nil? + bounds(node.rparen_loc) + on_rparen(")") bounds(node.lparen_loc) parameters = on_paren(parameters) end @@ -1849,8 +1923,18 @@ def visit_defined_node(node) bounds(node.keyword_loc) on_kw("defined?") + if node.lparen_loc + bounds(node.lparen_loc) + on_lparen("(") + end + expression = visit(node.value) + if node.rparen_loc + bounds(node.rparen_loc) + on_rparen(")") + end + # Very weird circumstances here where something like: # # defined? @@ -1956,6 +2040,10 @@ def visit_false_node(node) def visit_find_pattern_node(node) constant = visit(node.constant) + if node.opening_loc + bounds(node.opening_loc) + node.opening == "[" ? on_lbracket("[") : on_lparen("(") + end bounds(node.left.operator_loc) on_op("*") @@ -1980,6 +2068,10 @@ def visit_find_pattern_node(node) visit(node.right.expression) end + if node.closing_loc + bounds(node.closing_loc) + node.closing == "]" ? on_rbracket("]") : on_rparen(")") + end bounds(node.location) on_fndptn(constant, left, requireds, right) end @@ -2151,6 +2243,9 @@ def visit_global_variable_target_node(node) # {} # ^^ def visit_hash_node(node) + bounds(node.opening_loc) + on_lbrace("{") + elements = if node.elements.any? args = visit_all(node.elements) @@ -2159,6 +2254,8 @@ def visit_hash_node(node) on_assoclist_from_args(args) end + bounds(node.closing_loc) + on_rbrace("}") bounds(node.location) on_hash(elements) end @@ -2167,6 +2264,15 @@ def visit_hash_node(node) # ^^ def visit_hash_pattern_node(node) constant = visit(node.constant) + + if node.constant + bounds(node.opening_loc) + node.opening == "[" ? on_lbracket("[") : on_lparen("(") + elsif node.opening_loc + bounds(node.opening_loc) + on_lbrace("{") + end + elements = if node.elements.any? || !node.rest.nil? node.elements.map do |element| @@ -2197,6 +2303,13 @@ def visit_hash_pattern_node(node) on_var_field(visit(node.rest)) end + if node.constant + bounds(node.closing_loc) + node.closing == "]" ? on_rbracket("]") : on_rparen(")") + elsif node.closing_loc + bounds(node.closing_loc) + on_rbrace("}") + end bounds(node.location) on_hshptn(constant, elements, rest) end @@ -2311,7 +2424,14 @@ def visit_in_node(node) # ^^^^^^^^^^^^^^^ def visit_index_operator_write_node(node) receiver = visit(node.receiver) - arguments, _, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc)) + + bounds(node.opening_loc) + on_lbracket("[") + + arguments, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc)) + + bounds(node.closing_loc) + on_rbracket("]") bounds(node.location) target = on_aref_field(receiver, arguments) @@ -2328,7 +2448,14 @@ def visit_index_operator_write_node(node) # ^^^^^^^^^^^^^^^^ def visit_index_and_write_node(node) receiver = visit(node.receiver) - arguments, _, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc)) + + bounds(node.opening_loc) + on_lbracket("[") + + arguments, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc)) + + bounds(node.closing_loc) + on_rbracket("]") bounds(node.location) target = on_aref_field(receiver, arguments) @@ -2345,7 +2472,14 @@ def visit_index_and_write_node(node) # ^^^^^^^^^^^^^^^^ def visit_index_or_write_node(node) receiver = visit(node.receiver) - arguments, _, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc)) + + bounds(node.opening_loc) + on_lbracket("[") + + arguments, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc)) + + bounds(node.closing_loc) + on_rbracket("]") bounds(node.location) target = on_aref_field(receiver, arguments) @@ -2362,7 +2496,14 @@ def visit_index_or_write_node(node) # ^^^^^^^^ def visit_index_target_node(node) receiver = visit(node.receiver) - arguments, _, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc)) + + bounds(node.opening_loc) + on_lbracket("[") + + arguments, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc)) + + bounds(node.closing_loc) + on_rbracket("]") bounds(node.location) on_aref_field(receiver, arguments) @@ -2609,6 +2750,11 @@ def visit_lambda_node(node) parameters = if node.parameters.is_a?(BlockParametersNode) + if node.parameters.opening_loc + bounds(node.parameters.opening_loc) + on_lparen("(") + end + # Ripper does not track block-locals within lambdas, so we skip # directly to the parameters here. params = @@ -2621,6 +2767,11 @@ def visit_lambda_node(node) visit_all(node.parameters.locals) + if node.parameters.closing_loc + bounds(node.parameters.closing_loc) + on_rparen(")") + end + if node.parameters.opening_loc.nil? params else @@ -2633,13 +2784,10 @@ def visit_lambda_node(node) end braces = node.opening == "{" + bounds(node.opening_loc) if braces - bounds(node.opening_loc) on_tlambeg(node.opening) - end - - unless braces - bounds(node.opening_loc) + else on_kw("do") end @@ -2664,8 +2812,10 @@ def visit_lambda_node(node) raise end - unless braces bounds(node.closing_loc) + if braces + on_rbrace("}") + else on_kw("end") end @@ -2821,9 +2971,19 @@ def visit_module_node(node) # (foo, bar), bar = qux # ^^^^^^^^^^ def visit_multi_target_node(node) + if node.lparen_loc + bounds(node.lparen_loc) + on_lparen("(") + end + bounds(node.location) targets = visit_multi_target_node_targets(node.lefts, node.rest, node.rights, true) + if node.rparen_loc + bounds(node.rparen_loc) + on_rparen(")") + end + if node.lparen_loc.nil? targets else @@ -2875,9 +3035,19 @@ def visit_multi_target_node(node) # foo, bar = baz # ^^^^^^^^^^^^^^ def visit_multi_write_node(node) + if node.lparen_loc + bounds(node.lparen_loc) + on_lparen("(") + end + bounds(node.location) targets = visit_multi_target_node_targets(node.lefts, node.rest, node.rights, true) + if node.rparen_loc + bounds(node.rparen_loc) + on_rparen(")") + end + bounds(node.operator_loc) on_op("=") @@ -3014,9 +3184,19 @@ def visit_parameters_node(node) # Visit a destructured positional parameter node. private def visit_destructured_parameter_node(node) + if node.lparen_loc + bounds(node.lparen_loc) + on_lparen("(") + end + bounds(node.location) targets = visit_multi_target_node_targets(node.lefts, node.rest, node.rights, false) + if node.rparen_loc + bounds(node.rparen_loc) + on_rparen(")") + end + bounds(node.lparen_loc) on_mlhs_paren(targets) end @@ -3027,6 +3207,9 @@ def visit_parameters_node(node) # (1) # ^^^ def visit_parentheses_node(node) + bounds(node.opening_loc) + on_lparen("(") + body = if node.body.nil? on_stmts_add(on_stmts_new, on_void_stmt) @@ -3034,6 +3217,8 @@ def visit_parentheses_node(node) visit(node.body) end + bounds(node.closing_loc) + on_rparen(")") bounds(node.location) on_paren(body) end @@ -3043,9 +3228,13 @@ def visit_parentheses_node(node) def visit_pinned_expression_node(node) bounds(node.operator_loc) on_op("^") + bounds(node.lparen_loc) + on_lparen("(") expression = visit(node.expression) + bounds(node.rparen_loc) + on_rparen(")") bounds(node.location) on_begin(expression) end @@ -3064,6 +3253,8 @@ def visit_pinned_variable_node(node) def visit_post_execution_node(node) bounds(node.keyword_loc) on_kw("END") + bounds(node.opening_loc) + on_lbrace("{") statements = if node.statements.nil? @@ -3073,6 +3264,8 @@ def visit_post_execution_node(node) visit(node.statements) end + bounds(node.closing_loc) + on_rbrace("}") bounds(node.location) on_END(statements) end @@ -3082,6 +3275,8 @@ def visit_post_execution_node(node) def visit_pre_execution_node(node) bounds(node.keyword_loc) on_kw("BEGIN") + bounds(node.opening_loc) + on_lbrace("{") statements = if node.statements.nil? @@ -3091,6 +3286,8 @@ def visit_pre_execution_node(node) visit(node.statements) end + bounds(node.closing_loc) + on_rbrace("}") bounds(node.location) on_BEGIN(statements) end @@ -3532,7 +3729,19 @@ def visit_super_node(node) bounds(node.keyword_loc) on_kw("super") - arguments, block, has_ripper_block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.rparen_loc || node.location)) + if node.lparen_loc + bounds(node.lparen_loc) + on_lparen("(") + end + + arguments, block_node = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.rparen_loc || node.location)) + + if node.rparen_loc + bounds(node.rparen_loc) + on_rparen(")") + end + + block = visit(block_node) if !node.lparen_loc.nil? bounds(node.lparen_loc) @@ -3542,7 +3751,7 @@ def visit_super_node(node) bounds(node.location) call = on_super(arguments) - if has_ripper_block + if block_node bounds(node.block.location) on_method_add_block(call, block) else @@ -3773,6 +3982,11 @@ def visit_yield_node(node) bounds(node.location) on_yield0 else + if node.lparen_loc + bounds(node.lparen_loc) + on_lparen("(") + end + arguments = if node.arguments.nil? bounds(node.location) @@ -3782,6 +3996,8 @@ def visit_yield_node(node) end unless node.lparen_loc.nil? + bounds(node.rparen_loc) + on_rparen(")") bounds(node.lparen_loc) arguments = on_paren(arguments) end diff --git a/test/prism/ruby/ripper_test.rb b/test/prism/ruby/ripper_test.rb index 6abe1bb2e5..61065e3ffc 100644 --- a/test/prism/ruby/ripper_test.rb +++ b/test/prism/ruby/ripper_test.rb @@ -83,6 +83,8 @@ class RipperTest < TestCase ] omitted_scan = [ + "bom_leading_space.txt", + "bom_spaces.txt", "dos_endings.txt", "heredocs_with_fake_newlines.txt", "rescue_modifier.txt", @@ -136,95 +138,11 @@ def test_lex_ignored_missing_heredoc_end end end + UNSUPPORTED_EVENTS = %i[comma ignored_nl label_end nl semicolon sp words_sep ignored_sp] # Events that are currently not emitted - UNSUPPORTED_EVENTS = %i[comma ignored_nl label_end lbrace lbracket lparen nl rbrace rbracket rparen semicolon sp words_sep ignored_sp] SUPPORTED_EVENTS = Translation::Ripper::EVENTS - UNSUPPORTED_EVENTS # Events that assert against their line/column - CHECK_LOCATION_EVENTS = %i[kw op] - IGNORE_FOR_SORT_EVENTS = %i[ - stmts_new stmts_add bodystmt void_stmt - args_new args_add args_add_star args_add_block arg_paren method_add_arg - mlhs_new mlhs_add mlhs_add_star mlhs_add_post - mrhs_new mrhs_add mrhs_add_star mrhs_new_from_args - word_new words_new symbols_new qwords_new qsymbols_new xstring_new regexp_new - words_add symbols_add qwords_add qsymbols_add - regexp_end tstring_end heredoc_end - call command fcall vcall - field aref_field var_field var_ref block_var ident params - string_content heredoc_dedent unary binary dyna_symbol - excessed_comma rest_param - comment magic_comment embdoc embdoc_beg embdoc_end arg_ambiguous - ] - SORT_IGNORE = { - aref: [ - "blocks.txt", - "command_method_call.txt", - "whitequark/ruby_bug_13547.txt", - ], - assoc_new: [ - "case_in_hash_key.txt", - "whitequark/parser_bug_525.txt", - "whitequark/ruby_bug_11380.txt", - ], - bare_assoc_hash: [ - "case_in_hash_key.txt", - "method_calls.txt", - "whitequark/parser_bug_525.txt", - "whitequark/ruby_bug_11380.txt", - ], - brace_block: [ - "super.txt", - "unparser/corpus/literal/super.txt" - ], - command_call: [ - "blocks.txt", - "case_in_hash_key.txt", - "seattlerb/block_call_dot_op2_cmd_args_do_block.txt", - "seattlerb/block_call_operation_colon.txt", - "seattlerb/block_call_operation_dot.txt", - ], - const_path_field: [ - "seattlerb/const_2_op_asgn_or2.txt", - "seattlerb/const_op_asgn_or.txt", - "whitequark/const_op_asgn.txt", - ], - const_path_ref: ["unparser/corpus/literal/defs.txt"], - do_block: ["whitequark/super_block.txt"], - embexpr_end: ["seattlerb/str_interp_ternary_or_label.txt"], - top_const_field: [ - "seattlerb/const_3_op_asgn_or.txt", - "seattlerb/const_op_asgn_and1.txt", - "seattlerb/const_op_asgn_and2.txt", - "whitequark/const_op_asgn.txt", - ], - mlhs_paren: ["unparser/corpus/literal/for.txt"], - kw: [ - "defined.txt", - "for.txt", - "seattlerb/case_in_42.txt", - "seattlerb/case_in_67.txt", - "seattlerb/case_in_86_2.txt", - "seattlerb/case_in_86.txt", - "seattlerb/case_in_hash_pat_paren_true.txt", - "seattlerb/flip2_env_lvar.txt", - "unless.txt", - "unparser/corpus/semantic/and.txt", - "whitequark/class.txt", - "whitequark/find_pattern.txt", - "whitequark/pattern_matching_hash.txt", - "whitequark/pattern_matching_implicit_array_match.txt", - "whitequark/pattern_matching_ranges.txt", - "whitequark/super_block.txt", - "write_command_operator.txt", - ], - op: [ - "ranges.txt", - "ternary_operator.txt", - "whitequark/args_args_assocs.txt", - ] - } - SORT_IGNORE.default = [] - SORT_EVENTS = SUPPORTED_EVENTS - IGNORE_FOR_SORT_EVENTS + CHECK_LOCATION_EVENTS = %i[kw op lbrace rbrace lbracket rbracket lparen rparen] module Events attr_reader :events @@ -234,20 +152,12 @@ def initialize(...) @events = [] end - def sorted_events - @events.select do |e,| - next false if e == :kw && @events.any? { |e,| e == :if_mod || e == :while_mod || e == :until_mod || e == :rescue || e == :rescue_mod || e == :while || e == :ensure } - next false if e == :op && @events.any? { |e,| e == :const_path_field || e == :const_ref || e == :top_const_field || e == :top_const_ref } - SORT_EVENTS.include?(e) && !SORT_IGNORE[e].include?(filename) - end - end - SUPPORTED_EVENTS.each do |event| define_method(:"on_#{event}") do |*args| if CHECK_LOCATION_EVENTS.include?(event) - @events << [event, lineno, column, *args.map(&:to_s)] + @events << [event, lineno, column, *args] else - @events << [event, *args.map(&:to_s)] + @events << [event, *args] end super(*args) end @@ -281,9 +191,7 @@ class ObjectEvents < Translation::Ripper ripper.parse prism.parse # Check that the same events are emitted, regardless of order - assert_equal(ripper.events.sort, prism.events.sort) - # Check a subset of events against the correct order - assert_equal(ripper.sorted_events, prism.sorted_events) + assert_equal(ripper.events.sort_by(&:inspect), prism.events.sort_by(&:inspect)) end end