diff --git a/lib/prism/translation/ripper.rb b/lib/prism/translation/ripper.rb index bbfa1f4d05..2f66bab97e 100644 --- a/lib/prism/translation/ripper.rb +++ b/lib/prism/translation/ripper.rb @@ -606,6 +606,9 @@ def parse # alias foo bar # ^^^^^^^^^^^^^ def visit_alias_method_node(node) + bounds(node.keyword_loc) + on_kw("alias") + new_name = visit(node.new_name) old_name = visit(node.old_name) @@ -616,6 +619,9 @@ def visit_alias_method_node(node) # alias $foo $bar # ^^^^^^^^^^^^^^^ def visit_alias_global_variable_node(node) + bounds(node.keyword_loc) + on_kw("alias") + new_name = visit_alias_global_variable_node_value(node.new_name) old_name = visit_alias_global_variable_node_value(node.old_name) @@ -661,6 +667,10 @@ def visit_alternation_pattern_node(node) # ^^^^^^^ def visit_and_node(node) left = visit(node.left) + if node.operator == "and" + bounds(node.operator_loc) + on_kw("and") + end right = visit(node.right) bounds(node.location) @@ -887,8 +897,18 @@ def visit_back_reference_read_node(node) # begin end # ^^^^^^^^^ def visit_begin_node(node) + if node.begin_keyword_loc + bounds(node.begin_keyword_loc) + on_kw("begin") + end + clauses = visit_begin_node_clauses(node.begin_keyword_loc, node, false) + if node.end_keyword_loc + bounds(node.end_keyword_loc) + on_kw("end") + end + bounds(node.location) on_begin(clauses) end @@ -909,6 +929,9 @@ def visit_begin_node(node) rescue_clause = visit(node.rescue_clause) else_clause = unless (else_clause_node = node.else_clause).nil? + bounds(else_clause_node.else_keyword_loc) + on_kw("else") + else_statements = if else_clause_node.statements.nil? [nil] @@ -966,6 +989,11 @@ def visit_block_node(node) braces = node.opening == "{" parameters = visit(node.parameters) + unless braces + bounds(node.opening_loc) + on_kw("do") + end + body = case node.body when nil @@ -987,6 +1015,11 @@ def visit_block_node(node) raise end + unless braces + bounds(node.closing_loc) + on_kw("end") + end + if braces bounds(node.location) on_brace_block(parameters, body) @@ -1037,6 +1070,9 @@ def visit_block_parameters_node(node) # break foo # ^^^^^^^^^ def visit_break_node(node) + bounds(node.keyword_loc) + on_kw("break") + if node.arguments.nil? bounds(node.location) on_break(on_args_new) @@ -1103,6 +1139,9 @@ def visit_call_node(node) on_unary(node.name, receiver) when :! if node.message == "not" + bounds(node.message_loc) + on_kw("not") + receiver = if !node.receiver.is_a?(ParenthesesNode) || !node.receiver.body.nil? visit(node.receiver) @@ -1347,10 +1386,21 @@ def visit_capture_pattern_node(node) # case foo; when bar; end # ^^^^^^^^^^^^^^^^^^^^^^^ def visit_case_node(node) + bounds(node.case_keyword_loc) + on_kw("case") + predicate = visit(node.predicate) + visited_conditions = node.conditions.map { |condition| visit(condition) } + visited_else_clause = visit(node.else_clause) + + if !node.else_clause + bounds(node.end_keyword_loc) + on_kw("end") + end + clauses = - node.conditions.reverse_each.inject(visit(node.else_clause)) do |current, condition| - on_when(*visit(condition), current) + visited_conditions.reverse_each.inject(visited_else_clause) do |current, condition| + on_when(*condition, current) end bounds(node.location) @@ -1360,10 +1410,23 @@ def visit_case_node(node) # case foo; in bar; end # ^^^^^^^^^^^^^^^^^^^^^ def visit_case_match_node(node) + bounds(node.case_keyword_loc) + on_kw("case") + predicate = visit(node.predicate) + visited_conditions = node.conditions.map do | condition| + visit(condition) + end + visited_else_clause = visit(node.else_clause) + + if !node.else_clause + bounds(node.end_keyword_loc) + on_kw("end") + end + clauses = - node.conditions.reverse_each.inject(visit(node.else_clause)) do |current, condition| - on_in(*visit(condition), current) + visited_conditions.reverse_each.inject(visited_else_clause) do |current, condition| + on_in(*condition, current) end bounds(node.location) @@ -1373,6 +1436,9 @@ def visit_case_match_node(node) # class Foo; end # ^^^^^^^^^^^^^^ def visit_class_node(node) + bounds(node.class_keyword_loc) + on_kw("class") + constant_path = if node.constant_path.is_a?(ConstantReadNode) bounds(node.constant_path.location) @@ -1384,6 +1450,9 @@ def visit_class_node(node) superclass = visit(node.superclass) bodystmt = visit_body_node(node.superclass&.location || node.constant_path.location, node.body, node.superclass.nil?) + bounds(node.end_keyword_loc) + on_kw("end") + bounds(node.location) on_class(constant_path, superclass, bodystmt) end @@ -1631,6 +1700,9 @@ def visit_constant_path_target_node(node) # def self.foo; end # ^^^^^^^^^^^^^^^^^ def visit_def_node(node) + bounds(node.def_keyword_loc) + on_kw("def") + receiver = visit(node.receiver) operator = if !node.operator_loc.nil? @@ -1664,6 +1736,11 @@ def visit_def_node(node) on_bodystmt(body, nil, nil, nil) end + if node.end_keyword_loc + bounds(node.end_keyword_loc) + on_kw("end") + end + bounds(node.location) if receiver on_defs(receiver, operator, name, parameters, bodystmt) @@ -1678,6 +1755,9 @@ def visit_def_node(node) # defined?(a) # ^^^^^^^^^^^ def visit_defined_node(node) + bounds(node.keyword_loc) + on_kw("defined?") + expression = visit(node.value) # Very weird circumstances here where something like: @@ -1700,6 +1780,9 @@ def visit_defined_node(node) # if foo then bar else baz end # ^^^^^^^^^^^^ def visit_else_node(node) + bounds(node.else_keyword_loc) + on_kw("else") + statements = if node.statements.nil? [nil] @@ -1709,8 +1792,12 @@ def visit_else_node(node) body end + else_statements = visit_statements_node_body(statements) + + bounds(node.end_keyword_loc) + on_kw("end") bounds(node.location) - on_else(visit_statements_node_body(statements)) + on_else(else_statements) end # "foo #{bar}" @@ -1748,6 +1835,9 @@ def visit_embedded_variable_node(node) # Visit an EnsureNode node. def visit_ensure_node(node) + bounds(node.ensure_keyword_loc) + on_kw("ensure") + statements = if node.statements.nil? [nil] @@ -1818,8 +1908,18 @@ def visit_float_node(node) # for foo in bar do end # ^^^^^^^^^^^^^^^^^^^^^ def visit_for_node(node) + bounds(node.for_keyword_loc) + on_kw("for") + index = visit(node.index) + bounds(node.in_keyword_loc) + on_kw("in") + collection = visit(node.collection) + if node.do_keyword_loc + bounds(node.do_keyword_loc) + on_kw("do") + end statements = if node.statements.nil? bounds(node.location) @@ -1828,6 +1928,9 @@ def visit_for_node(node) visit(node.statements) end + bounds(node.end_keyword_loc) + on_kw("end") + bounds(node.location) on_for(index, collection, statements) end @@ -1852,6 +1955,9 @@ def visit_forwarding_parameter_node(node) # super {} # ^^^^^^^^ def visit_forwarding_super_node(node) + bounds(node.keyword_loc) + on_kw("super") + if node.block.nil? bounds(node.location) on_zsuper @@ -2001,7 +2107,13 @@ def visit_if_node(node) bounds(node.location) on_ifop(predicate, truthy, falsy) elsif node.statements.nil? || (node.predicate.location.start_offset < node.statements.location.start_offset) + bounds(node.if_keyword_loc) + on_kw(node.if_keyword) predicate = visit(node.predicate) + if node.then_keyword_loc && node.then_keyword != "?" + bounds(node.then_keyword_loc) + on_kw("then") + end statements = if node.statements.nil? bounds(node.location) @@ -2011,6 +2123,11 @@ def visit_if_node(node) end subsequent = visit(node.subsequent) + if node.end_keyword_loc && !node.subsequent + bounds(node.end_keyword_loc) + on_kw("end") + end + bounds(node.location) if node.if_keyword == "if" on_if(predicate, statements, subsequent) @@ -2019,6 +2136,8 @@ def visit_if_node(node) end else statements = visit(node.statements.body.first) + bounds(node.if_keyword_loc) + on_kw(node.if_keyword) predicate = visit(node.predicate) bounds(node.location) @@ -2050,7 +2169,14 @@ def visit_in_node(node) # This is a special case where we're not going to call on_in directly # because we don't have access to the subsequent. Instead, we'll return # the component parts and let the parent node handle it. + bounds(node.in_loc) + on_kw("in") + pattern = visit_pattern_node(node.pattern) + if node.then_loc + bounds(node.then_loc) + on_kw("then") + end statements = if node.statements.nil? bounds(node.location) @@ -2386,6 +2512,11 @@ def visit_lambda_node(node) on_tlambeg(node.opening) end + unless braces + bounds(node.opening_loc) + on_kw("do") + end + body = case node.body when nil @@ -2407,6 +2538,11 @@ def visit_lambda_node(node) raise end + unless braces + bounds(node.closing_loc) + on_kw("end") + end + bounds(node.location) on_lambda(parameters, body) end @@ -2497,6 +2633,8 @@ def visit_match_last_line_node(node) # ^^^^^^^^^^ def visit_match_predicate_node(node) value = visit(node.value) + bounds(node.operator_loc) + on_kw("in") pattern = on_in(visit_pattern_node(node.pattern), nil, nil) on_case(value, pattern) @@ -2526,6 +2664,9 @@ def visit_error_recovery_node(node) # module Foo; end # ^^^^^^^^^^^^^^^ def visit_module_node(node) + bounds(node.module_keyword_loc) + on_kw("module") + constant_path = if node.constant_path.is_a?(ConstantReadNode) bounds(node.constant_path.location) @@ -2536,6 +2677,9 @@ def visit_module_node(node) bodystmt = visit_body_node(node.constant_path.location, node.body, true) + bounds(node.end_keyword_loc) + on_kw("end") + bounds(node.location) on_module(constant_path, bodystmt) end @@ -2617,6 +2761,9 @@ def visit_multi_write_node(node) # next foo # ^^^^^^^^ def visit_next_node(node) + bounds(node.keyword_loc) + on_kw("next") + if node.arguments.nil? bounds(node.location) on_next(on_args_new) @@ -2638,6 +2785,8 @@ def visit_nil_node(node) # def foo(&nil); end # ^^^^ def visit_no_block_parameter_node(node) + bounds(node.keyword_loc) + on_kw("nil") bounds(node.location) on_blockarg(:nil) end @@ -2645,6 +2794,8 @@ def visit_no_block_parameter_node(node) # def foo(**nil); end # ^^^^^ def visit_no_keywords_parameter_node(node) + bounds(node.keyword_loc) + on_kw("nil") bounds(node.location) on_nokw_param(nil) @@ -2687,6 +2838,10 @@ def visit_optional_parameter_node(node) # ^^^^^^ def visit_or_node(node) left = visit(node.left) + if node.operator == "or" + bounds(node.operator_loc) + on_kw("or") + end right = visit(node.right) bounds(node.location) @@ -2752,6 +2907,9 @@ def visit_pinned_variable_node(node) # END {} # ^^^^^^ def visit_post_execution_node(node) + bounds(node.keyword_loc) + on_kw("END") + statements = if node.statements.nil? bounds(node.location) @@ -2767,6 +2925,9 @@ def visit_post_execution_node(node) # BEGIN {} # ^^^^^^^^ def visit_pre_execution_node(node) + bounds(node.keyword_loc) + on_kw("BEGIN") + statements = if node.statements.nil? bounds(node.location) @@ -2813,6 +2974,7 @@ def visit_rational_node(node) # ^^^^ def visit_redo_node(node) bounds(node.location) + on_kw("redo") on_redo end @@ -2855,6 +3017,9 @@ def visit_required_parameter_node(node) # foo rescue bar # ^^^^^^^^^^^^^^ def visit_rescue_modifier_node(node) + bounds(node.keyword_loc) + on_kw("rescue") + expression = visit_write_value(node.expression) rescue_expression = visit(node.rescue_expression) @@ -2865,6 +3030,9 @@ def visit_rescue_modifier_node(node) # begin; rescue; end # ^^^^^^^ def visit_rescue_node(node) + bounds(node.keyword_loc) + on_kw("rescue") + exceptions = case node.exceptions.length when 0 @@ -2936,6 +3104,7 @@ def visit_rest_parameter_node(node) # ^^^^^ def visit_retry_node(node) bounds(node.location) + on_kw("retry") on_retry end @@ -2945,6 +3114,9 @@ def visit_retry_node(node) # return 1 # ^^^^^^^^ def visit_return_node(node) + bounds(node.keyword_loc) + on_kw("return") + if node.arguments.nil? bounds(node.location) on_return0 @@ -2971,9 +3143,15 @@ def visit_shareable_constant_node(node) # class << self; end # ^^^^^^^^^^^^^^^^^^ def visit_singleton_class_node(node) + bounds(node.class_keyword_loc) + on_kw("class") + expression = visit(node.expression) bodystmt = visit_body_node(node.body&.location || node.end_keyword_loc, node.body) + bounds(node.end_keyword_loc) + on_kw("end") + bounds(node.location) on_sclass(expression, bodystmt) end @@ -3180,6 +3358,9 @@ def visit_string_node(node) # super(foo) # ^^^^^^^^^^ def visit_super_node(node) + bounds(node.keyword_loc) + on_kw("super") + arguments, block, has_ripper_block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.rparen_loc || node.location)) if !node.lparen_loc.nil? @@ -3233,6 +3414,9 @@ def visit_true_node(node) # undef foo # ^^^^^^^^^ def visit_undef_node(node) + bounds(node.keyword_loc) + on_kw("undef") + names = visit_all(node.names) bounds(node.location) @@ -3246,7 +3430,13 @@ def visit_undef_node(node) # ^^^^^^^^^^^^^^ def visit_unless_node(node) if node.statements.nil? || (node.predicate.location.start_offset < node.statements.location.start_offset) + bounds(node.keyword_loc) + on_kw("unless") predicate = visit(node.predicate) + if node.then_keyword_loc + bounds(node.then_keyword_loc) + on_kw("then") + end statements = if node.statements.nil? bounds(node.location) @@ -3256,10 +3446,17 @@ def visit_unless_node(node) end else_clause = visit(node.else_clause) + if node.end_keyword_loc && !node.else_clause + bounds(node.end_keyword_loc) + on_kw("end") + end + bounds(node.location) on_unless(predicate, statements, else_clause) else statements = visit(node.statements.body.first) + bounds(node.keyword_loc) + on_kw("unless") predicate = visit(node.predicate) bounds(node.location) @@ -3273,7 +3470,14 @@ def visit_unless_node(node) # bar until foo # ^^^^^^^^^^^^^ def visit_until_node(node) + bounds(node.keyword_loc) + on_kw("until") + if node.statements.nil? || (node.predicate.location.start_offset < node.statements.location.start_offset) + if node.do_keyword_loc + bounds(node.do_keyword_loc) + on_kw("do") + end predicate = visit(node.predicate) statements = if node.statements.nil? @@ -3283,6 +3487,11 @@ def visit_until_node(node) visit(node.statements) end + if node.closing_loc + bounds(node.closing_loc) + on_kw("end") + end + bounds(node.location) on_until(predicate, statements) else @@ -3300,7 +3509,14 @@ def visit_when_node(node) # This is a special case where we're not going to call on_when directly # because we don't have access to the subsequent. Instead, we'll return # the component parts and let the parent node handle it. + bounds(node.keyword_loc) + on_kw("when") + conditions = visit_arguments(node.conditions) + if node.then_keyword_loc + bounds(node.then_keyword_loc) + on_kw("then") + end statements = if node.statements.nil? bounds(node.location) @@ -3319,7 +3535,17 @@ def visit_when_node(node) # ^^^^^^^^^^^^^ def visit_while_node(node) if node.statements.nil? || (node.predicate.location.start_offset < node.statements.location.start_offset) + bounds(node.keyword_loc) + on_kw("while") + if node.do_keyword_loc + bounds(node.do_keyword_loc) + on_kw("do") + end predicate = visit(node.predicate) + if node.closing_loc + bounds(node.closing_loc) + on_kw("end") + end statements = if node.statements.nil? bounds(node.location) @@ -3332,6 +3558,8 @@ def visit_while_node(node) on_while(predicate, statements) else statements = visit(node.statements.body.first) + bounds(node.keyword_loc) + on_kw("while") predicate = visit(node.predicate) bounds(node.location) @@ -3367,6 +3595,9 @@ def visit_x_string_node(node) # yield 1 # ^^^^^^^ def visit_yield_node(node) + bounds(node.keyword_loc) + on_kw("yield") + if node.arguments.nil? && node.lparen_loc.nil? bounds(node.location) on_yield0 diff --git a/test/prism/newline_test.rb b/test/prism/newline_test.rb index c8914b57dc..97e698202d 100644 --- a/test/prism/newline_test.rb +++ b/test/prism/newline_test.rb @@ -20,6 +20,7 @@ class NewlineTest < TestCase ruby/find_fixtures.rb ruby/find_test.rb ruby/parser_test.rb + ruby/ripper_test.rb ruby/ruby_parser_test.rb ] diff --git a/test/prism/ruby/ripper_test.rb b/test/prism/ruby/ripper_test.rb index 7274454e1b..1d20bceb40 100644 --- a/test/prism/ruby/ripper_test.rb +++ b/test/prism/ruby/ripper_test.rb @@ -136,8 +136,93 @@ def test_lex_ignored_missing_heredoc_end end end - UNSUPPORTED_EVENTS = %i[comma ignored_nl kw label_end lbrace lbracket lparen nl op rbrace rbracket rparen semicolon sp words_sep ignored_sp] + # Events that are currently not emitted + UNSUPPORTED_EVENTS = %i[comma ignored_nl label_end lbrace lbracket lparen nl op rbrace rbracket rparen semicolon sp words_sep ignored_sp] SUPPORTED_EVENTS = Translation::Ripper::EVENTS - UNSUPPORTED_EVENTS + # Events that assert against their line/column + CHECK_LOCATION_EVENTS = %i[kw] + IGNORE_FOR_SORT_EVENTS = %i[ + stmts_new stmts_add bodystmt void_stmt + args_new args_add args_add_star args_add_block arg_paren method_add_arg + mlhs_new mlhs_add_star + word_new words_new symbols_new qwords_new qsymbols_new xstring_new regexp_new + words_add symbols_add qwords_add qsymbols_add + regexp_end tstring_end heredoc_end + call command fcall vcall + field aref_field var_field var_ref block_var ident params + string_content heredoc_dedent unary binary dyna_symbol + comment magic_comment embdoc embdoc_beg embdoc_end arg_ambiguous + ] + SORT_IGNORE = { + aref: [ + "blocks.txt", + "command_method_call.txt", + "whitequark/ruby_bug_13547.txt", + ], + assoc_new: [ + "case_in_hash_key.txt", + "whitequark/parser_bug_525.txt", + "whitequark/ruby_bug_11380.txt", + ], + bare_assoc_hash: [ + "case_in_hash_key.txt", + "method_calls.txt", + "whitequark/parser_bug_525.txt", + "whitequark/ruby_bug_11380.txt", + ], + brace_block: [ + "super.txt", + "unparser/corpus/literal/super.txt" + ], + command_call: [ + "blocks.txt", + "case_in_hash_key.txt", + "seattlerb/block_call_dot_op2_cmd_args_do_block.txt", + "seattlerb/block_call_operation_colon.txt", + "seattlerb/block_call_operation_dot.txt", + ], + const_path_field: [ + "seattlerb/const_2_op_asgn_or2.txt", + "seattlerb/const_op_asgn_or.txt", + "whitequark/const_op_asgn.txt", + ], + const_path_ref: ["unparser/corpus/literal/defs.txt"], + do_block: ["whitequark/super_block.txt"], + embexpr_end: ["seattlerb/str_interp_ternary_or_label.txt"], + rest_param: ["whitequark/send_lambda.txt"], + top_const_field: [ + "seattlerb/const_3_op_asgn_or.txt", + "seattlerb/const_op_asgn_and1.txt", + "seattlerb/const_op_asgn_and2.txt", + "whitequark/const_op_asgn.txt", + ], + mlhs_paren: ["unparser/corpus/literal/for.txt"], + mlhs_add: [ + "whitequark/for_mlhs.txt", + ], + kw: [ + "defined.txt", + "for.txt", + "seattlerb/block_kw__required.txt", + "seattlerb/case_in_42.txt", + "seattlerb/case_in_67.txt", + "seattlerb/case_in_86_2.txt", + "seattlerb/case_in_86.txt", + "seattlerb/case_in_hash_pat_paren_true.txt", + "seattlerb/flip2_env_lvar.txt", + "unless.txt", + "unparser/corpus/semantic/and.txt", + "whitequark/class.txt", + "whitequark/find_pattern.txt", + "whitequark/pattern_matching_hash.txt", + "whitequark/pattern_matching_implicit_array_match.txt", + "whitequark/pattern_matching_ranges.txt", + "whitequark/super_block.txt", + "write_command_operator.txt", + ], + } + SORT_IGNORE.default = [] + SORT_EVENTS = SUPPORTED_EVENTS - IGNORE_FOR_SORT_EVENTS module Events attr_reader :events @@ -147,9 +232,20 @@ def initialize(...) @events = [] end + def sorted_events + @events.select do |e,| + next false if e == :kw && @events.any? { |e,| e == :if_mod || e == :while_mod || e == :until_mod || e == :rescue || e == :rescue_mod || e == :while || e == :ensure } + SORT_EVENTS.include?(e) && !SORT_IGNORE[e].include?(filename) + end + end + SUPPORTED_EVENTS.each do |event| define_method(:"on_#{event}") do |*args| - @events << [event, *args.map(&:to_s)] + if CHECK_LOCATION_EVENTS.include?(event) + @events << [event, lineno, column, *args.map(&:to_s)] + else + @events << [event, *args.map(&:to_s)] + end super(*args) end end @@ -177,12 +273,14 @@ class ObjectEvents < Translation::Ripper object_events = ObjectEvents.new(source) assert_nothing_raised { object_events.parse } - ripper = RipperEvents.new(source) - prism = PrismEvents.new(source) + ripper = RipperEvents.new(source, fixture.path) + prism = PrismEvents.new(source, fixture.path) ripper.parse prism.parse - # This makes sure that the content is the same. Ordering is not correct for now. + # Check that the same events are emitted, regardless of order assert_equal(ripper.events.sort, prism.events.sort) + # Check a subset of events against the correct order + assert_equal(ripper.sorted_events, prism.sorted_events) end end