Commit 58f11a5a authored by Rene Saarsoo's avatar Rene Saarsoo
Browse files

Added line-number support to Lexer. 

Currently we only need line-number information for doc-comment tokens.

Line-number detection was inspired by ruby_parser library:
https://github.com/seattlerb/ruby_parser/blob/master/lib/ruby_parser_extras.rb
parent 228769f3

lib/jsduck/lexer.rb

+5 −0
Original line number Diff line number Diff line
@@ -53,6 +53,9 @@ module JsDuck 
    #

    #     {:type => :ident, :value => "foo"}

    #

    # For doc-comments the full token also contains the field :linenr,

    # pointing to the line where the doc-comment began.

    #

    def next(full=false)

      tok = @tokens.shift

      full ? tok : tok[:value]
@@ -82,6 +85,8 @@ module JsDuck 
        elsif @input.check(/\/\*\*/) then

          @tokens << {

            :type => :doc_comment,

            # Calculate current line number, starting with 1

            :linenr => @input.string[0...@input.pos].count("\n") + 1,

            :value => @input.scan_until(/\*\/|\Z/)

          }

        elsif @input.check(/"/) then

spec/lexer_spec.rb

+17 −3
Original line number Diff line number Diff line
@@ -8,6 +8,9 @@ describe JsDuck::Lexer do 
    while !lex.empty?

      t = lex.next(true)

      tokens << [t[:type], t[:value]]

      if t[:linenr]

        tokens.last << t[:linenr]

      end

    end

    tokens

  end
@@ -96,8 +99,19 @@ describe JsDuck::Lexer do 
    lex("a /* foo */ b").should == [[:ident, "a"], [:ident, "b"]]

  end



  it "identifies doc-comments" do

    lex("/** foo */").should == [[:doc_comment, "/** foo */"]]

  it "identifies doc-comments together with line numbers" do

    lex("/** foo */").should == [[:doc_comment, "/** foo */", 1]]

  end



  it "counts line numbers correctly" do

    tokens = lex(<<-EOS)

      foo = {

        bar: foo,

        /**

         * My comment.

         */

    EOS

    tokens.last.last.should == 3

  end



  describe "handles unfinished" do
@@ -111,7 +125,7 @@ describe JsDuck::Lexer do 
    end



    it "doc-comment" do

      lex("/** ").should == [[:doc_comment, "/** "]]

      lex("/** ").should == [[:doc_comment, "/** ", 1]]

    end

  end