Commit 5bcd246b authored by Rene Saarsoo's avatar Rene Saarsoo
Browse files

Fix crash when string "function" after doc-comment. 

All keywords are now treated as different token types, so we don't
confuse function and "function".
parent fd4f1cea

lib/jsduck/js_parser.rb

+9 −8
Original line number Diff line number Diff line
@@ -73,9 +73,9 @@ module JsDuck 
    # <code-block> := <function> | <var-declaration> | <ext-define> |

    #                 <assignment> | <property-literal>

    def code_block

      if look("function")

      if look(:function)

        function

      elsif look("var")

      elsif look(:var)

        var_declaration

      elsif ext_look(:ns, ".", "define", "(", :string)

        ext_define
@@ -86,7 +86,7 @@ module JsDuck 
      elsif look(",", :ident, ":") || look(",", :string, ":")

        match(",")

        property_literal

      elsif look(:ident) || look("this")

      elsif look(:ident) || look(:this)

        maybe_assignment

      elsif look(:string)

        {:type => :assignment, :left => [match(:string)[:value]]}
@@ -97,7 +97,7 @@ module JsDuck 


    # <function> := "function" [ <ident> ] <function-parameters> <function-body>

    def function

      match("function")

      match(:function)

      return {

        :type => :function,

        :name => look(:ident) ? match(:ident)[:value] : "",
@@ -124,7 +124,7 @@ module JsDuck 


    # <var-declaration> := "var" <assignment>

    def var_declaration

      match("var")

      match(:var)

      maybe_assignment

    end
@@ -144,8 +144,9 @@ module JsDuck 


    # <ident-chain> := [ "this" | <ident> ]  [ "." <ident> ]*

    def ident_chain

      if look("this")

        chain = [match("this")[:value]]

      if look(:this)

        match(:this)

        chain = ["this"]

      else

        chain = [match(:ident)[:value]]

      end
@@ -158,7 +159,7 @@ module JsDuck 


    # <expression> := <function> | <ext-extend> | <ext-base-css-prefix> | <literal>

    def expression

      if look("function")

      if look(:function)

        function

      elsif ext_look(:ns, ".", "extend")

        ext_extend

lib/jsduck/lexer.rb

+32 −29
Original line number Diff line number Diff line
@@ -9,12 +9,14 @@ module JsDuck 
  #

  # - :number      -- 25

  # - :string      -- "Hello world"

  # - :keyword     -- "typeof"

  # - :ident       -- "foo"

  # - :regex       -- "/abc/i"

  # - :operator    -- "+"

  # - :doc_comment -- "/** My comment */"

  #

  # Plus a separate types for all keywords: :if, :while, :function, ...

  # For keywords the type and value are the same.

  #

  # Notice that doc-comments are recognized as tokens while normal

  # comments are ignored just as whitespace.

  #
@@ -120,9 +122,10 @@ module JsDuck 
          }

        elsif @input.check(/[a-zA-Z_$]/)

          value = @input.scan(/[$\w]+/)

          kw = KEYWORDS[value]

          return {

            :type => KEYWORDS[value] ? :keyword : :ident,

            :value => value

            :type => kw || :ident,

            :value => kw || value

          }

        elsif @input.check(/'/)

          return {
@@ -189,7 +192,7 @@ module JsDuck 
        value = @previous_token[:value]

        if type == :ident || type == :number

          return false

        elsif type == :keyword && value == "this"

        elsif type == :this

          return false

        elsif type == :operator && (value == ")" || value == "]")

          return false
@@ -217,31 +220,31 @@ module JsDuck 
    }x



    KEYWORDS = {

      "break" => true,

      "case" => true,

      "catch" => true,

      "continue" => true,

      "default" => true,

      "delete" => true,

      "do" => true,

      "else" => true,

      "finally" => true,

      "for" => true,

      "function" => true,

      "if" => true,

      "in" => true,

      "instanceof" => true,

      "new" => true,

      "return" => true,

      "switch" => true,

      "this" => true,

      "throw" => true,

      "try" => true,

      "typeof" => true,

      "var" => true,

      "void" => true,

      "while" => true,

      "with" => true,

      "break" => :break,

      "case" => :case,

      "catch" => :catch,

      "continue" => :continue,

      "default" => :default,

      "delete" => :delete,

      "do" => :do,

      "else" => :else,

      "finally" => :finally,

      "for" => :for,

      "function" => :function,

      "if" => :if,

      "in" => :in,

      "instanceof" => :instanceof,

      "new" => :new,

      "return" => :return,

      "switch" => :switch,

      "this" => :this,

      "throw" => :throw,

      "try" => :try,

      "typeof" => :typeof,

      "var" => :var,

      "void" => :void,

      "while" => :while,

      "with" => :with,

    }

  end

spec/aggregator_methods_and_events_spec.rb

+10 −0
Original line number Diff line number Diff line
@@ -151,6 +151,16 @@ describe JsDuck::Aggregator do 
    end

  end



  describe "doc-comment followed by 'function'" do

    before do

      @doc = parse("/** Some function */ 'function';")[0]

    end



    it "isn't detected as method" do

      @doc[:tagname].should_not == :method

    end

  end



  describe "explicit @method without @param-s" do

    before do

      @doc = parse(<<-EOS)[0]

spec/lexer_spec.rb

+4 −4
Original line number Diff line number Diff line
@@ -17,7 +17,7 @@ describe JsDuck::Lexer do 


  it "tokenizes simple expression" do

    lex("var foo = 8;").should == [

      [:keyword, "var"],

      [:var, :var],

      [:ident, "foo"],

      [:operator, "="],

      [:number, "8"],
@@ -49,7 +49,7 @@ describe JsDuck::Lexer do 


    it "when regex after return" do

      lex("return /foo/.test;").should == [

        [:keyword, "return"],

        [:return, :return],

        [:regex, "/foo/"],

        [:operator, "."],

        [:ident, "test"],
@@ -59,7 +59,7 @@ describe JsDuck::Lexer do 


    it "when regex after typeof" do

      lex("typeof /foo/;").should == [

        [:keyword, "typeof"],

        [:typeof, :typeof],

        [:regex, "/foo/"],

        [:operator, ";"]

      ]
@@ -67,7 +67,7 @@ describe JsDuck::Lexer do 


    it "when division after this" do

      lex("this / 3").should == [

        [:keyword, "this"],

        [:this, :this],

        [:operator, "/"],

        [:number, "3"]

      ]