Commit f5d00d78 authored by Rene Saarsoo's avatar Rene Saarsoo
Browse files

Fix handling of Null and Boolean tokens.

parent 7463b8ef

lib/jsduck/esprima_tokenizer.rb

+2 −2
Original line number Diff line number Diff line
@@ -52,8 +52,8 @@ module JsDuck 
      when "Numeric"

        {:type => :number, :value => tok["value"]}

      when "String"

        {:type => :string, :value => tok["value"]}

      when "Identifier"

        {:type => :string, :value => tok["value"].gsub(/\A['"]|['"]\Z/m, "")}

      when "Identifier", "Boolean", "Null"

        {:type => :ident, :value => tok["value"]}

      when "RegularExpression"

        {:type => :regex, :value => tok["value"]}

spec/esprima_tokenizer_spec.rb

+14 −2
Original line number Diff line number Diff line
@@ -2,8 +2,12 @@ require "jsduck/esprima_tokenizer" 


describe JsDuck::EsprimaTokenizer do



  def raw_tokenize(js)

    JsDuck::EsprimaTokenizer.instance.tokenize(js)

  end



  def tokenize(js)

    JsDuck::EsprimaTokenizer.instance.tokenize(js).map {|t| t[:type] }

    raw_tokenize(js).map {|t| t[:type] }

  end



  describe "tokenize" do
@@ -44,11 +48,19 @@ describe JsDuck::EsprimaTokenizer do 
    end



    it "augments :doc_comment token with line number" do

      tokens = JsDuck::EsprimaTokenizer.instance.tokenize("\n \n /**Com1*/ \n /**Com2\n*/ /**Com3*/")

      tokens = raw_tokenize("\n \n /**Com1*/ \n /**Com2\n*/ /**Com3*/")

      tokens[0][:linenr].should == 3

      tokens[1][:linenr].should == 4

      tokens[2][:linenr].should == 5

    end



    it "converts boolean token to ident" do

      raw_tokenize("true")[0].should == {:type => :ident, :value => "true"}

    end



    it "converts null token to ident" do

      raw_tokenize("null")[0].should == {:type => :ident, :value => "null"}

    end

  end



end