Remove all the old Esprima Lexer stuff.

# Script for benchmarking the lexer.

#

# Takes bunch of filenames as arguments and runs them all through lexer.

#

$:.unshift File.dirname(File.dirname(__FILE__)) + "/lib"

require 'jsduck/esprima_lexer'

ARGV.each do |fname|

  lex = JsDuck::EsprimaLexer.new(IO.read(fname))

  while !lex.empty?

    lex.next

  end

end

# Script for benchmarking the lexer.

#

# Takes bunch of filenames as arguments and runs them all through lexer.

#

$:.unshift File.dirname(File.dirname(__FILE__)) + "/lib"

require 'jsduck/esprima_tokenizer'

ARGV.each do |fname|

  JsDuck::EsprimaTokenizer.instance.tokenize(IO.read(fname))

end

require 'jsduck/esprima_tokenizer'

module JsDuck

  # New experimental lexer that uses Esprima.js through V8.

  class EsprimaLexer

    def initialize(input)

      @tokens = EsprimaTokenizer.instance.tokenize(input)

      @position = 0

    end

    def look(*tokens)

      i = @position

      tokens.all? do |t|

        tok = @tokens[i]

        i += 1

        if !tok

          false

        elsif t.instance_of?(Symbol)

          tok[:type] == t

        else

          tok[:value] == t

        end

      end

    end

    def next(full=false)

      tok = @tokens[@position]

      @position += 1

      full ? tok : tok[:value]

    end

    def empty?

      !@tokens[@position]

    end

  end

end

require 'v8'

require 'json'

require 'singleton'

module JsDuck

  # Uses Esprima.js engine through V8 to tokenize JavaScript string.

  class EsprimaTokenizer

    include Singleton

    def initialize

      @v8 = V8::Context.new

      esprima = File.dirname(File.dirname(File.dirname(File.expand_path(__FILE__))))+"/esprima/esprima.js";

      @v8.load(esprima)

      wrapper = File.dirname((File.expand_path(__FILE__)))+"/esprima_wrapper.js";

      @v8.load(wrapper)

    end

    # Input must be a String.

    def tokenize(input)

      @v8['js'] = @input = input

      out = JSON.parse(@v8.eval("EsprimaWrapper.parse(js)"))

      len = out["type"].length

      out_type = out["type"]

      out_value = out["value"]

      out_linenr = out["linenr"]

      type_array = [

        :number,

        :string,

        :ident,

        :regex,

        :operator,

        :keyword,

        :doc_comment,

      ]

      value_array = out["valueArray"]

      tokens = []

      for i in (0..(len-1))

        t = type_array[out_type[i]]

        if t == :doc_comment

          tokens << { :type => t, :value => out_value[i], :linenr => out_linenr[i] }

        elsif t == :keyword

          kw = value_array[out_value[i]].to_sym

          tokens << { :type => kw, :value => kw }

        else

          tokens << { :type => t, :value => value_array[out_value[i]] }

        end

      end

      tokens

    end

  end

end

require "jsduck/esprima_lexer"

describe JsDuck::EsprimaLexer do

  def lexer(input)

    JsDuck::EsprimaLexer.new(input)

  end

  describe "empty?" do

    it "is true when no tokens" do

      lexer("").empty?.should == true

    end

    it "is false when there are tokens" do

      lexer(";").empty?.should == false

    end

  end

  describe "next()" do

    it "gives value of next token" do

      lexer("var x;").next.should == :var

    end

    it "gives value of the n-th token when called n times" do

      lex = lexer("var x;")

      lex.next

      lex.next

      lex.next.should == ";"

    end

  end

  describe "next(true)" do

    it "gives full next token" do

      lexer(";").next(true).should == {:type => :operator, :value => ";"}

    end

  end

  describe "look()" do

    it "is true when all params match" do

      lexer("var x = 10;").look(:var, "x", "=").should == true

    end

    it "is false when at least one param doesn't match" do

      lexer("var x = 10;").look(:var, "y", "=").should == false

    end

    it "is false when not enough tokens" do

      lexer(";").look(";", :var).should == false

    end

  end

end