Commit 875ff58c authored by Rene Saarsoo's avatar Rene Saarsoo
Browse files

Using "parallel" gem to parse files in parallel. 

To make this performance-boost work, some restructuring was required:

- Parser now includes DocParser, returning the doc-blocks
  in fully parsed form.

- Aggregator no more includes Parser.  Instead of parse() method
  it now has aggregate(), which expects output of Parser as input.
  Because of this some changes to unit tests were also needed.

- App performs the parallel parsing of source code and then
  feeds the results of parsing sequencially to Aggregator.

- Timing now reports three measures: parsing, aggregating, generating.
  But because of the parallel-phase, the report of parsing isn't
  that accurate as it also includes the generation of code files.
parent 4744d76d

lib/jsduck/aggregator.rb

+7 −10
Original line number Diff line number Diff line 
require 'jsduck/parser'

require 'jsduck/doc_parser'

require 'jsduck/merger'



module JsDuck
@@ -12,22 +10,21 @@ module JsDuck 
      @classes = {}

      @orphans = []

      @current_class = nil

      @doc_parser = DocParser.new

      @merger = Merger.new

    end



    # Parses chunk of JavaScript.  The resulting documentation is

    # accumulated inside this class and can be later accessed through

    # #result method.

    # Combines chunk of parsed JavaScript together with previously

    # added chunks.  The resulting documentation is accumulated inside

    # this class and can be later accessed through #result method.

    #

    # - input  the JavaScript source

    # - input  parse result from JsDuck::Parser

    # - filename  name of the JS file where it came from

    # - html_filename  name of the HTML file where the source was saved.

    #

    def parse(input, filename="", html_filename="")

    def aggregate(input, filename="", html_filename="")

      @current_class = nil

      Parser.new(input).parse.each do |docset|

        doc = @doc_parser.parse(docset[:comment])

      input.each do |docset|

        doc = docset[:comment]

        code = docset[:code]

        href = html_filename + "#line-" + docset[:linenr].to_s

        register(add_href(@merger.merge(doc, code), href, filename))

lib/jsduck/app.rb

+24 −10
Original line number Diff line number Diff line 
require 'rubygems'

require 'jsduck/parser'

require 'jsduck/aggregator'

require 'jsduck/source_formatter'

require 'jsduck/class'
@@ -9,6 +10,7 @@ require 'jsduck/page' 
require 'jsduck/timer'

require 'json'

require 'fileutils'

require 'parallel'



module JsDuck
@@ -31,24 +33,36 @@ module JsDuck 
    # Call this after input parameters set

    def run

      copy_template(@template_dir, @output_dir)

      result = parse_files(@input_files)

      classes = @timer.time(:parsing) { filter_classes(result) }



      parsed_files = @timer.time(:parsing) { parallel_parse(@input_files) }

      result = @timer.time(:aggregating) { aggregate(parsed_files) }

      classes = @timer.time(:aggregating) { filter_classes(result) }

      @timer.time(:generating) { write_tree(@output_dir+"/output/tree.js", classes) }

      @timer.time(:generating) { write_pages(@output_dir+"/output", classes) }



      @timer.report if @verbose

    end



    # Given array of filenames, parses all files and returns array of

    # documented items in all of those files.

    def parse_files(filenames)

      agr = Aggregator.new

    # Parses the files in parallel using as many processes as available CPU-s

    def parallel_parse(filenames)

      src = SourceFormatter.new(@output_dir + "/source")

      filenames.each do |fname|

      Parallel.map(filenames) do |fname|

        puts "Parsing #{fname} ..." if @verbose

        code = @timer.time(:parsing) { IO.read(fname) }

        src_fname = @timer.time(:generating) { src.write(code, fname) }

        @timer.time(:parsing) { agr.parse(code, File.basename(fname), File.basename(src_fname)) }

        code = IO.read(fname)

        {

          :name => fname,

          :src_name => src.write(code, fname),

          :data => Parser.new(code).parse,

        }

      end

    end



    # Aggregates parsing results sequencially

    def aggregate(parsed_files)

      agr = Aggregator.new

      parsed_files.each do |file|

        puts "Aggregating #{file[:name]} ..." if @verbose

        agr.aggregate(file[:data], File.basename(file[:name]), File.basename(file[:src_name]))

      end

      agr.result

    end

lib/jsduck/parser.rb

+10 −5
Original line number Diff line number Diff line 
require 'jsduck/lexer'

require 'jsduck/doc_parser'



module JsDuck



  class Parser

    def initialize(input)

      @lex = Lexer.new(input)

      @doc_parser = DocParser.new

      @docs = []

    end



    # Parses the whole JavaScript block and returns array where for

    # each doc-comment there is a hash of three values: the comment

    # itself as string, number of the line where the comment starts,

    # and parsed structure of the code that immediately follows the

    # comment.

    # structure created by DocParser, number of the line where the

    # comment starts, and parsed structure of the code that

    # immediately follows the comment.

    #

    # For example with the following JavaScript input:

    #
@@ -26,7 +28,10 @@ module JsDuck 
    #

    # [

    #   {

    #     :comment => "/**\n * @param {String} foo\n */",

    #     :comment => [

    #       {:tagname => :default, :doc => "Method description"},

    #       {:tagname => :return, :type => "Number", :doc => ""},

    #     ],

    #     :linenr => 1,

    #     :code => {

    #       :type => :assignment,
@@ -48,7 +53,7 @@ module JsDuck 
        if look(:doc_comment)

          comment = @lex.next(true)

          @docs << {

            :comment => comment[:value],

            :comment => @doc_parser.parse(comment[:value]),

            :linenr => comment[:linenr],

            :code => code_block

          }

spec/aggregator_cfgs_and_properties_spec.rb

+2 −1
Original line number Diff line number Diff line 
require "jsduck/aggregator"

require "jsduck/parser"



describe JsDuck::Aggregator do



  def parse(string)

    agr = JsDuck::Aggregator.new

    agr.parse(string)

    agr.aggregate(JsDuck::Parser.new(string).parse)

    agr.result

  end

spec/aggregator_classes_spec.rb

+2 −1
Original line number Diff line number Diff line 
require "jsduck/aggregator"

require "jsduck/parser"



describe JsDuck::Aggregator do



  def parse(string)

    agr = JsDuck::Aggregator.new

    agr.parse(string)

    agr.aggregate(JsDuck::Parser.new(string).parse)

    agr.result

  end