Merge branch 'cache'

require 'jsduck/parser'

require 'jsduck/source/file'

require 'jsduck/logger'

require 'jsduck/cache'

module JsDuck

  class BatchParser

    def self.parse(opts)

      Util::Parallel.map(opts.input_files) do |fname|

      cache = Cache.create(opts)

      results = Util::Parallel.map(opts.input_files) do |fname|

        Logger.log("Parsing", fname)

        begin

          source = Util::IO.read(fname)

          docs = nil

          unless docs = cache.read(fname, source)

            docs = Parser.new.parse(source, fname, opts)

          Source::File.new(source, docs, fname)

            cache.write(fname, source, docs)

          end

          {

            :file => Source::File.new(source, docs, fname),

            :cache => cache.previous_entry,

          }

        rescue

          Logger.fatal_backtrace("Error while parsing #{fname}", $!)

          exit(1)

        end

      end

      cache.cleanup( results.map {|r| r[:cache] }.compact )

      return results.map {|r| r[:file] }

    end

  end

require 'digest/md5'

require 'fileutils'

require 'jsduck/util/null_object'

require 'set'

module JsDuck

  # Reads/writes parsed files in cache.

  #

  # When writing to cache:

  #

  # - makes MD5 hash of <file name> + <file contents>

  # - Dumps the the parsed data structure using Marshal into <md5>.dat

  #

  # When reading from cache:

  #

  # - makes MD5 hash of <file name> + <file contents>

  # - Reads the parsed data structure using Marshal from <md5>.dat

  #

  # Additionally a manifest.txt file is saved into the cache

  # directory, the contents of which is a string like the following:

  #

  #     Ruby: 1.9.3, JSDuck: 5.2.0

  #

  # This file is consulted before all other cache operations.  When

  # the version numbers in there don't match with current Ruby and

  # JSDuck versions, the whole cache gets invalidated - all cached

  # files get deleted.  This is to avoid problems with the Marshal

  # file format changes between Ruby versions and parsed data

  # structure changes between JSDuck versions.

  #

  # After all files have been checked into cache, the files that

  # weren't touched get deleted (using the #cleanup method).  This

  # ensures that the number of files in cache only grows when more

  # files are added to the documentation.

  #

  class Cache

    # Factory method to produce a cache object.  When caching is

    # disabled, returns a NullObject which emulates a cache that's

    # always empty.

    def self.create(opts)

      # Check also for cache_dir, which will be nil when output_dir is :stdout

      if opts.cache && opts.cache_dir

        Cache.new(opts)

      else

        Util::NullObject.new(

          :read => nil,

          :write => nil,

          :previous_entry => nil,

          :cleanup => nil

          )

      end

    end

    # The name of the cache file that was previously read or written.

    # When the #read call failed to find the file, it will be nil.

    # But it will always be available after the #write call.

    attr_reader :previous_entry

    def initialize(opts)

      @jsduck_version = opts.version

      @cache_dir = opts.cache_dir

      @manifest_file = @cache_dir + "/manifest.txt"

      @previous_entry = nil

      FileUtils.mkdir_p(@cache_dir) unless File.exists?(@cache_dir)

      # Invalidate the whole cache when it was generated with a

      # different Ruby and/or JSDuck version.

      invalidate_all! unless valid_manifest?

    end

    # Given the name and contents of a source file, reads the already

    # parsed data structure from cache.  Returns nil when not found.

    def read(file_name, file_contents)

      fname = cache_file_name(file_name, file_contents)

      if File.exists?(fname)

        @previous_entry = fname

        File.open(fname, "rb") {|file| Marshal::load(file) }

      else

        @previous_entry = nil

        nil

      end

    end

    # Writes parse data into cache under a name generated from the

    # name and contents of a source file.

    def write(file_name, file_contents, data)

      fname = cache_file_name(file_name, file_contents)

      @previous_entry = fname

      File.open(fname, "wb") {|file| Marshal::dump(data, file) }

    end

    # Given listing of used cache files (those that were either read

    # or written during this jsduck run) removes rest of the files

    # from cache directory that were unused.

    def cleanup(used_cache_entries)

      used = Set.new(used_cache_entries)

      Dir[@cache_dir + "/*.dat"].each do |file|

        FileUtils.rm_rf(file) unless used.include?(file)

      end

    end

    private

    def cache_file_name(file_name, file_contents)

      @cache_dir + "/" + md5(file_name + file_contents) + ".dat"

    end

    def md5(string)

      Digest::MD5.hexdigest(string)

    end

    def valid_manifest?

      manifest = File.exists?(@manifest_file) ? Util::IO.read(@manifest_file) : ""

      return manifest == current_manifest

    end

    def invalidate_all!

      FileUtils.rm_rf(@cache_dir)

      FileUtils.mkdir(@cache_dir)

      save_manifest

    end

    def save_manifest

      File.open(@manifest_file, "w") {|f| f.write(current_manifest) }

    end

    def current_manifest

      "Ruby: #{RUBY_VERSION}, JSDuck: #{@jsduck_version}\n"

    end

  end

end

    end

    def write_dir(dir, extension)

      FileUtils.mkdir(dir)

      FileUtils.mkdir(dir) unless File.exists?(dir)

      Util::Parallel.each(@relations.classes) do |cls|

        filename = dir + "/" + cls[:name] + extension

        Logger.log("Writing docs", filename)

require 'jsduck/format/batch'

require 'jsduck/class_writer'

require 'jsduck/guide_writer'

require 'jsduck/output_dir'

require 'fileutils'

module JsDuck

    # -- util routines --

    def clean_output_dir

      FileUtils.rm_rf(@opts.output_dir)

      OutputDir.clean(@opts)

    end

    def format_classes

    attr_accessor :ignore_global

    attr_accessor :external_classes

    attr_accessor :ext4_events

    attr_accessor :version

    # Customizing output

    attr_accessor :title

    # Debugging

    attr_accessor :warnings_exit_nonzero

    attr_accessor :cache

    attr_accessor :cache_dir

    attr_accessor :template_dir

    attr_accessor :template_links

    attr_accessor :extjs_path

      # Debugging

      @warnings_exit_nonzero = false

      @cache = false

      @cache_dir = nil

      @root_dir = File.dirname(File.dirname(File.dirname(__FILE__)))

      @template_dir = @root_dir + "/template-min"

      @template_links = false

          "This option is REQUIRED.  When the directory exists,",

          "it will be overwritten.  Give dash '-' as argument",

          "to write docs to STDOUT (works only with --export).") do |path|

          @output_dir = path == "-" ? :stdout : canonical(path)

          if path == "-"

            @output_dir = :stdout

          else

            @output_dir = canonical(path)

            @cache_dir = @output_dir + "/.cache" unless @cache_dir

          end

        end

        opts.on('--export=full/examples',

          Util::Parallel.in_processes = count.to_i

        end

        opts.on('--[no-]cache',

          "Turns parser cache on/off (EXPERIMENTAL).",

          "",

          "Defaults to off.",

          "",

          "When enabled, the results of parsing source files is saved",

          "inside the JSDuck output directory. Next time JSDuck runs,",

          "only the files that have changed are parsed again, others",

          "are read from the cache.",

          "",

          "Note that switching between Ruby and/or JSDuck versions",

          "invalidates the whole cache.  But changes in custom tags",

          "don't invalidate the cache, so avoid caching when developing",

          "your custom tags.",

          "",

          "To change the cache directory location, use --cache-dir.") do |enabled|

          @cache = enabled

        end

        opts.on('--cache-dir=PATH',

          "Directory where to cache the parsed source.",

          "",

          "Defaults to: <output-dir>/.cache",

          "",

          "Each project needs to have a separate cache directory.",

          "Instead of writing the cache into the output directory,",

          "one might consider keeping it together with the source",

          "files.",

          "",

          "Note that JSDuck ensures that the <output-dir>/.cache",

          "dir is preserved when the rest of the <output-dir> gets",

          "wiped clean during the docs generation.  If you specify",

          "cache dir like <output-dir>/.mycache, then this will also",

          "be cleaned up during docs generation, and the caching",

          "won't work.",

          "",

          "This option only has an effect when --cache is also used.",) do |path|

          @cache_dir = path

        end

        opts.on('--pretty-json',

          "Turns on pretty-printing of JSON.",

          "",