diff --git a/lib/jsduck/batch_parser.rb b/lib/jsduck/batch_parser.rb index 0eae6fe736db3874caa71e7b78aaa330046c7d41..085243c9674e80ba56717e25c0e3cdffe446a86d 100644 --- a/lib/jsduck/batch_parser.rb +++ b/lib/jsduck/batch_parser.rb @@ -3,6 +3,7 @@ require 'jsduck/util/io' require 'jsduck/parser' require 'jsduck/source/file' require 'jsduck/logger' +require 'jsduck/cache' module JsDuck @@ -11,17 +12,33 @@ module JsDuck class BatchParser def self.parse(opts) - Util::Parallel.map(opts.input_files) do |fname| + cache = Cache.create(opts) + + results = Util::Parallel.map(opts.input_files) do |fname| Logger.log("Parsing", fname) + begin source = Util::IO.read(fname) - docs = Parser.new.parse(source, fname, opts) - Source::File.new(source, docs, fname) + docs = nil + + unless docs = cache.read(fname, source) + docs = Parser.new.parse(source, fname, opts) + cache.write(fname, source, docs) + end + + { + :file => Source::File.new(source, docs, fname), + :cache => cache.previous_entry, + } rescue Logger.fatal_backtrace("Error while parsing #{fname}", $!) exit(1) end end + + cache.cleanup( results.map {|r| r[:cache] }.compact ) + + return results.map {|r| r[:file] } end end diff --git a/lib/jsduck/cache.rb b/lib/jsduck/cache.rb new file mode 100644 index 0000000000000000000000000000000000000000..91ac58df1c2eadee2165e5ccd2f641e1c9513d67 --- /dev/null +++ b/lib/jsduck/cache.rb @@ -0,0 +1,137 @@ +require 'digest/md5' +require 'fileutils' +require 'jsduck/util/null_object' +require 'set' + +module JsDuck + + # Reads/writes parsed files in cache. + # + # When writing to cache: + # + # - makes MD5 hash of + + # - Dumps the the parsed data structure using Marshal into .dat + # + # When reading from cache: + # + # - makes MD5 hash of + + # - Reads the parsed data structure using Marshal from .dat + # + # Additionally a manifest.txt file is saved into the cache + # directory, the contents of which is a string like the following: + # + # Ruby: 1.9.3, JSDuck: 5.2.0 + # + # This file is consulted before all other cache operations. When + # the version numbers in there don't match with current Ruby and + # JSDuck versions, the whole cache gets invalidated - all cached + # files get deleted. This is to avoid problems with the Marshal + # file format changes between Ruby versions and parsed data + # structure changes between JSDuck versions. + # + # After all files have been checked into cache, the files that + # weren't touched get deleted (using the #cleanup method). This + # ensures that the number of files in cache only grows when more + # files are added to the documentation. + # + class Cache + + # Factory method to produce a cache object. When caching is + # disabled, returns a NullObject which emulates a cache that's + # always empty. + def self.create(opts) + # Check also for cache_dir, which will be nil when output_dir is :stdout + if opts.cache && opts.cache_dir + Cache.new(opts) + else + Util::NullObject.new( + :read => nil, + :write => nil, + :previous_entry => nil, + :cleanup => nil + ) + end + end + + # The name of the cache file that was previously read or written. + # When the #read call failed to find the file, it will be nil. + # But it will always be available after the #write call. + attr_reader :previous_entry + + def initialize(opts) + @jsduck_version = opts.version + @cache_dir = opts.cache_dir + @manifest_file = @cache_dir + "/manifest.txt" + @previous_entry = nil + + FileUtils.mkdir_p(@cache_dir) unless File.exists?(@cache_dir) + + # Invalidate the whole cache when it was generated with a + # different Ruby and/or JSDuck version. + invalidate_all! unless valid_manifest? + end + + # Given the name and contents of a source file, reads the already + # parsed data structure from cache. Returns nil when not found. + def read(file_name, file_contents) + fname = cache_file_name(file_name, file_contents) + if File.exists?(fname) + @previous_entry = fname + File.open(fname, "rb") {|file| Marshal::load(file) } + else + @previous_entry = nil + nil + end + end + + # Writes parse data into cache under a name generated from the + # name and contents of a source file. + def write(file_name, file_contents, data) + fname = cache_file_name(file_name, file_contents) + @previous_entry = fname + File.open(fname, "wb") {|file| Marshal::dump(data, file) } + end + + # Given listing of used cache files (those that were either read + # or written during this jsduck run) removes rest of the files + # from cache directory that were unused. + def cleanup(used_cache_entries) + used = Set.new(used_cache_entries) + + Dir[@cache_dir + "/*.dat"].each do |file| + FileUtils.rm_rf(file) unless used.include?(file) + end + end + + private + + def cache_file_name(file_name, file_contents) + @cache_dir + "/" + md5(file_name + file_contents) + ".dat" + end + + def md5(string) + Digest::MD5.hexdigest(string) + end + + def valid_manifest? + manifest = File.exists?(@manifest_file) ? Util::IO.read(@manifest_file) : "" + return manifest == current_manifest + end + + def invalidate_all! + FileUtils.rm_rf(@cache_dir) + FileUtils.mkdir(@cache_dir) + save_manifest + end + + def save_manifest + File.open(@manifest_file, "w") {|f| f.write(current_manifest) } + end + + def current_manifest + "Ruby: #{RUBY_VERSION}, JSDuck: #{@jsduck_version}\n" + end + + end + +end diff --git a/lib/jsduck/class_writer.rb b/lib/jsduck/class_writer.rb index f8532c75c6558d5cdaad3a02ee18a4edd8ff9f29..1c0be7b21c0939cdd7976e185e31ef4383589b35 100644 --- a/lib/jsduck/class_writer.rb +++ b/lib/jsduck/class_writer.rb @@ -29,7 +29,8 @@ module JsDuck end def write_dir(dir, extension) - FileUtils.mkdir(dir) + FileUtils.mkdir(dir) unless File.exists?(dir) + Util::Parallel.each(@relations.classes) do |cls| filename = dir + "/" + cls[:name] + extension Logger.log("Writing docs", filename) diff --git a/lib/jsduck/export_writer.rb b/lib/jsduck/export_writer.rb index a310f0dffeb7070ded428164556aac50db2badc3..48cfb2dae2f5b3f774cbd0a9c181a27f0a3a9b2e 100644 --- a/lib/jsduck/export_writer.rb +++ b/lib/jsduck/export_writer.rb @@ -4,6 +4,7 @@ require 'jsduck/exporter/examples' require 'jsduck/format/batch' require 'jsduck/class_writer' require 'jsduck/guide_writer' +require 'jsduck/output_dir' require 'fileutils' module JsDuck @@ -50,7 +51,7 @@ module JsDuck # -- util routines -- def clean_output_dir - FileUtils.rm_rf(@opts.output_dir) + OutputDir.clean(@opts) end def format_classes diff --git a/lib/jsduck/options.rb b/lib/jsduck/options.rb index b7f95f38ad56b3ab0c6575889ef40b8cdf2d8495..716a746bd65f504574044de994877dc101f794d7 100644 --- a/lib/jsduck/options.rb +++ b/lib/jsduck/options.rb @@ -18,6 +18,7 @@ module JsDuck attr_accessor :ignore_global attr_accessor :external_classes attr_accessor :ext4_events + attr_accessor :version # Customizing output attr_accessor :title @@ -48,6 +49,8 @@ module JsDuck # Debugging attr_accessor :warnings_exit_nonzero + attr_accessor :cache + attr_accessor :cache_dir attr_accessor :template_dir attr_accessor :template_links attr_accessor :extjs_path @@ -132,6 +135,8 @@ module JsDuck # Debugging @warnings_exit_nonzero = false + @cache = false + @cache_dir = nil @root_dir = File.dirname(File.dirname(File.dirname(__FILE__))) @template_dir = @root_dir + "/template-min" @template_links = false @@ -203,7 +208,12 @@ module JsDuck "This option is REQUIRED. When the directory exists,", "it will be overwritten. Give dash '-' as argument", "to write docs to STDOUT (works only with --export).") do |path| - @output_dir = path == "-" ? :stdout : canonical(path) + if path == "-" + @output_dir = :stdout + else + @output_dir = canonical(path) + @cache_dir = @output_dir + "/.cache" unless @cache_dir + end end opts.on('--export=full/examples', @@ -740,6 +750,46 @@ module JsDuck Util::Parallel.in_processes = count.to_i end + opts.on('--[no-]cache', + "Turns parser cache on/off (EXPERIMENTAL).", + "", + "Defaults to off.", + "", + "When enabled, the results of parsing source files is saved", + "inside the JSDuck output directory. Next time JSDuck runs,", + "only the files that have changed are parsed again, others", + "are read from the cache.", + "", + "Note that switching between Ruby and/or JSDuck versions", + "invalidates the whole cache. But changes in custom tags", + "don't invalidate the cache, so avoid caching when developing", + "your custom tags.", + "", + "To change the cache directory location, use --cache-dir.") do |enabled| + @cache = enabled + end + + opts.on('--cache-dir=PATH', + "Directory where to cache the parsed source.", + "", + "Defaults to: /.cache", + "", + "Each project needs to have a separate cache directory.", + "Instead of writing the cache into the output directory,", + "one might consider keeping it together with the source", + "files.", + "", + "Note that JSDuck ensures that the /.cache", + "dir is preserved when the rest of the gets", + "wiped clean during the docs generation. If you specify", + "cache dir like /.mycache, then this will also", + "be cleaned up during docs generation, and the caching", + "won't work.", + "", + "This option only has an effect when --cache is also used.",) do |path| + @cache_dir = path + end + opts.on('--pretty-json', "Turns on pretty-printing of JSON.", "", diff --git a/lib/jsduck/output_dir.rb b/lib/jsduck/output_dir.rb new file mode 100644 index 0000000000000000000000000000000000000000..c32e9c2faa9875769e480000dd28a29c8bccb36c --- /dev/null +++ b/lib/jsduck/output_dir.rb @@ -0,0 +1,29 @@ +require 'fileutils' + +module JsDuck + # Cleans up the output dir from previous JSDuck run. If the output + # dir contains a .cache directory (and this dir is currently used + # for caching), it gets preserved, otherwise just an empty output + # dir is created. + class OutputDir + + # Initializes empty output directory (with optional .cache inside). + def self.clean(opts) + if opts.cache && cache_dir_needs_preserving(opts) + # Remove all files inside except .cache/ + Dir[opts.output_dir + "/*"].each do |file| + FileUtils.rm_rf(file) unless file =~ /\/.cache\z/ + end + else + # Remove and recreate the entire + FileUtils.rm_rf(opts.output_dir) + FileUtils.mkdir(opts.output_dir) + end + end + + def self.cache_dir_needs_preserving(opts) + opts.cache_dir == opts.output_dir + "/.cache" && File.exists?(opts.cache_dir) + end + + end +end diff --git a/lib/jsduck/web/template.rb b/lib/jsduck/web/template.rb index e4803a523db821c366f6b86108276f4494a0ca2f..9a636ea25b531bb81f8696e0a3b0d1bc6079257c 100644 --- a/lib/jsduck/web/template.rb +++ b/lib/jsduck/web/template.rb @@ -20,7 +20,6 @@ module JsDuck end def write - FileUtils.mkdir(@opts.output_dir) if @opts.template_links Logger.log("Linking template files to", @opts.output_dir) move_files(:symlink) diff --git a/lib/jsduck/web/writer.rb b/lib/jsduck/web/writer.rb index 8c96073361fbe353a925b3906796fa6a26c55a30..f15d9718b11b5443a4e9729548d57cfa6df05cbb 100644 --- a/lib/jsduck/web/writer.rb +++ b/lib/jsduck/web/writer.rb @@ -2,6 +2,7 @@ require 'jsduck/exporter/app' require 'jsduck/format/batch' require 'jsduck/class_writer' require 'jsduck/inline_examples' +require 'jsduck/output_dir' require 'jsduck/web/template' require 'jsduck/web/index_html' require 'jsduck/web/data' @@ -23,6 +24,8 @@ module JsDuck end def write + clean_output_dir + write_template_files write_member_icons @@ -42,9 +45,12 @@ module JsDuck @assets.write end - # Clean output dir and copy over template files + def clean_output_dir + OutputDir.clean(@opts) + end + + # Copy over template files def write_template_files - FileUtils.rm_rf(@opts.output_dir) Web::Template.new(@opts).write end