Merge improved CssParser into esprima-parser. (ff705101) · Commits · extjs / jsduck

lib/jsduck/css_lexer.rb

0 → 100644

+201 −0

Original line number	Diff line number	Diff line
		require 'strscan'

		module JsDuck

		# Tokenizes CSS or SCSS code into lexical tokens.
		#
		# Each token has a type and value.
		# Types and possible values for them are as follows:
		#
		# - :number -- "25.8"
		# - :percentage -- "25%"
		# - :dimension -- "2em"
		# - :string -- '"Hello world"'
		# - :ident -- "foo-bar"
		# - :at_keyword -- "@mixin"
		# - :hash -- "#00FF66"
		# - :delim -- "{"
		# - :doc_comment -- "/** My comment */"
		#
		# Notice that doc-comments are recognized as tokens while normal
		# comments are ignored just as whitespace.
		#
		class CssLexer
		# Initializes lexer with input string.
		def initialize(input)
		@input = StringScanner.new(input)
		@buffer = []
		end

		# Tests if given pattern matches the tokens that follow at current
		# position.
		#
		# Takes list of strings and symbols. Symbols are compared to
		# token type, while strings to token value. For example:
		#
		# look(:ident, ":", :dimension)
		#
		def look(*tokens)
		buffer_tokens(tokens.length)
		i = 0
		tokens.all? do \|t\|
		tok = @buffer[i]
		i += 1
		if !tok
		false
		elsif t.instance_of?(Symbol)
		tok[:type] == t
		else
		tok[:value] == t
		end
		end
		end

		# Returns the value of next token, moving the current token cursor
		# also to next token.
		#
		# When full=true, returns full token as hash like so:
		#
		# {:type => :ident, :value => "foo"}
		#
		# For doc-comments the full token also contains the field :linenr,
		# pointing to the line where the doc-comment began.
		#
		def next(full=false)
		buffer_tokens(1)
		tok = @buffer.shift
		# advance the scanpointer to the position after this token
		@input.pos = tok[:pos]
		full ? tok : tok[:value]
		end

		# True when no more tokens.
		def empty?
		buffer_tokens(1)
		return !@buffer.first
		end

		# Ensures next n tokens are read in buffer
		#
		# At the end of buffering the initial position scanpointer is
		# restored. Only the #next method will advance the scanpointer in
		# a way that's visible outside this class.
		def buffer_tokens(n)
		prev_pos = @input.pos
		@input.pos = @buffer.last[:pos] if @buffer.last
		(n - @buffer.length).times do
		@previous_token = tok = next_token
		if tok
		# remember scanpointer position after each token
		tok[:pos] = @input.pos
		@buffer << tok
		end
		end
		@input.pos = prev_pos
		end

		# Parses out next token from input stream.
		def next_token
		while !@input.eos? do
		skip_white
		if @input.check(IDENT)
		return {
		:type => :ident,
		:value => @input.scan(IDENT)
		}
		elsif @input.check(/'/)
		return {
		:type => :string,
		:value => @input.scan(/'([^'\\]\|\\.)*('\|\Z)/m)
		}
		elsif @input.check(/"/)
		return {
		:type => :string,
		:value => @input.scan(/"([^"\\]\|\\.)*("\|\Z)/m)
		}
		elsif @input.check(/\//)
		# Several things begin with dash:
		# - comments, regexes, division-operators
		if @input.check(/\/\\[^\/]/)
		return {
		:type => :doc_comment,
		# Calculate current line number, starting with 1
		:linenr => @input.string[0...@input.pos].count("\n") + 1,
		:value => @input.scan_until(/\\/\|\Z/).sub(/\A\/\\/, "").sub(/\\/\Z/, "")
		}
		elsif @input.check(/\/\*/)
		# skip multiline comment
		@input.scan_until(/\*\/\|\Z/)
		elsif @input.check(/\/\//)
		# skip line comment
		@input.scan_until(/\n\|\Z/)
		else
		return {
		:type => :operator,
		:value => @input.scan(/\//)
		}
		end
		elsif @input.check(NUM)
		nr = @input.scan(NUM)
		if @input.check(/%/)
		return {
		:type => :percentage,
		:value => nr + @input.scan(/%/)
		}
		elsif @input.check(IDENT)
		return {
		:type => :dimension,
		:value => nr + @input.scan(IDENT)
		}
		else
		return {
		:type => :number,
		:value => nr
		}
		end
		elsif @input.check(/@/)
		return maybe(:at_keyword, /@/, IDENT)
		elsif @input.check(/#/)
		return maybe(:hash, /#/, NAME)
		elsif @input.check(/\$/)
		return maybe(:var, /\$/, IDENT)
		elsif @input.check(/./)
		return {
		:type => :delim,
		:value => @input.scan(/./)
		}
		end
		end
		end

		# Returns token of given type when both regexes match.
		# Otherwise returns :delim token with value of first regex match.
		# First regex must always match.
		def maybe(token_type, before_re, after_re)
		before = @input.scan(before_re)
		if @input.check(after_re)
		return {
		:type => token_type,
		:value => before + @input.scan(after_re)
		}
		else
		return {
		:type => :delim,
		:value => before
		}
		end
		end

		def skip_white
		@input.scan(/\s+/)
		end

		# Simplified token syntax based on:
		# http://www.w3.org/TR/CSS21/syndata.html
		IDENT = /-?[_a-z][_a-z0-9-]*/i
		NAME = /[_a-z0-9-]+/i
		NUM = /[0-9]*\.[0-9]+\|[0-9]+/

		end

		end

lib/jsduck/css_parser.rb

+59 −15

Original line number	Diff line number	Diff line
		require 'jsduck/lexer'
		require 'jsduck/css_lexer'

		module JsDuck

		class CssParser
		def initialize(input, options = {})
		@lex = Lexer.new(input)
		@lex = CssLexer.new(input)
		@docs = []
		end

		@@ -27,31 +27,75 @@ module JsDuck
		@docs
		end

		# <code-block> := <mixin> \| <nop>
		# <code-block> := <mixin-declaration> \| <var-declaration> \| <nop>
		def code_block
		if look("@", "mixin")
		mixin
		if look("@mixin")
		mixin_declaration
		elsif look(:var, ":")
		var_declaration
		else
		{:tagname => :nop}
		end
		end

		# <mixin> := "@mixin" <css-ident>
		def mixin
		match("@", "mixin")
		# <mixin-declaration> := "@mixin" <ident>
		def mixin_declaration
		match("@mixin")
		return {
		:tagname => :css_mixin,
		:name => look(:ident) ? css_ident : nil,
		:name => look(:ident) ? match(:ident) : nil,
		}
		end

		# <css-ident> := <ident> [ "-" <ident> ]*
		def css_ident
		chain = [match(:ident)]
		while look("-", :ident) do
		chain << match("-", :ident)
		# <var-declaration> := <var> ":" <css-value>
		def var_declaration
		name = match(:var)
		match(":")
		value_list = css_value
		return {
		:tagname => :css_var,
		:name => name,
		:default => value_list.map {\|v\| v[:value] }.join(""),
		:type => value_type(value_list),
		}
		end

		# <css-value> := ...anything up to... [ ";" \| "}" \| "!default" ]
		def css_value
		val = []
		while !look(";") && !look("}") && !look("!", "default")
		val << @lex.next(true)
		end
		val
		end

		# Determines type of CSS value
		def value_type(val)
		case val[0][:type]
		when :number
		"number"
		when :dimension
		"measurement"
		when :percentage
		"percentage"
		when :string
		"string"
		when :hash
		"color"
		when :ident
		case val[0][:value]
		when "true", "false"
		return "boolean"
		when "rgb", "rgba", "hsl", "hsla"
		return "color"
		when "black", "silver", "gray", "white", "maroon",
		"red", "purple", "fuchsia", "green", "lime", "olive",
		"yellow", "navy", "blue", "teal", "aqua", "orange"
		return "color"
		when "transparent"
		return "color"
		end
		end
		return chain.join("-")
		end

		# Matches all arguments, returns the value of last match

lib/jsduck/doc_ast.rb

+1 −0

Original line number	Diff line number	Diff line
		@@ -113,6 +113,7 @@ module JsDuck
		:name => detect_name(:css_var, doc_map),
		:owner => detect_owner(doc_map),
		:type => detect_type(:css_var, doc_map),
		:default => detect_default(:css_var, doc_map),
		:doc => detect_doc(docs),
		}, doc_map)
		end

lib/jsduck/doc_parser.rb

+1 −1

Original line number	Diff line number	Diff line
		@@ -268,7 +268,7 @@ module JsDuck
		match(/@var/)
		add_tag(:css_var)
		maybe_type
		maybe_name
		maybe_name_with_default
		skip_white
		end

spec/aggregator_css_spec.rb

+122 −0

Original line number	Diff line number	Diff line
		@@ -52,6 +52,128 @@ describe JsDuck::Aggregator do
		end
		end

		describe "CSS @var with explicit default value" do
		before do
		@doc = parse(<<-EOCSS)[0]
		/**
		* @var {measurement} [$button-height=25px]
		*/
		EOCSS
		end

		it "detects default value" do
		@doc[:default].should == "25px"
		end
		end

		describe "CSS doc-comment followed with $var-name:" do
		before do
		@doc = parse(<<-EOCSS)[0]
		/**
		* Default height for buttons.
		*/
		$button-height: 25px;
		EOCSS
		end

		it "detects variable" do
		@doc[:tagname].should == :css_var
		end
		it "detects variable name" do
		@doc[:name].should == "$button-height"
		end
		it "detects variable type" do
		@doc[:type].should == "measurement"
		end
		it "detects variable default value" do
		@doc[:default].should == "25px"
		end
		end

		describe "$var-name: value followed by !default" do
		before do
		@doc = parse(<<-EOCSS)[0]
		/** */
		$button-height: 25px !default;
		EOCSS
		end

		it "detects variable" do
		@doc[:tagname].should == :css_var
		end
		it "detects variable type" do
		@doc[:type].should == "measurement"
		end
		it "detects variable default value" do
		@doc[:default].should == "25px"
		end
		end

		def detect_type(value)
		return parse(<<-EOCSS)[0][:type]
		/** */
		$foo: #{value};
		EOCSS
		end

		describe "auto-detection of CSS variable types" do
		it "detects integer" do
		detect_type("15").should == "number"
		end
		it "detects float" do
		detect_type("15.6").should == "number"
		end
		it "detects float begging with dot" do
		detect_type(".6").should == "number"
		end
		it "detects measurement" do
		detect_type("15em").should == "measurement"
		end
		it "detects percentage" do
		detect_type("99.9%").should == "percentage"
		end
		it "detects boolean true" do
		detect_type("true").should == "boolean"
		end
		it "detects boolean false" do
		detect_type("false").should == "boolean"
		end
		it "detects string" do
		detect_type('"Hello"').should == "string"
		end
		it "detects #000 color" do
		detect_type("#F0a").should == "color"
		end
		it "detects #000000 color" do
		detect_type("#FF00aa").should == "color"
		end
		it "detects rgb(...) color" do
		detect_type("rgb(255, 0, 0)").should == "color"
		end
		it "detects rgba(...) color" do
		detect_type("rgba(100%, 0%, 0%, 0.5)").should == "color"
		end
		it "detects hsl(...) color" do
		detect_type("hsl(255, 0, 0)").should == "color"
		end
		it "detects hsla(...) color" do
		detect_type("hsla(100%, 0%, 0%, 0.5)").should == "color"
		end

		# basic CSS color keywords
		"black silver gray white maroon red purple fuchsia green lime olive yellow navy blue teal aqua".split(/ /).each do \|c\|
		it "detects #{c} color keyword" do
		detect_type(c).should == "color"
		end
		end
		it "detects wide-supported orange color keyword" do
		detect_type("orange").should == "color"
		end
		it "detects transparent color keyword" do
		detect_type("transparent").should == "color"
		end
		end

		describe "CSS doc-comment followed by @mixin" do
		before do
		@doc = parse(<<-EOCSS)[0]