diff --git a/build/javascript.hxml b/build/javascript.hxml index 817b075..9881998 100755 --- a/build/javascript.hxml +++ b/build/javascript.hxml @@ -1,4 +1,5 @@ -js ../htdocs/harmoniouscode.js -main JavaScriptTarget -resource ../data/functions_tokens_cache.hxd --cp ../src \ No newline at end of file +-resource ../data/constant_tokens_cache.hxd +-cp ../src diff --git a/build/javascript.sh b/build/javascript.sh index abd6ca2..e9d794a 100755 --- a/build/javascript.sh +++ b/build/javascript.sh @@ -1,6 +1,6 @@ #!/bin/bash ./setup.sh -haxe -cp ../src -neko ../neko/regenerate.n -main RegenerateDataFiles && neko ../neko/regenerate.n +./regenerate_data_files.sh rm ../htdocs/harmoniouscode.js haxe javascript.hxml diff --git a/build/phpdoc_setup.sh b/build/phpdoc_setup.sh new file mode 100755 index 0000000..beaf0f1 --- /dev/null +++ b/build/phpdoc_setup.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +if [ -z $1 ]; then + echo "Need to specify a phpdoc directory" + exit 1 +fi + +if [ ! -e $1 ]; then + echo "Provided phpdoc directory $1 not found" + exit 1 +fi + +if [ ! -d $1 ]; then + echo "Provided phpdoc directory $1 is not a directory." + exit 1 +fi + +./setup.sh +cd ../data +rm phpdoc_* + +ln -s $1/phpbook/phpbook-xsl/version.xml phpdoc_function_versions.xml + +for constant_file in $(find $1/en/reference -name "constants.xml" -exec grep -L "no.constants" {} \;) ; do + echo $constant_file + constant_module=$(expr "$constant_file" : '.*/reference/\(.*\)/') + ln -s $constant_file "phpdoc_constants_${constant_module}.xml"; +done \ No newline at end of file diff --git a/build/regenerate_data_files.sh b/build/regenerate_data_files.sh new file mode 100755 index 0000000..a67f873 --- /dev/null +++ b/build/regenerate_data_files.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +haxe -cp ../src -main RegenerateDataFiles -neko ../neko/regenerate.n && neko ../neko/regenerate.n + diff --git a/build/test.sh b/build/test.sh index 0f4f308..0944533 100755 --- a/build/test.sh +++ b/build/test.sh @@ -1,5 +1,6 @@ #!/bin/bash ./setup.sh +./regenerate_data_files.sh haxe tests.hxml && neko ../neko/my_tests.n diff --git a/build/tests.hxml b/build/tests.hxml index b3a69c4..f9b7c18 100644 --- a/build/tests.hxml +++ b/build/tests.hxml @@ -1,4 +1,5 @@ -main MyTests -neko ../neko/my_tests.n -resource ../data/functions_tokens_cache.hxd --cp ../src \ No newline at end of file +-resource ../data/constant_tokens_cache.hxd +-cp ../src diff --git a/src/CodeParser.hx b/src/CodeParser.hx index 8a8cdbf..482f9ae 100644 --- a/src/CodeParser.hx +++ b/src/CodeParser.hx @@ -15,22 +15,35 @@ class CodeParser { /** Load all possible token processors from disk. **/ - public function loadProcessorsFromDisk() { - var functionProcessor = new FunctionTokenProcessor(); - if (!functionProcessor.load_from_cache()) { - functionProcessor.populate_from_file(); - functionProcessor.save_to_cache(); + public function load_processors_from_disk() { + var function_processor = new FunctionTokenProcessor(); + if (!function_processor.load_from_cache()) { + function_processor.populate_from_file(); + function_processor.save_to_cache(); } - this.token_processors.set(Type.getClassName(Type.getClass(functionProcessor)), functionProcessor); + this.token_processors.set(Type.getClassName(Type.getClass(function_processor)), function_processor); + + var constant_processor = new ConstantTokenProcessor(); + if (!constant_processor.load_from_cache()) { + constant_processor.populate_from_files(); + constant_processor.save_to_cache(); + } + + this.token_processors.set(Type.getClassName(Type.getClass(constant_processor)), constant_processor); } #end - public function loadProcessorsFromResources() { - var functionProcessor = new FunctionTokenProcessor(); - functionProcessor.load_from_resource(); + public function load_processors_from_resources() { + var function_processor = new FunctionTokenProcessor(); + function_processor.load_from_resource(); - this.token_processors.set(Type.getClassName(Type.getClass(functionProcessor)), functionProcessor); + this.token_processors.set(Type.getClassName(Type.getClass(function_processor)), function_processor); + + var constant_processor = new ConstantTokenProcessor(); + constant_processor.load_from_resource(); + + this.token_processors.set(Type.getClassName(Type.getClass(constant_processor)), constant_processor); } public function get_token_processors() { return this.token_processors; } @@ -51,8 +64,9 @@ class CodeParser { this.ignored_modules = new Hash(); var function_token_processor = this.token_processors.get("FunctionTokenProcessor"); + var constant_token_processor = this.token_processors.get("ConstantTokenProcessor"); - var function_tokens_found = new Hash(); + var tokens_found = new Hash(); var tokens_to_ignore = new Array>(); var flattened_tokens = new Hash(); @@ -103,15 +117,19 @@ class CodeParser { is_capturing = false; - if (!flattened_tokens.exists(token)) { - if (is_function) { - if (!function_tokens_found.exists(token)) { + if (!tokens_found.exists(token)) { + if (!flattened_tokens.exists(token)) { + if (is_function) { if (function_token_processor.tokenHash.exists(token)) { results.push(function_token_processor.tokenHash.get(token).toResult()); } - function_tokens_found.set(token, true); + } else { + if (constant_token_processor.tokenHash.exists(token)) { + results.push(constant_token_processor.tokenHash.get(token).toResult()); + } } } + tokens_found.set(token, true); } } else { if (current == "/") { @@ -151,6 +169,14 @@ class CodeParser { index++; } + if (is_capturing) { + var token = s.substr(capture_index, index - capture_index); + + if (constant_token_processor.tokenHash.exists(token)) { + results.push(constant_token_processor.tokenHash.get(token).toResult()); + } + } + results.sort(Result.compare); return results; diff --git a/src/ConstantTokenProcessor.hx b/src/ConstantTokenProcessor.hx index 57d2453..fb455ff 100644 --- a/src/ConstantTokenProcessor.hx +++ b/src/ConstantTokenProcessor.hx @@ -1,25 +1,169 @@ class ConstantTokenProcessor extends TokenProcessor { - public static var cachePath : String = "constant_tokens_cache.hxd"; + public static var cachePath : String = "../data/constant_tokens_cache.hxd"; override public function get_cache_path() { return ConstantTokenProcessor.cachePath; } + public static var source_path : String = "../data"; + public static var source_file_pattern : EReg = ~/phpdoc_constants_.*\.xml/; + public static var version_match = ~/since php ([0-9\.]+)/i; + public static var node_skip_information = [ + [ "para", "variablelist" ], + [ "section", "para" ], + [ "section", "table" ], + [ "para", "table" ], + [ "para", "informaltable" ], + [ "para", "itemizedlist" ], + [ "section", "variablelist" ] + ]; + #if neko - public function populate_from_file(path : String) { - this.populate_from_string(neko.io.File.getContent(path)); + public function populate_from_files() { + this.tokenHash = new Hash(); + for (file in neko.FileSystem.readDirectory(source_path)) { + if (source_file_pattern.match(file)) { + trace(file + ": " + this.append_from_string(neko.io.File.getContent(source_path + "/" + file))); + } + } + } + + public function append_from_string(s : String) : String { + var type = "none"; + for (child in Xml.parse(s).firstElement()) { + if (child.nodeType == Xml.Element) { + var any_skipped; + do { + any_skipped = false; + for (nodes_to_skip in node_skip_information) { + if (child.nodeName == nodes_to_skip[0]) { + for (subchild in child) { + if (subchild.nodeType == Xml.Element) { + if (subchild.nodeName == nodes_to_skip[1]) { + child = subchild; + any_skipped = true; + break; + } + } + } + } + } + } while (any_skipped == true); + + // itemizedlist + if (child.nodeName == "itemizedlist") { + type = "itemizedlist"; + var fast_child = new haxe.xml.Fast(child); + if (fast_child.hasNode.listitem) { + for (item in fast_child.nodes.listitem) { + if (item.hasNode.simpara) { + var token_name : String = null; + var token_version : String = "4"; + for (simpara in item.nodes.simpara) { + if (simpara.hasNode.constant) { + try { + token_name = simpara.node.constant.innerData; + } catch (e : Dynamic) {} + } + try { + var description_string = simpara.innerHTML; + if (version_match.match(description_string)) { + token_version = ~/\.$/.replace(version_match.matched(1), ""); + } + } catch (e : Dynamic) {} + } + if (token_name != null) { + this.tokenHash.set(token_name, new ConstantToken(token_name, "PHP " + token_version)); + } + } + } + } + } + + // variablelist + if (child.nodeName == "variablelist") { + type = "variablelist"; + for (variable in child) { + if (variable.nodeType == Xml.Element) { + var token_name : String = null; + var token_version : String = "4"; + var fast_variable = new haxe.xml.Fast(variable); + if (fast_variable.hasNode.term) { + var term_variable = fast_variable.node.term; + if (term_variable.hasNode.constant) { + try { + token_name = term_variable.node.constant.innerData; + } catch (e : Dynamic) {} + } + } + if (fast_variable.hasNode.listitem) { + var listitem_variable = fast_variable.node.listitem; + if (listitem_variable.hasNode.simpara) { + try { + var description_string = listitem_variable.node.simpara.innerData; + if (version_match.match(description_string)) { + token_version = ~/\.$/.replace(version_match.matched(1), ""); + } + } catch (e : Dynamic) {} + } + } + + if (token_name != null) { + this.tokenHash.set(token_name, new ConstantToken(token_name, "PHP " + token_version)); + } + } + } + } + + // table + if ((child.nodeName == "table") || (child.nodeName == "informaltable")) { + type = "table"; + + var node_drilldown = [ "tgroup" ]; + for (node_name in node_drilldown) { + for (node in child) { + if (node.nodeType == Xml.Element) { + if (node.nodeName == node_name) { + child = node; break; + } + } + } + } + + var fast_child = new haxe.xml.Fast(child); + if (fast_child.hasNode.tbody) { + if (fast_child.node.tbody.hasNode.row) { + for (row in fast_child.node.tbody.nodes.row) { + var token_name : String = null; + var token_version : String = "4"; + if (row.hasNode.entry) { + for (entry in row.nodes.entry) { + if (entry.hasNode.constant) { + try { + token_name = entry.node.constant.innerData; + } catch (e : Dynamic) {} + } else { + try { + var description_string = entry.innerHTML; + if (version_match.match(description_string)) { + token_version = ~/\.$/.replace(version_match.matched(1), ""); + } + } catch (e : Dynamic) {} + } + } + } + if (token_name != null) { + this.tokenHash.set(token_name, new ConstantToken(token_name, "PHP " + token_version)); + } + } + } + } + } + } + } + return type; } public function populate_from_string(s : String) { this.tokenHash = new Hash(); - for (child in Xml.parse(s).firstElement()) { - if (child.nodeType == Xml.Element) { - if (child.nodeName == "function") { - var version = child.get("from"); - version = ~/PECL /.replace(version, ""); - version = ~/\:/.replace(version, " "); - var token = child.get("name"); - this.tokenHash.set(token, new FunctionToken(child.get("name"), version)); - } - } - } + this.append_from_string(s); } #end } \ No newline at end of file diff --git a/src/FunctionTokenProcessor.hx b/src/FunctionTokenProcessor.hx index 37c288f..bd9f046 100644 --- a/src/FunctionTokenProcessor.hx +++ b/src/FunctionTokenProcessor.hx @@ -1,7 +1,7 @@ class FunctionTokenProcessor extends TokenProcessor { public static var cachePath : String = "../data/functions_tokens_cache.hxd"; override public function get_cache_path() { return FunctionTokenProcessor.cachePath; } - public static var sourcePath : String = "../data/versions.xml"; + public static var sourcePath : String = "../data/phpdoc_function_versions.xml"; #if neko public function populate_from_file() { diff --git a/src/JavaScriptTarget.hx b/src/JavaScriptTarget.hx index 52d6cf2..285f229 100644 --- a/src/JavaScriptTarget.hx +++ b/src/JavaScriptTarget.hx @@ -10,9 +10,10 @@ class JavaScriptTarget { static public function main() { var function_token = new FunctionToken("a","a"); + var constant_token = new ConstantToken("a","a"); code_parser = new CodeParser(); - code_parser.loadProcessorsFromResources(); + code_parser.load_processors_from_resources(); show_only_modules = new Hash(); ignored_modules = new Hash(); diff --git a/src/MyTests.hx b/src/MyTests.hx index cf229f7..16ab0c4 100644 --- a/src/MyTests.hx +++ b/src/MyTests.hx @@ -5,6 +5,7 @@ class MyTests { r.add(new TestFunctionToken()); r.add(new TestFunctionTokenProcessor()); r.add(new TestConstantToken()); + r.add(new TestConstantTokenProcessor()); r.add(new TestCodeParser()); r.add(new TestCodeVersionInformation()); r.add(new TestResult()); diff --git a/src/RegenerateDataFiles.hx b/src/RegenerateDataFiles.hx index 92262c3..51e97cc 100644 --- a/src/RegenerateDataFiles.hx +++ b/src/RegenerateDataFiles.hx @@ -10,6 +10,8 @@ class RegenerateDataFiles { var constantProcessor = new ConstantTokenProcessor(); if (!constantProcessor.load_from_cache()) { neko.Lib.print("Regenerating constants cache...\n"); + constantProcessor.populate_from_files(); + constantProcessor.save_to_cache(); } } } \ No newline at end of file diff --git a/src/TestCodeParser.hx b/src/TestCodeParser.hx index 2c44420..3d9c768 100644 --- a/src/TestCodeParser.hx +++ b/src/TestCodeParser.hx @@ -7,19 +7,21 @@ class TestCodeParser extends haxe.unit.TestCase { [ "//harmonious json_encode\narray_shift() json_encode()\n//harmonious_end", "1", "{minimum => {PHP => 4}, maximum => {}}" ], [ "//harmonious json_encode\narray_shift() json_encode()\n//harmonious_end\njson_encode()", "2", "{minimum => {PHP => 5.2.0, json => 1.2.0}, maximum => {}}" ], [ "//harmonious @json\narray_shift() json_encode()\n//harmonious_end\njson_encode()", "2", "{minimum => {PHP => 5.2.0}, maximum => {}}" ], - [ "//harmonious @PHP\narray_shift()", "1", "{minimum => {PHP => 4}, maximum => {}}" ] + [ "//harmonious @PHP\narray_shift()", "1", "{minimum => {PHP => 4}, maximum => {}}" ], + [ "PATHINFO_BASENAME", "1", "{minimum => {PHP => 4}, maximum => {}}" ] ]; #if neko function testCodeParserLoadTokens() { var p = new CodeParser(); - p.loadProcessorsFromDisk(); + p.load_processors_from_disk(); assertTrue(p.token_processors.exists("FunctionTokenProcessor")); + assertTrue(p.token_processors.exists("ConstantTokenProcessor")); } function testProcessCode() { var p = new CodeParser(); - p.loadProcessorsFromDisk(); + p.load_processors_from_disk(); for (code in test_code) { var result = p.parse(code[0]); diff --git a/src/TestConstantTokenProcessor.hx b/src/TestConstantTokenProcessor.hx new file mode 100644 index 0000000..7ddc8f6 --- /dev/null +++ b/src/TestConstantTokenProcessor.hx @@ -0,0 +1,27 @@ +class TestConstantTokenProcessor extends haxe.unit.TestCase { + static var constant_name : String = "TEST"; + static var constant_from : String = "5.2"; + static var test_xml_strings = [ + " $constant Since PHP $version ", + " $constant Since PHP $version ", + "
$constant Since PHP $version
", + " $constant since PHP $version
", + "
$constant since PHP $version
", + " $constant since PHP $version
", + " $constant since PHP $version ", + " $constant since PHP $version " + ]; + + public function testConstantLists() { + for (string in test_xml_strings) { + string = ~/\$constant/.replace(string, constant_name); + string = ~/\$version/.replace(string, constant_from); + + var tokenProcessor = new ConstantTokenProcessor(); + tokenProcessor.populate_from_string(string); + + assertTrue(tokenProcessor.tokenHash.exists(constant_name)); + assertEquals("PHP " + constant_from, tokenProcessor.tokenHash.get(constant_name).version); + } + } +} \ No newline at end of file diff --git a/src/TestFunctionTokenProcessor.hx b/src/TestFunctionTokenProcessor.hx index ffd08ff..e1405ae 100644 --- a/src/TestFunctionTokenProcessor.hx +++ b/src/TestFunctionTokenProcessor.hx @@ -5,7 +5,7 @@ class TestFunctionTokenProcessor extends haxe.unit.TestCase { var tokenProcessor : FunctionTokenProcessor; public override function setup() { - testXml = ""; + testXml = " "; tokenProcessor = new FunctionTokenProcessor(); tokenProcessor.populate_from_string(testXml); }