got constants working, as well as phpdoc symlink setup script written

This commit is contained in:
John Bintz 2008-10-12 20:16:49 -04:00
parent 22c596b1f9
commit 350337d3b9
15 changed files with 276 additions and 38 deletions

View File

@ -1,4 +1,5 @@
-js ../htdocs/harmoniouscode.js -js ../htdocs/harmoniouscode.js
-main JavaScriptTarget -main JavaScriptTarget
-resource ../data/functions_tokens_cache.hxd -resource ../data/functions_tokens_cache.hxd
-cp ../src -resource ../data/constant_tokens_cache.hxd
-cp ../src

View File

@ -1,6 +1,6 @@
#!/bin/bash #!/bin/bash
./setup.sh ./setup.sh
haxe -cp ../src -neko ../neko/regenerate.n -main RegenerateDataFiles && neko ../neko/regenerate.n ./regenerate_data_files.sh
rm ../htdocs/harmoniouscode.js rm ../htdocs/harmoniouscode.js
haxe javascript.hxml haxe javascript.hxml

28
build/phpdoc_setup.sh Executable file
View File

@ -0,0 +1,28 @@
#!/bin/bash
if [ -z $1 ]; then
echo "Need to specify a phpdoc directory"
exit 1
fi
if [ ! -e $1 ]; then
echo "Provided phpdoc directory $1 not found"
exit 1
fi
if [ ! -d $1 ]; then
echo "Provided phpdoc directory $1 is not a directory."
exit 1
fi
./setup.sh
cd ../data
rm phpdoc_*
ln -s $1/phpbook/phpbook-xsl/version.xml phpdoc_function_versions.xml
for constant_file in $(find $1/en/reference -name "constants.xml" -exec grep -L "no.constants" {} \;) ; do
echo $constant_file
constant_module=$(expr "$constant_file" : '.*/reference/\(.*\)/')
ln -s $constant_file "phpdoc_constants_${constant_module}.xml";
done

4
build/regenerate_data_files.sh Executable file
View File

@ -0,0 +1,4 @@
#!/bin/bash
haxe -cp ../src -main RegenerateDataFiles -neko ../neko/regenerate.n && neko ../neko/regenerate.n

View File

@ -1,5 +1,6 @@
#!/bin/bash #!/bin/bash
./setup.sh ./setup.sh
./regenerate_data_files.sh
haxe tests.hxml && neko ../neko/my_tests.n haxe tests.hxml && neko ../neko/my_tests.n

View File

@ -1,4 +1,5 @@
-main MyTests -main MyTests
-neko ../neko/my_tests.n -neko ../neko/my_tests.n
-resource ../data/functions_tokens_cache.hxd -resource ../data/functions_tokens_cache.hxd
-cp ../src -resource ../data/constant_tokens_cache.hxd
-cp ../src

View File

@ -15,22 +15,35 @@ class CodeParser {
/** /**
Load all possible token processors from disk. Load all possible token processors from disk.
**/ **/
public function loadProcessorsFromDisk() { public function load_processors_from_disk() {
var functionProcessor = new FunctionTokenProcessor(); var function_processor = new FunctionTokenProcessor();
if (!functionProcessor.load_from_cache()) { if (!function_processor.load_from_cache()) {
functionProcessor.populate_from_file(); function_processor.populate_from_file();
functionProcessor.save_to_cache(); function_processor.save_to_cache();
} }
this.token_processors.set(Type.getClassName(Type.getClass(functionProcessor)), functionProcessor); this.token_processors.set(Type.getClassName(Type.getClass(function_processor)), function_processor);
var constant_processor = new ConstantTokenProcessor();
if (!constant_processor.load_from_cache()) {
constant_processor.populate_from_files();
constant_processor.save_to_cache();
}
this.token_processors.set(Type.getClassName(Type.getClass(constant_processor)), constant_processor);
} }
#end #end
public function loadProcessorsFromResources() { public function load_processors_from_resources() {
var functionProcessor = new FunctionTokenProcessor(); var function_processor = new FunctionTokenProcessor();
functionProcessor.load_from_resource(); function_processor.load_from_resource();
this.token_processors.set(Type.getClassName(Type.getClass(functionProcessor)), functionProcessor); this.token_processors.set(Type.getClassName(Type.getClass(function_processor)), function_processor);
var constant_processor = new ConstantTokenProcessor();
constant_processor.load_from_resource();
this.token_processors.set(Type.getClassName(Type.getClass(constant_processor)), constant_processor);
} }
public function get_token_processors() { return this.token_processors; } public function get_token_processors() { return this.token_processors; }
@ -51,8 +64,9 @@ class CodeParser {
this.ignored_modules = new Hash<Bool>(); this.ignored_modules = new Hash<Bool>();
var function_token_processor = this.token_processors.get("FunctionTokenProcessor"); var function_token_processor = this.token_processors.get("FunctionTokenProcessor");
var constant_token_processor = this.token_processors.get("ConstantTokenProcessor");
var function_tokens_found = new Hash<Bool>(); var tokens_found = new Hash<Bool>();
var tokens_to_ignore = new Array<Hash<Bool>>(); var tokens_to_ignore = new Array<Hash<Bool>>();
var flattened_tokens = new Hash<Bool>(); var flattened_tokens = new Hash<Bool>();
@ -103,15 +117,19 @@ class CodeParser {
is_capturing = false; is_capturing = false;
if (!flattened_tokens.exists(token)) { if (!tokens_found.exists(token)) {
if (is_function) { if (!flattened_tokens.exists(token)) {
if (!function_tokens_found.exists(token)) { if (is_function) {
if (function_token_processor.tokenHash.exists(token)) { if (function_token_processor.tokenHash.exists(token)) {
results.push(function_token_processor.tokenHash.get(token).toResult()); results.push(function_token_processor.tokenHash.get(token).toResult());
} }
function_tokens_found.set(token, true); } else {
if (constant_token_processor.tokenHash.exists(token)) {
results.push(constant_token_processor.tokenHash.get(token).toResult());
}
} }
} }
tokens_found.set(token, true);
} }
} else { } else {
if (current == "/") { if (current == "/") {
@ -151,6 +169,14 @@ class CodeParser {
index++; index++;
} }
if (is_capturing) {
var token = s.substr(capture_index, index - capture_index);
if (constant_token_processor.tokenHash.exists(token)) {
results.push(constant_token_processor.tokenHash.get(token).toResult());
}
}
results.sort(Result.compare); results.sort(Result.compare);
return results; return results;

View File

@ -1,25 +1,169 @@
class ConstantTokenProcessor extends TokenProcessor { class ConstantTokenProcessor extends TokenProcessor {
public static var cachePath : String = "constant_tokens_cache.hxd"; public static var cachePath : String = "../data/constant_tokens_cache.hxd";
override public function get_cache_path() { return ConstantTokenProcessor.cachePath; } override public function get_cache_path() { return ConstantTokenProcessor.cachePath; }
public static var source_path : String = "../data";
public static var source_file_pattern : EReg = ~/phpdoc_constants_.*\.xml/;
public static var version_match = ~/since php ([0-9\.]+)/i;
public static var node_skip_information = [
[ "para", "variablelist" ],
[ "section", "para" ],
[ "section", "table" ],
[ "para", "table" ],
[ "para", "informaltable" ],
[ "para", "itemizedlist" ],
[ "section", "variablelist" ]
];
#if neko #if neko
public function populate_from_file(path : String) { public function populate_from_files() {
this.populate_from_string(neko.io.File.getContent(path)); this.tokenHash = new Hash<Token>();
for (file in neko.FileSystem.readDirectory(source_path)) {
if (source_file_pattern.match(file)) {
trace(file + ": " + this.append_from_string(neko.io.File.getContent(source_path + "/" + file)));
}
}
}
public function append_from_string(s : String) : String {
var type = "none";
for (child in Xml.parse(s).firstElement()) {
if (child.nodeType == Xml.Element) {
var any_skipped;
do {
any_skipped = false;
for (nodes_to_skip in node_skip_information) {
if (child.nodeName == nodes_to_skip[0]) {
for (subchild in child) {
if (subchild.nodeType == Xml.Element) {
if (subchild.nodeName == nodes_to_skip[1]) {
child = subchild;
any_skipped = true;
break;
}
}
}
}
}
} while (any_skipped == true);
// itemizedlist
if (child.nodeName == "itemizedlist") {
type = "itemizedlist";
var fast_child = new haxe.xml.Fast(child);
if (fast_child.hasNode.listitem) {
for (item in fast_child.nodes.listitem) {
if (item.hasNode.simpara) {
var token_name : String = null;
var token_version : String = "4";
for (simpara in item.nodes.simpara) {
if (simpara.hasNode.constant) {
try {
token_name = simpara.node.constant.innerData;
} catch (e : Dynamic) {}
}
try {
var description_string = simpara.innerHTML;
if (version_match.match(description_string)) {
token_version = ~/\.$/.replace(version_match.matched(1), "");
}
} catch (e : Dynamic) {}
}
if (token_name != null) {
this.tokenHash.set(token_name, new ConstantToken(token_name, "PHP " + token_version));
}
}
}
}
}
// variablelist
if (child.nodeName == "variablelist") {
type = "variablelist";
for (variable in child) {
if (variable.nodeType == Xml.Element) {
var token_name : String = null;
var token_version : String = "4";
var fast_variable = new haxe.xml.Fast(variable);
if (fast_variable.hasNode.term) {
var term_variable = fast_variable.node.term;
if (term_variable.hasNode.constant) {
try {
token_name = term_variable.node.constant.innerData;
} catch (e : Dynamic) {}
}
}
if (fast_variable.hasNode.listitem) {
var listitem_variable = fast_variable.node.listitem;
if (listitem_variable.hasNode.simpara) {
try {
var description_string = listitem_variable.node.simpara.innerData;
if (version_match.match(description_string)) {
token_version = ~/\.$/.replace(version_match.matched(1), "");
}
} catch (e : Dynamic) {}
}
}
if (token_name != null) {
this.tokenHash.set(token_name, new ConstantToken(token_name, "PHP " + token_version));
}
}
}
}
// table
if ((child.nodeName == "table") || (child.nodeName == "informaltable")) {
type = "table";
var node_drilldown = [ "tgroup" ];
for (node_name in node_drilldown) {
for (node in child) {
if (node.nodeType == Xml.Element) {
if (node.nodeName == node_name) {
child = node; break;
}
}
}
}
var fast_child = new haxe.xml.Fast(child);
if (fast_child.hasNode.tbody) {
if (fast_child.node.tbody.hasNode.row) {
for (row in fast_child.node.tbody.nodes.row) {
var token_name : String = null;
var token_version : String = "4";
if (row.hasNode.entry) {
for (entry in row.nodes.entry) {
if (entry.hasNode.constant) {
try {
token_name = entry.node.constant.innerData;
} catch (e : Dynamic) {}
} else {
try {
var description_string = entry.innerHTML;
if (version_match.match(description_string)) {
token_version = ~/\.$/.replace(version_match.matched(1), "");
}
} catch (e : Dynamic) {}
}
}
}
if (token_name != null) {
this.tokenHash.set(token_name, new ConstantToken(token_name, "PHP " + token_version));
}
}
}
}
}
}
}
return type;
} }
public function populate_from_string(s : String) { public function populate_from_string(s : String) {
this.tokenHash = new Hash<Token>(); this.tokenHash = new Hash<Token>();
for (child in Xml.parse(s).firstElement()) { this.append_from_string(s);
if (child.nodeType == Xml.Element) {
if (child.nodeName == "function") {
var version = child.get("from");
version = ~/PECL /.replace(version, "");
version = ~/\:/.replace(version, " ");
var token = child.get("name");
this.tokenHash.set(token, new FunctionToken(child.get("name"), version));
}
}
}
} }
#end #end
} }

View File

@ -1,7 +1,7 @@
class FunctionTokenProcessor extends TokenProcessor { class FunctionTokenProcessor extends TokenProcessor {
public static var cachePath : String = "../data/functions_tokens_cache.hxd"; public static var cachePath : String = "../data/functions_tokens_cache.hxd";
override public function get_cache_path() { return FunctionTokenProcessor.cachePath; } override public function get_cache_path() { return FunctionTokenProcessor.cachePath; }
public static var sourcePath : String = "../data/versions.xml"; public static var sourcePath : String = "../data/phpdoc_function_versions.xml";
#if neko #if neko
public function populate_from_file() { public function populate_from_file() {

View File

@ -10,9 +10,10 @@ class JavaScriptTarget {
static public function main() { static public function main() {
var function_token = new FunctionToken("a","a"); var function_token = new FunctionToken("a","a");
var constant_token = new ConstantToken("a","a");
code_parser = new CodeParser(); code_parser = new CodeParser();
code_parser.loadProcessorsFromResources(); code_parser.load_processors_from_resources();
show_only_modules = new Hash<Bool>(); show_only_modules = new Hash<Bool>();
ignored_modules = new Hash<Bool>(); ignored_modules = new Hash<Bool>();

View File

@ -5,6 +5,7 @@ class MyTests {
r.add(new TestFunctionToken()); r.add(new TestFunctionToken());
r.add(new TestFunctionTokenProcessor()); r.add(new TestFunctionTokenProcessor());
r.add(new TestConstantToken()); r.add(new TestConstantToken());
r.add(new TestConstantTokenProcessor());
r.add(new TestCodeParser()); r.add(new TestCodeParser());
r.add(new TestCodeVersionInformation()); r.add(new TestCodeVersionInformation());
r.add(new TestResult()); r.add(new TestResult());

View File

@ -10,6 +10,8 @@ class RegenerateDataFiles {
var constantProcessor = new ConstantTokenProcessor(); var constantProcessor = new ConstantTokenProcessor();
if (!constantProcessor.load_from_cache()) { if (!constantProcessor.load_from_cache()) {
neko.Lib.print("Regenerating constants cache...\n"); neko.Lib.print("Regenerating constants cache...\n");
constantProcessor.populate_from_files();
constantProcessor.save_to_cache();
} }
} }
} }

View File

@ -7,19 +7,21 @@ class TestCodeParser extends haxe.unit.TestCase {
[ "//harmonious json_encode\narray_shift() json_encode()\n//harmonious_end", "1", "{minimum => {PHP => 4}, maximum => {}}" ], [ "//harmonious json_encode\narray_shift() json_encode()\n//harmonious_end", "1", "{minimum => {PHP => 4}, maximum => {}}" ],
[ "//harmonious json_encode\narray_shift() json_encode()\n//harmonious_end\njson_encode()", "2", "{minimum => {PHP => 5.2.0, json => 1.2.0}, maximum => {}}" ], [ "//harmonious json_encode\narray_shift() json_encode()\n//harmonious_end\njson_encode()", "2", "{minimum => {PHP => 5.2.0, json => 1.2.0}, maximum => {}}" ],
[ "//harmonious @json\narray_shift() json_encode()\n//harmonious_end\njson_encode()", "2", "{minimum => {PHP => 5.2.0}, maximum => {}}" ], [ "//harmonious @json\narray_shift() json_encode()\n//harmonious_end\njson_encode()", "2", "{minimum => {PHP => 5.2.0}, maximum => {}}" ],
[ "//harmonious @PHP\narray_shift()", "1", "{minimum => {PHP => 4}, maximum => {}}" ] [ "//harmonious @PHP\narray_shift()", "1", "{minimum => {PHP => 4}, maximum => {}}" ],
[ "PATHINFO_BASENAME", "1", "{minimum => {PHP => 4}, maximum => {}}" ]
]; ];
#if neko #if neko
function testCodeParserLoadTokens() { function testCodeParserLoadTokens() {
var p = new CodeParser(); var p = new CodeParser();
p.loadProcessorsFromDisk(); p.load_processors_from_disk();
assertTrue(p.token_processors.exists("FunctionTokenProcessor")); assertTrue(p.token_processors.exists("FunctionTokenProcessor"));
assertTrue(p.token_processors.exists("ConstantTokenProcessor"));
} }
function testProcessCode() { function testProcessCode() {
var p = new CodeParser(); var p = new CodeParser();
p.loadProcessorsFromDisk(); p.load_processors_from_disk();
for (code in test_code) { for (code in test_code) {
var result = p.parse(code[0]); var result = p.parse(code[0]);

View File

@ -0,0 +1,27 @@
class TestConstantTokenProcessor extends haxe.unit.TestCase {
static var constant_name : String = "TEST";
static var constant_from : String = "5.2";
static var test_xml_strings = [
"<appendix> <variablelist> <varlistentry> <term> <constant>$constant</constant> </term> <listitem> <simpara>Since PHP $version</simpara> </listitem> </varlistentry> </variablelist> </appendix>",
"<appendix> <para> <variablelist> <varlistentry> <term> <constant>$constant</constant> </term> <listitem> <simpara>Since PHP $version</simpara> </listitem> </varlistentry> </variablelist> </para> </appendix>",
"<appendix> <section> <variablelist> <varlistentry> <term> <constant>$constant</constant> </term> <listitem> <simpara>Since PHP $version</simpara> </listitem> </varlistentry> </variablelist> </section> </appendix>",
"<appendix> <table> <tbody> <row> <entry> <constant>$constant</constant> </entry> <entry> since PHP $version</entry> </row> </tbody> </table> </appendix>",
"<appendix> <section> <para> <table> <tgroup> <tbody> <row> <entry> <constant>$constant</constant> </entry> <entry>since PHP $version</entry> </row> </tbody> </tgroup> </table> </para> </section> </appendix>",
"<appendix> <para> <table> <tgroup> <tbody> <row> <entry> <constant>$constant</constant> </entry> <entry>since PHP $version</entry> </row> </tbody> </tgroup> </table> </para> </appendix>",
"<appendix> <para> <informaltable> <tgroup> <tbody> <row> <entry> <constant>$constant</constant> </entry> <entry>since PHP $version</entry> </row> </tbody> </tgroup> </informaltable> </para> </appendix>",
"<appendix> <para> <itemizedlist> <listitem> <simpara> <constant>$constant</constant> </simpara> <simpara>since PHP $version</simpara> </listitem> </itemizedlist> </para> </appendix>"
];
public function testConstantLists() {
for (string in test_xml_strings) {
string = ~/\$constant/.replace(string, constant_name);
string = ~/\$version/.replace(string, constant_from);
var tokenProcessor = new ConstantTokenProcessor();
tokenProcessor.populate_from_string(string);
assertTrue(tokenProcessor.tokenHash.exists(constant_name));
assertEquals("PHP " + constant_from, tokenProcessor.tokenHash.get(constant_name).version);
}
}
}

View File

@ -5,7 +5,7 @@ class TestFunctionTokenProcessor extends haxe.unit.TestCase {
var tokenProcessor : FunctionTokenProcessor; var tokenProcessor : FunctionTokenProcessor;
public override function setup() { public override function setup() {
testXml = "<versions><function name='" + functionName + "' from='" + functionFrom + "'/></versions>"; testXml = "<versions> <function name='" + functionName + "' from='" + functionFrom + "'/> </versions>";
tokenProcessor = new FunctionTokenProcessor(); tokenProcessor = new FunctionTokenProcessor();
tokenProcessor.populate_from_string(testXml); tokenProcessor.populate_from_string(testXml);
} }