got constants working, as well as phpdoc symlink setup script written

build/javascript.hxml

@@ -1,4 +1,5 @@
 -js ../htdocs/harmoniouscode.js
 -main JavaScriptTarget
 -resource ../data/functions_tokens_cache.hxd
+-resource ../data/constant_tokens_cache.hxd
 -cp ../src

build/javascript.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
 ./setup.sh
-haxe -cp ../src -neko ../neko/regenerate.n -main RegenerateDataFiles && neko ../neko/regenerate.n
+./regenerate_data_files.sh
 rm ../htdocs/harmoniouscode.js
 haxe javascript.hxml

build/phpdoc_setup.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+if [ -z $1 ]; then
+  echo "Need to specify a phpdoc directory"
+  exit 1
+fi
+
+if [ ! -e $1 ]; then
+  echo "Provided phpdoc directory $1 not found"
+  exit 1
+fi
+
+if [ ! -d $1 ]; then
+  echo "Provided phpdoc directory $1 is not a directory."
+  exit 1
+fi
+
+./setup.sh
+cd ../data
+rm phpdoc_*
+
+ln -s $1/phpbook/phpbook-xsl/version.xml phpdoc_function_versions.xml
+
+for constant_file in $(find $1/en/reference -name "constants.xml" -exec grep -L "no.constants" {} \;) ; do
+  echo $constant_file
+  constant_module=$(expr "$constant_file" : '.*/reference/\(.*\)/')
+  ln -s $constant_file "phpdoc_constants_${constant_module}.xml";
+done

build/regenerate_data_files.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+haxe -cp ../src -main RegenerateDataFiles -neko ../neko/regenerate.n && neko ../neko/regenerate.n
+

build/test.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 
 ./setup.sh
+./regenerate_data_files.sh
 haxe tests.hxml && neko ../neko/my_tests.n
 

build/tests.hxml

@@ -1,4 +1,5 @@
 -main MyTests
 -neko ../neko/my_tests.n
 -resource ../data/functions_tokens_cache.hxd
+-resource ../data/constant_tokens_cache.hxd
 -cp ../src

src/CodeParser.hx

@@ -15,22 +15,35 @@ class CodeParser {
     /**
       Load all possible token processors from disk.
     **/
-    public function loadProcessorsFromDisk() {
+    public function load_processors_from_disk() {
-      var functionProcessor = new FunctionTokenProcessor();
+      var function_processor = new FunctionTokenProcessor();
-      if (!functionProcessor.load_from_cache()) {
+      if (!function_processor.load_from_cache()) {
-        functionProcessor.populate_from_file();
+        function_processor.populate_from_file();
-        functionProcessor.save_to_cache();
+        function_processor.save_to_cache();
       }
 
-      this.token_processors.set(Type.getClassName(Type.getClass(functionProcessor)), functionProcessor);
+      this.token_processors.set(Type.getClassName(Type.getClass(function_processor)), function_processor);
+
+      var constant_processor = new ConstantTokenProcessor();
+      if (!constant_processor.load_from_cache()) {
+        constant_processor.populate_from_files();
+        constant_processor.save_to_cache();
+      }
+
+      this.token_processors.set(Type.getClassName(Type.getClass(constant_processor)), constant_processor);
     }
   #end
 
-  public function loadProcessorsFromResources() {
+  public function load_processors_from_resources() {
-    var functionProcessor = new FunctionTokenProcessor();
+    var function_processor = new FunctionTokenProcessor();
-    functionProcessor.load_from_resource();
+    function_processor.load_from_resource();
 
-    this.token_processors.set(Type.getClassName(Type.getClass(functionProcessor)), functionProcessor);
+    this.token_processors.set(Type.getClassName(Type.getClass(function_processor)), function_processor);
+
+    var constant_processor = new ConstantTokenProcessor();
+    constant_processor.load_from_resource();
+
+    this.token_processors.set(Type.getClassName(Type.getClass(constant_processor)), constant_processor);
   }
 
   public function get_token_processors() { return this.token_processors; }
@@ -51,8 +64,9 @@ class CodeParser {
     this.ignored_modules = new Hash<Bool>();
 
     var function_token_processor = this.token_processors.get("FunctionTokenProcessor");
+    var constant_token_processor = this.token_processors.get("ConstantTokenProcessor");
 
-    var function_tokens_found = new Hash<Bool>();
+    var tokens_found = new Hash<Bool>();
     var tokens_to_ignore = new Array<Hash<Bool>>();
     var flattened_tokens = new Hash<Bool>();
 
@@ -103,16 +117,20 @@ class CodeParser {
 
           is_capturing = false;
 
+          if (!tokens_found.exists(token)) {
             if (!flattened_tokens.exists(token)) {
               if (is_function) {
-              if (!function_tokens_found.exists(token)) {
                 if (function_token_processor.tokenHash.exists(token)) {
                   results.push(function_token_processor.tokenHash.get(token).toResult());
                 }
-                function_tokens_found.set(token, true);
+              } else {
+                if (constant_token_processor.tokenHash.exists(token)) {
+                  results.push(constant_token_processor.tokenHash.get(token).toResult());
                 }
               }
             }
+            tokens_found.set(token, true);
+          }
         } else {
           if (current == "/") {
             if (s.indexOf("//harmonious", index) == index) {
@@ -151,6 +169,14 @@ class CodeParser {
       index++;
     }
 
+    if (is_capturing) {
+      var token = s.substr(capture_index, index - capture_index);
+
+      if (constant_token_processor.tokenHash.exists(token)) {
+        results.push(constant_token_processor.tokenHash.get(token).toResult());
+      }
+    }
+
     results.sort(Result.compare);
 
     return results;

src/ConstantTokenProcessor.hx

@@ -1,25 +1,169 @@
 class ConstantTokenProcessor extends TokenProcessor {
-  public static var cachePath : String = "constant_tokens_cache.hxd";
+  public static var cachePath : String = "../data/constant_tokens_cache.hxd";
   override public function get_cache_path() { return ConstantTokenProcessor.cachePath; }
 
+  public static var source_path : String = "../data";
+  public static var source_file_pattern : EReg = ~/phpdoc_constants_.*\.xml/;
+  public static var version_match = ~/since php ([0-9\.]+)/i;
+  public static var node_skip_information = [
+    [ "para", "variablelist" ],
+    [ "section", "para" ],
+    [ "section", "table" ],
+    [ "para", "table" ],
+    [ "para", "informaltable" ],
+    [ "para", "itemizedlist" ],
+    [ "section", "variablelist" ]
+  ];
+
   #if neko
-    public function populate_from_file(path : String) {
+    public function populate_from_files() {
-      this.populate_from_string(neko.io.File.getContent(path));
+      this.tokenHash = new Hash<Token>();
+      for (file in neko.FileSystem.readDirectory(source_path)) {
+        if (source_file_pattern.match(file)) {
+          trace(file + ": " + this.append_from_string(neko.io.File.getContent(source_path + "/" + file)));
+        }
+      }
+    }
+
+    public function append_from_string(s : String) : String {
+      var type = "none";
+      for (child in Xml.parse(s).firstElement()) {
+        if (child.nodeType == Xml.Element) {
+          var any_skipped;
+          do {
+            any_skipped = false;
+            for (nodes_to_skip in node_skip_information) {
+              if (child.nodeName == nodes_to_skip[0]) {
+                for (subchild in child) {
+                  if (subchild.nodeType == Xml.Element) {
+                    if (subchild.nodeName == nodes_to_skip[1]) {
+                      child = subchild;
+                      any_skipped = true;
+                      break;
+                    }
+                  }
+                }
+              }
+            }
+          } while (any_skipped == true);
+
+          // itemizedlist
+          if (child.nodeName == "itemizedlist") {
+            type = "itemizedlist";
+            var fast_child = new haxe.xml.Fast(child);
+            if (fast_child.hasNode.listitem) {
+              for (item in fast_child.nodes.listitem) {
+                if (item.hasNode.simpara) {
+                  var token_name : String = null;
+                  var token_version : String = "4";
+                  for (simpara in item.nodes.simpara) {
+                    if (simpara.hasNode.constant) {
+                      try {
+                        token_name = simpara.node.constant.innerData;
+                      } catch (e : Dynamic) {}
+                    }
+                    try {
+                      var description_string = simpara.innerHTML;
+                      if (version_match.match(description_string)) {
+                        token_version = ~/\.$/.replace(version_match.matched(1), "");
+                      }
+                    } catch (e : Dynamic) {}
+                  }
+                  if (token_name != null) {
+                    this.tokenHash.set(token_name, new ConstantToken(token_name, "PHP " + token_version));
+                  }
+                }
+              }
+            }
+          }
+          
+          // variablelist
+          if (child.nodeName == "variablelist") {
+            type = "variablelist";
+            for (variable in child) {
+              if (variable.nodeType == Xml.Element) {
+                var token_name : String = null;
+                var token_version : String = "4";
+                var fast_variable = new haxe.xml.Fast(variable);
+                if (fast_variable.hasNode.term) {
+                  var term_variable = fast_variable.node.term;
+                  if (term_variable.hasNode.constant) {
+                    try {
+                      token_name = term_variable.node.constant.innerData;
+                    } catch (e : Dynamic) {}
+                  }
+                }
+                if (fast_variable.hasNode.listitem) {
+                  var listitem_variable = fast_variable.node.listitem;
+                  if (listitem_variable.hasNode.simpara) {
+                    try {
+                      var description_string = listitem_variable.node.simpara.innerData;
+                      if (version_match.match(description_string)) {
+                        token_version = ~/\.$/.replace(version_match.matched(1), "");
+                      }
+                    } catch (e : Dynamic) {}
+                  }
+                }
+
+                if (token_name != null) {
+                  this.tokenHash.set(token_name, new ConstantToken(token_name, "PHP " + token_version));
+                }
+              }
+            }
+          }
+
+          // table
+          if ((child.nodeName == "table") || (child.nodeName == "informaltable")) {
+            type = "table";
+
+            var node_drilldown = [ "tgroup" ];
+            for (node_name in node_drilldown) {
+              for (node in child) {
+                if (node.nodeType == Xml.Element) {
+                  if (node.nodeName == node_name) {
+                    child = node; break;
+                  }
+                }
+              }
+            }
+
+            var fast_child = new haxe.xml.Fast(child);
+            if (fast_child.hasNode.tbody) {
+              if (fast_child.node.tbody.hasNode.row) {
+                for (row in fast_child.node.tbody.nodes.row) {
+                  var token_name : String = null;
+                  var token_version : String = "4";
+                  if (row.hasNode.entry) {
+                    for (entry in row.nodes.entry) {
+                      if (entry.hasNode.constant) {
+                        try {
+                          token_name = entry.node.constant.innerData;
+                        } catch (e : Dynamic) {}
+                      } else {
+                        try {
+                          var description_string = entry.innerHTML;
+                          if (version_match.match(description_string)) {
+                            token_version = ~/\.$/.replace(version_match.matched(1), "");
+                          }
+                        } catch (e : Dynamic) {}
+                      }
+                    }
+                  }
+                  if (token_name != null) {
+                    this.tokenHash.set(token_name, new ConstantToken(token_name, "PHP " + token_version));
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+      return type;
     }
 
     public function populate_from_string(s : String) {
       this.tokenHash = new Hash<Token>();
-      for (child in Xml.parse(s).firstElement()) {
+      this.append_from_string(s);
-        if (child.nodeType == Xml.Element) {
-          if (child.nodeName == "function") {
-            var version = child.get("from");
-            version = ~/PECL /.replace(version, "");
-            version = ~/\:/.replace(version, " ");
-            var token = child.get("name");
-            this.tokenHash.set(token, new FunctionToken(child.get("name"), version));
-          }
-        }
-      }
     }
   #end
 }

src/FunctionTokenProcessor.hx

@@ -1,7 +1,7 @@
 class FunctionTokenProcessor extends TokenProcessor {
   public static var cachePath : String = "../data/functions_tokens_cache.hxd";
   override public function get_cache_path() { return FunctionTokenProcessor.cachePath; }
-  public static var sourcePath : String = "../data/versions.xml";
+  public static var sourcePath : String = "../data/phpdoc_function_versions.xml";
 
   #if neko
     public function populate_from_file() {

src/JavaScriptTarget.hx

@@ -10,9 +10,10 @@ class JavaScriptTarget {
 
   static public function main() {
     var function_token = new FunctionToken("a","a");
+    var constant_token = new ConstantToken("a","a");
 
     code_parser = new CodeParser();
-    code_parser.loadProcessorsFromResources();
+    code_parser.load_processors_from_resources();
 
     show_only_modules = new Hash<Bool>();
     ignored_modules = new Hash<Bool>();

src/MyTests.hx

@@ -5,6 +5,7 @@ class MyTests {
     r.add(new TestFunctionToken());
     r.add(new TestFunctionTokenProcessor());
     r.add(new TestConstantToken());
+    r.add(new TestConstantTokenProcessor());
     r.add(new TestCodeParser());
     r.add(new TestCodeVersionInformation());
     r.add(new TestResult());

src/RegenerateDataFiles.hx

@@ -10,6 +10,8 @@ class RegenerateDataFiles {
     var constantProcessor = new ConstantTokenProcessor();
     if (!constantProcessor.load_from_cache()) {
       neko.Lib.print("Regenerating constants cache...\n");
+      constantProcessor.populate_from_files();
+      constantProcessor.save_to_cache();
     }
   }
 }

src/TestCodeParser.hx

@@ -7,19 +7,21 @@ class TestCodeParser extends haxe.unit.TestCase {
     [ "//harmonious json_encode\narray_shift() json_encode()\n//harmonious_end", "1", "{minimum => {PHP => 4}, maximum => {}}" ],
     [ "//harmonious json_encode\narray_shift() json_encode()\n//harmonious_end\njson_encode()", "2", "{minimum => {PHP => 5.2.0, json => 1.2.0}, maximum => {}}" ],
     [ "//harmonious @json\narray_shift() json_encode()\n//harmonious_end\njson_encode()", "2", "{minimum => {PHP => 5.2.0}, maximum => {}}" ],
-    [ "//harmonious @PHP\narray_shift()", "1", "{minimum => {PHP => 4}, maximum => {}}" ]
+    [ "//harmonious @PHP\narray_shift()", "1", "{minimum => {PHP => 4}, maximum => {}}" ],
+    [ "PATHINFO_BASENAME", "1", "{minimum => {PHP => 4}, maximum => {}}" ]
   ];
 
   #if neko
     function testCodeParserLoadTokens() {
       var p = new CodeParser();
-      p.loadProcessorsFromDisk();
+      p.load_processors_from_disk();
       assertTrue(p.token_processors.exists("FunctionTokenProcessor"));
+      assertTrue(p.token_processors.exists("ConstantTokenProcessor"));
     }
 
     function testProcessCode() {
       var p = new CodeParser();
-      p.loadProcessorsFromDisk();
+      p.load_processors_from_disk();
 
       for (code in test_code) {
         var result = p.parse(code[0]);

src/TestConstantTokenProcessor.hx

@@ -0,0 +1,27 @@
+class TestConstantTokenProcessor extends haxe.unit.TestCase {
+  static var constant_name : String = "TEST";
+  static var constant_from : String = "5.2";
+  static var test_xml_strings = [
+    "<appendix> <variablelist> <varlistentry> <term> <constant>$constant</constant> </term> <listitem> <simpara>Since PHP $version</simpara> </listitem> </varlistentry> </variablelist> </appendix>",
+    "<appendix> <para> <variablelist> <varlistentry> <term> <constant>$constant</constant> </term> <listitem> <simpara>Since PHP $version</simpara> </listitem> </varlistentry> </variablelist> </para> </appendix>",
+    "<appendix> <section> <variablelist> <varlistentry> <term> <constant>$constant</constant> </term> <listitem> <simpara>Since PHP $version</simpara> </listitem> </varlistentry> </variablelist> </section> </appendix>",
+    "<appendix> <table> <tbody> <row> <entry> <constant>$constant</constant> </entry> <entry> since PHP $version</entry> </row> </tbody> </table> </appendix>",
+    "<appendix> <section> <para> <table> <tgroup> <tbody> <row> <entry> <constant>$constant</constant> </entry> <entry>since PHP $version</entry> </row> </tbody> </tgroup> </table> </para> </section> </appendix>",
+    "<appendix> <para> <table> <tgroup> <tbody> <row> <entry> <constant>$constant</constant> </entry> <entry>since PHP $version</entry> </row> </tbody> </tgroup> </table> </para> </appendix>",
+    "<appendix> <para> <informaltable> <tgroup> <tbody> <row> <entry> <constant>$constant</constant> </entry> <entry>since PHP $version</entry> </row> </tbody> </tgroup> </informaltable> </para> </appendix>",
+    "<appendix> <para> <itemizedlist> <listitem> <simpara> <constant>$constant</constant> </simpara> <simpara>since PHP $version</simpara> </listitem> </itemizedlist> </para> </appendix>"
+  ];
+
+  public function testConstantLists() {
+    for (string in test_xml_strings) {
+      string = ~/\$constant/.replace(string, constant_name);
+      string = ~/\$version/.replace(string, constant_from);
+
+      var tokenProcessor = new ConstantTokenProcessor();
+      tokenProcessor.populate_from_string(string);
+
+      assertTrue(tokenProcessor.tokenHash.exists(constant_name));
+      assertEquals("PHP " + constant_from, tokenProcessor.tokenHash.get(constant_name).version);
+    }
+  }
+}

src/TestFunctionTokenProcessor.hx

@@ -5,7 +5,7 @@ class TestFunctionTokenProcessor extends haxe.unit.TestCase {
   var tokenProcessor : FunctionTokenProcessor;
 
   public override function setup() {
-    testXml = "<versions><function name='" + functionName + "' from='" + functionFrom + "'/></versions>";
+    testXml = "<versions> <function name='" + functionName + "' from='" + functionFrom + "'/> </versions>";
     tokenProcessor = new FunctionTokenProcessor();
     tokenProcessor.populate_from_string(testXml);
   }