start working on reducing the size of serialized data

src/CodeParser.hx

@@ -6,6 +6,8 @@ class CodeParser {
   public var token_processors(get_token_processors, null) : Hash<TokenProcessor>;
   public var ignored_modules(get_ignored_modules, null) : Hash<Bool>;
 
+  public static var processor_types = [ "FunctionTokenProcessor", "ConstantTokenProcessor" ];
+
   public function new() {
     this.token_processors = new Hash<TokenProcessor>();
     this.ignored_modules  = new Hash<Bool>();
@@ -16,34 +18,30 @@ class CodeParser {
       Load all possible token processors from disk.
     **/
     public function load_processors_from_disk() {
-      var function_processor = new FunctionTokenProcessor();
+      for (processor_type_name in processor_types) {
-      if (!function_processor.load_from_cache()) {
+        var processor : TokenProcessor = Type.createInstance(Type.resolveClass(processor_type_name), []);
-        function_processor.populate_from_file();
+
-        function_processor.save_to_cache();
+        if (!processor.load_from_cache()) {
+          processor.populate_from_file();
+          processor.save_to_cache();
+        }
+
+        this.token_processors.set(processor_type_name, processor);
       }
-
-      this.token_processors.set(Type.getClassName(Type.getClass(function_processor)), function_processor);
-
-      var constant_processor = new ConstantTokenProcessor();
-      if (!constant_processor.load_from_cache()) {
-        constant_processor.populate_from_files();
-        constant_processor.save_to_cache();
-      }
-
-      this.token_processors.set(Type.getClassName(Type.getClass(constant_processor)), constant_processor);
     }
   #end
 
+  /**
+    Load all possible token processors form haXe Resources.
+  **/
   public function load_processors_from_resources() {
-    var function_processor = new FunctionTokenProcessor();
+    for (processor_type_name in processor_types) {
-    function_processor.load_from_resource();
+      var processor : TokenProcessor = Type.createInstance(Type.resolveClass(processor_type_name), []);
 
-    this.token_processors.set(Type.getClassName(Type.getClass(function_processor)), function_processor);
+      processor.load_from_resource();
 
-    var constant_processor = new ConstantTokenProcessor();
+      this.token_processors.set(processor_type_name, processor);
-    constant_processor.load_from_resource();
+    }
-
-    this.token_processors.set(Type.getClassName(Type.getClass(constant_processor)), constant_processor);
   }
 
   public function get_token_processors() { return this.token_processors; }
@@ -63,9 +61,6 @@ class CodeParser {
     var results = new Array<Result>();
     this.ignored_modules = new Hash<Bool>();
 
-    var function_token_processor = this.token_processors.get("FunctionTokenProcessor");
-    var constant_token_processor = this.token_processors.get("ConstantTokenProcessor");
-
     var tokens_found = new Hash<Bool>();
     var tokens_to_ignore = new Array<Hash<Bool>>();
     var flattened_tokens = new Hash<Bool>();
@@ -119,17 +114,13 @@ class CodeParser {
 
           if (!tokens_found.exists(token)) {
             if (!flattened_tokens.exists(token)) {
-              if (is_function) {
+              for (token_processor in this.token_processors.iterator()) {
-                if (function_token_processor.tokenHash.exists(token)) {
+                if (token_processor.tokenHash.exists(token)) {
-                  results.push(function_token_processor.tokenHash.get(token).toResult());
+                  results.push(token_processor.tokenHash.get(token).toResult()); break;
-                }
-              } else {
-                if (constant_token_processor.tokenHash.exists(token)) {
-                  results.push(constant_token_processor.tokenHash.get(token).toResult());
                 }
               }
+              tokens_found.set(token, true);
             }
-            tokens_found.set(token, true);
           }
         } else {
           if (current == "/") {
@@ -172,8 +163,10 @@ class CodeParser {
     if (is_capturing) {
       var token = s.substr(capture_index, index - capture_index);
 
-      if (constant_token_processor.tokenHash.exists(token)) {
+      for (token_processor in this.token_processors.iterator()) {
-        results.push(constant_token_processor.tokenHash.get(token).toResult());
+        if (token_processor.tokenHash.exists(token)) {
+          results.push(token_processor.tokenHash.get(token).toResult()); break;
+        }
       }
     }
 

src/ConstantToken.hx

@@ -1,5 +1,5 @@
 class ConstantToken extends Token {
-  override public function getTokenType() {
+  override public function get_token_type() {
     return ResultType.Constant;
   }
 }

src/ConstantTokenProcessor.hx

@@ -1,6 +1,7 @@
 class ConstantTokenProcessor extends TokenProcessor {
   public static var cachePath : String = "../data/constant_tokens_cache.hxd";
   override public function get_cache_path() { return ConstantTokenProcessor.cachePath; }
+  override public function get_default_token_type() { return ConstantToken; }
 
   public static var source_path : String = "../data";
   public static var source_file_pattern : EReg = ~/phpdoc_constants_.*\.xml/;
@@ -16,7 +17,7 @@ class ConstantTokenProcessor extends TokenProcessor {
   ];
 
   #if neko
-    public function populate_from_files() {
+    public override function populate_from_file() {
       this.tokenHash = new Hash<Token>();
       for (file in neko.FileSystem.readDirectory(source_path)) {
         if (source_file_pattern.match(file)) {

src/FunctionToken.hx

@@ -1,5 +1,5 @@
 class FunctionToken extends Token {
-  override public function getTokenType() {
+  override public function get_token_type() {
     return ResultType.Function;
   }
 }

src/FunctionTokenProcessor.hx

@@ -2,9 +2,10 @@ class FunctionTokenProcessor extends TokenProcessor {
   public static var cachePath : String = "../data/functions_tokens_cache.hxd";
   override public function get_cache_path() { return FunctionTokenProcessor.cachePath; }
   public static var sourcePath : String = "../data/phpdoc_function_versions.xml";
+  override public function get_default_token_type() { return FunctionToken; }
 
   #if neko
-    public function populate_from_file() {
+    public override function populate_from_file() {
       this.populate_from_string(neko.io.File.getContent(sourcePath));
     }
 

src/JavaScriptTarget.hx

@@ -1,3 +1,8 @@
+import FunctionTokenProcessor;
+import ConstantTokenProcessor;
+import FunctionToken;
+import ConstantToken;
+
 /**
   The JavaScript functionality of Harmonious Code.
 **/
@@ -9,9 +14,6 @@ class JavaScriptTarget {
   static public var manually_ignored_modules : Hash<Bool>;
 
   static public function main() {
-    var function_token = new FunctionToken("a","a");
-    var constant_token = new ConstantToken("a","a");
-
     code_parser = new CodeParser();
     code_parser.load_processors_from_resources();
 

src/RegenerateDataFiles.hx

@@ -10,7 +10,7 @@ class RegenerateDataFiles {
     var constantProcessor = new ConstantTokenProcessor();
     if (!constantProcessor.load_from_cache()) {
       neko.Lib.print("Regenerating constants cache...\n");
-      constantProcessor.populate_from_files();
+      constantProcessor.populate_from_file();
       constantProcessor.save_to_cache();
     }
   }

src/TestFunctionTokenProcessor.hx

@@ -1,17 +1,34 @@
 class TestFunctionTokenProcessor extends haxe.unit.TestCase {
-  static var functionName : String = "test";
+  static var function_name : String = "test";
-  static var functionFrom : String = "5.2";
+  static var function_from : String = "5.2";
-  var testXml : String;
+  var token_processor : FunctionTokenProcessor;
-  var tokenProcessor : FunctionTokenProcessor;
 
   public override function setup() {
-    testXml = "<versions> <function name='" + functionName + "' from='" + functionFrom + "'/> </versions>";
+    var test_xml = "<versions> <function name='" + function_name + "' from='" + function_from + "'/> </versions>";
-    tokenProcessor = new FunctionTokenProcessor();
+    token_processor = new FunctionTokenProcessor();
-    tokenProcessor.populate_from_string(testXml);
+    token_processor.populate_from_string(test_xml);
   }
 
   public function testGenerateSampleToken() {
-    var testTokenArray = [ new FunctionToken(functionName, functionFrom) ];
+    assertTrue(token_processor.tokenHash.exists(function_name));
-    assertTrue(tokenProcessor.tokenHash.exists(functionName));
+  }
+
+  public function testSerializeInfo() {
+    var test_xml = "<versions> <function name='one' from='PHP 4, PHP 5' /> <function name='two' from='PHP 4, PHP 5' /> </versions>";
+    token_processor.populate_from_string(test_xml);
+
+    var target_token_hash = "{one => { version => PHP 4, PHP 5, token => one }, two => { version => PHP 4, PHP 5, token => two }}";
+
+    assertEquals(target_token_hash, token_processor.tokenHash.toString());
+
+    var unwound_tokens = token_processor.unwind_tokens();
+
+    assertTrue(unwound_tokens.toString().length < target_token_hash.length);
+    trace(unwound_tokens.toString().length + " bytes vs. " + target_token_hash.length);
+
+    token_processor = new FunctionTokenProcessor();
+    token_processor.populate_from_unwound_tokens(unwound_tokens);
+
+    assertEquals(target_token_hash, token_processor.tokenHash.toString());
   }
 }

src/Token.hx

@@ -1,16 +1,16 @@
 class Token {
-  public var token(getToken, null) : String;
+  public var token(get_token, null) : String;
-  public var version(getVersion, null) : String;
+  public var version(get_version, null) : String;
-  public var token_type(getTokenType, null) : ResultType;
+  public var token_type(get_token_type, null) : ResultType;
 
   public function new(t : String, ?m : String) {
     this.token = t;
     this.version = m;
   }
 
-  public function getToken() { return this.token; }
+  public function get_token() { return this.token; }
-  public function getVersion() { return this.version; }
+  public function get_version() { return this.version; }
-  public function getTokenType() { return ResultType.Generic; }
+  public function get_token_type() { return ResultType.Generic; }
 
   public function toResult() {
     return new Result(this.token_type, this.token, this.version);

src/TokenProcessor.hx

@@ -4,6 +4,7 @@ class TokenProcessor {
 
   public function new() { this.tokenHash = new Hash<Token>(); }
   public function get_cache_path() { return TokenProcessor.cachePath; }
+  public function get_default_token_type() { return Token; }
 
   #if neko
     public function load_from_cache() : Bool {
@@ -20,9 +21,27 @@ class TokenProcessor {
       fh.writeString(haxe.Serializer.run(this.tokenHash));
       fh.close();
     }
+
+    public function populate_from_file() {}
   #end
 
   public function load_from_resource() {
     this.tokenHash = haxe.Unserializer.run(haxe.Resource.getString(this.get_cache_path()));
   }
+
+  public function unwind_tokens() : Hash<String> {
+    var unwound_tokens = new Hash<String>();
+    for (token in this.tokenHash.keys()) {
+      unwound_tokens.set(token, this.tokenHash.get(token).version);
+    }
+    return unwound_tokens;
+  }
+
+  public function populate_from_unwound_tokens(unwound_tokens : Hash<String>) {
+    this.tokenHash = new Hash<Token>();
+    var token_type = get_default_token_type();
+    for (token in unwound_tokens.keys()) {
+      this.tokenHash.set(token, Type.createInstance(token_type, [ token, unwound_tokens.get(token) ]));
+    }
+  }
 }