start working on reducing the size of serialized data
This commit is contained in:
parent
350337d3b9
commit
3374374af0
@ -6,6 +6,8 @@ class CodeParser {
|
|||||||
public var token_processors(get_token_processors, null) : Hash<TokenProcessor>;
|
public var token_processors(get_token_processors, null) : Hash<TokenProcessor>;
|
||||||
public var ignored_modules(get_ignored_modules, null) : Hash<Bool>;
|
public var ignored_modules(get_ignored_modules, null) : Hash<Bool>;
|
||||||
|
|
||||||
|
public static var processor_types = [ "FunctionTokenProcessor", "ConstantTokenProcessor" ];
|
||||||
|
|
||||||
public function new() {
|
public function new() {
|
||||||
this.token_processors = new Hash<TokenProcessor>();
|
this.token_processors = new Hash<TokenProcessor>();
|
||||||
this.ignored_modules = new Hash<Bool>();
|
this.ignored_modules = new Hash<Bool>();
|
||||||
@ -16,34 +18,30 @@ class CodeParser {
|
|||||||
Load all possible token processors from disk.
|
Load all possible token processors from disk.
|
||||||
**/
|
**/
|
||||||
public function load_processors_from_disk() {
|
public function load_processors_from_disk() {
|
||||||
var function_processor = new FunctionTokenProcessor();
|
for (processor_type_name in processor_types) {
|
||||||
if (!function_processor.load_from_cache()) {
|
var processor : TokenProcessor = Type.createInstance(Type.resolveClass(processor_type_name), []);
|
||||||
function_processor.populate_from_file();
|
|
||||||
function_processor.save_to_cache();
|
if (!processor.load_from_cache()) {
|
||||||
|
processor.populate_from_file();
|
||||||
|
processor.save_to_cache();
|
||||||
|
}
|
||||||
|
|
||||||
|
this.token_processors.set(processor_type_name, processor);
|
||||||
}
|
}
|
||||||
|
|
||||||
this.token_processors.set(Type.getClassName(Type.getClass(function_processor)), function_processor);
|
|
||||||
|
|
||||||
var constant_processor = new ConstantTokenProcessor();
|
|
||||||
if (!constant_processor.load_from_cache()) {
|
|
||||||
constant_processor.populate_from_files();
|
|
||||||
constant_processor.save_to_cache();
|
|
||||||
}
|
|
||||||
|
|
||||||
this.token_processors.set(Type.getClassName(Type.getClass(constant_processor)), constant_processor);
|
|
||||||
}
|
}
|
||||||
#end
|
#end
|
||||||
|
|
||||||
|
/**
|
||||||
|
Load all possible token processors form haXe Resources.
|
||||||
|
**/
|
||||||
public function load_processors_from_resources() {
|
public function load_processors_from_resources() {
|
||||||
var function_processor = new FunctionTokenProcessor();
|
for (processor_type_name in processor_types) {
|
||||||
function_processor.load_from_resource();
|
var processor : TokenProcessor = Type.createInstance(Type.resolveClass(processor_type_name), []);
|
||||||
|
|
||||||
this.token_processors.set(Type.getClassName(Type.getClass(function_processor)), function_processor);
|
processor.load_from_resource();
|
||||||
|
|
||||||
var constant_processor = new ConstantTokenProcessor();
|
this.token_processors.set(processor_type_name, processor);
|
||||||
constant_processor.load_from_resource();
|
}
|
||||||
|
|
||||||
this.token_processors.set(Type.getClassName(Type.getClass(constant_processor)), constant_processor);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public function get_token_processors() { return this.token_processors; }
|
public function get_token_processors() { return this.token_processors; }
|
||||||
@ -63,9 +61,6 @@ class CodeParser {
|
|||||||
var results = new Array<Result>();
|
var results = new Array<Result>();
|
||||||
this.ignored_modules = new Hash<Bool>();
|
this.ignored_modules = new Hash<Bool>();
|
||||||
|
|
||||||
var function_token_processor = this.token_processors.get("FunctionTokenProcessor");
|
|
||||||
var constant_token_processor = this.token_processors.get("ConstantTokenProcessor");
|
|
||||||
|
|
||||||
var tokens_found = new Hash<Bool>();
|
var tokens_found = new Hash<Bool>();
|
||||||
var tokens_to_ignore = new Array<Hash<Bool>>();
|
var tokens_to_ignore = new Array<Hash<Bool>>();
|
||||||
var flattened_tokens = new Hash<Bool>();
|
var flattened_tokens = new Hash<Bool>();
|
||||||
@ -119,17 +114,13 @@ class CodeParser {
|
|||||||
|
|
||||||
if (!tokens_found.exists(token)) {
|
if (!tokens_found.exists(token)) {
|
||||||
if (!flattened_tokens.exists(token)) {
|
if (!flattened_tokens.exists(token)) {
|
||||||
if (is_function) {
|
for (token_processor in this.token_processors.iterator()) {
|
||||||
if (function_token_processor.tokenHash.exists(token)) {
|
if (token_processor.tokenHash.exists(token)) {
|
||||||
results.push(function_token_processor.tokenHash.get(token).toResult());
|
results.push(token_processor.tokenHash.get(token).toResult()); break;
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if (constant_token_processor.tokenHash.exists(token)) {
|
|
||||||
results.push(constant_token_processor.tokenHash.get(token).toResult());
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
tokens_found.set(token, true);
|
||||||
}
|
}
|
||||||
tokens_found.set(token, true);
|
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
if (current == "/") {
|
if (current == "/") {
|
||||||
@ -172,8 +163,10 @@ class CodeParser {
|
|||||||
if (is_capturing) {
|
if (is_capturing) {
|
||||||
var token = s.substr(capture_index, index - capture_index);
|
var token = s.substr(capture_index, index - capture_index);
|
||||||
|
|
||||||
if (constant_token_processor.tokenHash.exists(token)) {
|
for (token_processor in this.token_processors.iterator()) {
|
||||||
results.push(constant_token_processor.tokenHash.get(token).toResult());
|
if (token_processor.tokenHash.exists(token)) {
|
||||||
|
results.push(token_processor.tokenHash.get(token).toResult()); break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1,5 +1,5 @@
|
|||||||
class ConstantToken extends Token {
|
class ConstantToken extends Token {
|
||||||
override public function getTokenType() {
|
override public function get_token_type() {
|
||||||
return ResultType.Constant;
|
return ResultType.Constant;
|
||||||
}
|
}
|
||||||
}
|
}
|
@ -1,6 +1,7 @@
|
|||||||
class ConstantTokenProcessor extends TokenProcessor {
|
class ConstantTokenProcessor extends TokenProcessor {
|
||||||
public static var cachePath : String = "../data/constant_tokens_cache.hxd";
|
public static var cachePath : String = "../data/constant_tokens_cache.hxd";
|
||||||
override public function get_cache_path() { return ConstantTokenProcessor.cachePath; }
|
override public function get_cache_path() { return ConstantTokenProcessor.cachePath; }
|
||||||
|
override public function get_default_token_type() { return ConstantToken; }
|
||||||
|
|
||||||
public static var source_path : String = "../data";
|
public static var source_path : String = "../data";
|
||||||
public static var source_file_pattern : EReg = ~/phpdoc_constants_.*\.xml/;
|
public static var source_file_pattern : EReg = ~/phpdoc_constants_.*\.xml/;
|
||||||
@ -16,7 +17,7 @@ class ConstantTokenProcessor extends TokenProcessor {
|
|||||||
];
|
];
|
||||||
|
|
||||||
#if neko
|
#if neko
|
||||||
public function populate_from_files() {
|
public override function populate_from_file() {
|
||||||
this.tokenHash = new Hash<Token>();
|
this.tokenHash = new Hash<Token>();
|
||||||
for (file in neko.FileSystem.readDirectory(source_path)) {
|
for (file in neko.FileSystem.readDirectory(source_path)) {
|
||||||
if (source_file_pattern.match(file)) {
|
if (source_file_pattern.match(file)) {
|
||||||
|
@ -1,5 +1,5 @@
|
|||||||
class FunctionToken extends Token {
|
class FunctionToken extends Token {
|
||||||
override public function getTokenType() {
|
override public function get_token_type() {
|
||||||
return ResultType.Function;
|
return ResultType.Function;
|
||||||
}
|
}
|
||||||
}
|
}
|
@ -2,9 +2,10 @@ class FunctionTokenProcessor extends TokenProcessor {
|
|||||||
public static var cachePath : String = "../data/functions_tokens_cache.hxd";
|
public static var cachePath : String = "../data/functions_tokens_cache.hxd";
|
||||||
override public function get_cache_path() { return FunctionTokenProcessor.cachePath; }
|
override public function get_cache_path() { return FunctionTokenProcessor.cachePath; }
|
||||||
public static var sourcePath : String = "../data/phpdoc_function_versions.xml";
|
public static var sourcePath : String = "../data/phpdoc_function_versions.xml";
|
||||||
|
override public function get_default_token_type() { return FunctionToken; }
|
||||||
|
|
||||||
#if neko
|
#if neko
|
||||||
public function populate_from_file() {
|
public override function populate_from_file() {
|
||||||
this.populate_from_string(neko.io.File.getContent(sourcePath));
|
this.populate_from_string(neko.io.File.getContent(sourcePath));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1,3 +1,8 @@
|
|||||||
|
import FunctionTokenProcessor;
|
||||||
|
import ConstantTokenProcessor;
|
||||||
|
import FunctionToken;
|
||||||
|
import ConstantToken;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
The JavaScript functionality of Harmonious Code.
|
The JavaScript functionality of Harmonious Code.
|
||||||
**/
|
**/
|
||||||
@ -9,9 +14,6 @@ class JavaScriptTarget {
|
|||||||
static public var manually_ignored_modules : Hash<Bool>;
|
static public var manually_ignored_modules : Hash<Bool>;
|
||||||
|
|
||||||
static public function main() {
|
static public function main() {
|
||||||
var function_token = new FunctionToken("a","a");
|
|
||||||
var constant_token = new ConstantToken("a","a");
|
|
||||||
|
|
||||||
code_parser = new CodeParser();
|
code_parser = new CodeParser();
|
||||||
code_parser.load_processors_from_resources();
|
code_parser.load_processors_from_resources();
|
||||||
|
|
||||||
|
@ -10,7 +10,7 @@ class RegenerateDataFiles {
|
|||||||
var constantProcessor = new ConstantTokenProcessor();
|
var constantProcessor = new ConstantTokenProcessor();
|
||||||
if (!constantProcessor.load_from_cache()) {
|
if (!constantProcessor.load_from_cache()) {
|
||||||
neko.Lib.print("Regenerating constants cache...\n");
|
neko.Lib.print("Regenerating constants cache...\n");
|
||||||
constantProcessor.populate_from_files();
|
constantProcessor.populate_from_file();
|
||||||
constantProcessor.save_to_cache();
|
constantProcessor.save_to_cache();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1,17 +1,34 @@
|
|||||||
class TestFunctionTokenProcessor extends haxe.unit.TestCase {
|
class TestFunctionTokenProcessor extends haxe.unit.TestCase {
|
||||||
static var functionName : String = "test";
|
static var function_name : String = "test";
|
||||||
static var functionFrom : String = "5.2";
|
static var function_from : String = "5.2";
|
||||||
var testXml : String;
|
var token_processor : FunctionTokenProcessor;
|
||||||
var tokenProcessor : FunctionTokenProcessor;
|
|
||||||
|
|
||||||
public override function setup() {
|
public override function setup() {
|
||||||
testXml = "<versions> <function name='" + functionName + "' from='" + functionFrom + "'/> </versions>";
|
var test_xml = "<versions> <function name='" + function_name + "' from='" + function_from + "'/> </versions>";
|
||||||
tokenProcessor = new FunctionTokenProcessor();
|
token_processor = new FunctionTokenProcessor();
|
||||||
tokenProcessor.populate_from_string(testXml);
|
token_processor.populate_from_string(test_xml);
|
||||||
}
|
}
|
||||||
|
|
||||||
public function testGenerateSampleToken() {
|
public function testGenerateSampleToken() {
|
||||||
var testTokenArray = [ new FunctionToken(functionName, functionFrom) ];
|
assertTrue(token_processor.tokenHash.exists(function_name));
|
||||||
assertTrue(tokenProcessor.tokenHash.exists(functionName));
|
}
|
||||||
|
|
||||||
|
public function testSerializeInfo() {
|
||||||
|
var test_xml = "<versions> <function name='one' from='PHP 4, PHP 5' /> <function name='two' from='PHP 4, PHP 5' /> </versions>";
|
||||||
|
token_processor.populate_from_string(test_xml);
|
||||||
|
|
||||||
|
var target_token_hash = "{one => { version => PHP 4, PHP 5, token => one }, two => { version => PHP 4, PHP 5, token => two }}";
|
||||||
|
|
||||||
|
assertEquals(target_token_hash, token_processor.tokenHash.toString());
|
||||||
|
|
||||||
|
var unwound_tokens = token_processor.unwind_tokens();
|
||||||
|
|
||||||
|
assertTrue(unwound_tokens.toString().length < target_token_hash.length);
|
||||||
|
trace(unwound_tokens.toString().length + " bytes vs. " + target_token_hash.length);
|
||||||
|
|
||||||
|
token_processor = new FunctionTokenProcessor();
|
||||||
|
token_processor.populate_from_unwound_tokens(unwound_tokens);
|
||||||
|
|
||||||
|
assertEquals(target_token_hash, token_processor.tokenHash.toString());
|
||||||
}
|
}
|
||||||
}
|
}
|
12
src/Token.hx
12
src/Token.hx
@ -1,16 +1,16 @@
|
|||||||
class Token {
|
class Token {
|
||||||
public var token(getToken, null) : String;
|
public var token(get_token, null) : String;
|
||||||
public var version(getVersion, null) : String;
|
public var version(get_version, null) : String;
|
||||||
public var token_type(getTokenType, null) : ResultType;
|
public var token_type(get_token_type, null) : ResultType;
|
||||||
|
|
||||||
public function new(t : String, ?m : String) {
|
public function new(t : String, ?m : String) {
|
||||||
this.token = t;
|
this.token = t;
|
||||||
this.version = m;
|
this.version = m;
|
||||||
}
|
}
|
||||||
|
|
||||||
public function getToken() { return this.token; }
|
public function get_token() { return this.token; }
|
||||||
public function getVersion() { return this.version; }
|
public function get_version() { return this.version; }
|
||||||
public function getTokenType() { return ResultType.Generic; }
|
public function get_token_type() { return ResultType.Generic; }
|
||||||
|
|
||||||
public function toResult() {
|
public function toResult() {
|
||||||
return new Result(this.token_type, this.token, this.version);
|
return new Result(this.token_type, this.token, this.version);
|
||||||
|
@ -4,6 +4,7 @@ class TokenProcessor {
|
|||||||
|
|
||||||
public function new() { this.tokenHash = new Hash<Token>(); }
|
public function new() { this.tokenHash = new Hash<Token>(); }
|
||||||
public function get_cache_path() { return TokenProcessor.cachePath; }
|
public function get_cache_path() { return TokenProcessor.cachePath; }
|
||||||
|
public function get_default_token_type() { return Token; }
|
||||||
|
|
||||||
#if neko
|
#if neko
|
||||||
public function load_from_cache() : Bool {
|
public function load_from_cache() : Bool {
|
||||||
@ -20,9 +21,27 @@ class TokenProcessor {
|
|||||||
fh.writeString(haxe.Serializer.run(this.tokenHash));
|
fh.writeString(haxe.Serializer.run(this.tokenHash));
|
||||||
fh.close();
|
fh.close();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public function populate_from_file() {}
|
||||||
#end
|
#end
|
||||||
|
|
||||||
public function load_from_resource() {
|
public function load_from_resource() {
|
||||||
this.tokenHash = haxe.Unserializer.run(haxe.Resource.getString(this.get_cache_path()));
|
this.tokenHash = haxe.Unserializer.run(haxe.Resource.getString(this.get_cache_path()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public function unwind_tokens() : Hash<String> {
|
||||||
|
var unwound_tokens = new Hash<String>();
|
||||||
|
for (token in this.tokenHash.keys()) {
|
||||||
|
unwound_tokens.set(token, this.tokenHash.get(token).version);
|
||||||
|
}
|
||||||
|
return unwound_tokens;
|
||||||
|
}
|
||||||
|
|
||||||
|
public function populate_from_unwound_tokens(unwound_tokens : Hash<String>) {
|
||||||
|
this.tokenHash = new Hash<Token>();
|
||||||
|
var token_type = get_default_token_type();
|
||||||
|
for (token in unwound_tokens.keys()) {
|
||||||
|
this.tokenHash.set(token, Type.createInstance(token_type, [ token, unwound_tokens.get(token) ]));
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
Loading…
Reference in New Issue
Block a user