github-migration/harmoniouscode
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

start working on reducing the size of serialized data

Browse Source
This commit is contained in:
John Bintz 2008-10-14 12:54:35 -04:00
parent 350337d3b9
commit 3374374af0
10 changed files with 90 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
src/CodeParser.hx
View File

@ -6,6 +6,8 @@ class CodeParser {
public var token_processors(get_token_processors, null) : Hash<TokenProcessor>;
public var ignored_modules(get_ignored_modules, null) : Hash<Bool>;
public static var processor_types = [ "FunctionTokenProcessor", "ConstantTokenProcessor" ];
public function new() {
this.token_processors = new Hash<TokenProcessor>();
this.ignored_modules = new Hash<Bool>();
@ -16,34 +18,30 @@ class CodeParser {
Load all possible token processors from disk.
**/
public function load_processors_from_disk() {
var function_processor = new FunctionTokenProcessor();
if (!function_processor.load_from_cache()) {
function_processor.populate_from_file();
function_processor.save_to_cache();
for (processor_type_name in processor_types) {
var processor : TokenProcessor = Type.createInstance(Type.resolveClass(processor_type_name), []);
if (!processor.load_from_cache()) {
processor.populate_from_file();
processor.save_to_cache();
}
this.token_processors.set(Type.getClassName(Type.getClass(function_processor)), function_processor);
var constant_processor = new ConstantTokenProcessor();
if (!constant_processor.load_from_cache()) {
constant_processor.populate_from_files();
constant_processor.save_to_cache();
this.token_processors.set(processor_type_name, processor);
}
this.token_processors.set(Type.getClassName(Type.getClass(constant_processor)), constant_processor);
}
#end
/**
Load all possible token processors form haXe Resources.
**/
public function load_processors_from_resources() {
var function_processor = new FunctionTokenProcessor();
function_processor.load_from_resource();
for (processor_type_name in processor_types) {
var processor : TokenProcessor = Type.createInstance(Type.resolveClass(processor_type_name), []);
this.token_processors.set(Type.getClassName(Type.getClass(function_processor)), function_processor);
processor.load_from_resource();
var constant_processor = new ConstantTokenProcessor();
constant_processor.load_from_resource();
this.token_processors.set(Type.getClassName(Type.getClass(constant_processor)), constant_processor);
this.token_processors.set(processor_type_name, processor);
}
}
public function get_token_processors() { return this.token_processors; }
@ -63,9 +61,6 @@ class CodeParser {
var results = new Array<Result>();
this.ignored_modules = new Hash<Bool>();
var function_token_processor = this.token_processors.get("FunctionTokenProcessor");
var constant_token_processor = this.token_processors.get("ConstantTokenProcessor");
var tokens_found = new Hash<Bool>();
var tokens_to_ignore = new Array<Hash<Bool>>();
var flattened_tokens = new Hash<Bool>();
@ -119,18 +114,14 @@ class CodeParser {
if (!tokens_found.exists(token)) {
if (!flattened_tokens.exists(token)) {
if (is_function) {
if (function_token_processor.tokenHash.exists(token)) {
results.push(function_token_processor.tokenHash.get(token).toResult());
}
} else {
if (constant_token_processor.tokenHash.exists(token)) {
results.push(constant_token_processor.tokenHash.get(token).toResult());
}
for (token_processor in this.token_processors.iterator()) {
if (token_processor.tokenHash.exists(token)) {
results.push(token_processor.tokenHash.get(token).toResult()); break;
}
}
tokens_found.set(token, true);
}
}
} else {
if (current == "/") {
if (s.indexOf("//harmonious", index) == index) {
@ -172,8 +163,10 @@ class CodeParser {
if (is_capturing) {
var token = s.substr(capture_index, index - capture_index);
if (constant_token_processor.tokenHash.exists(token)) {
results.push(constant_token_processor.tokenHash.get(token).toResult());
for (token_processor in this.token_processors.iterator()) {
if (token_processor.tokenHash.exists(token)) {
results.push(token_processor.tokenHash.get(token).toResult()); break;
}
}
}

2
src/ConstantToken.hx
View File

@ -1,5 +1,5 @@
class ConstantToken extends Token {
override public function getTokenType() {
override public function get_token_type() {
return ResultType.Constant;
}
}

3
src/ConstantTokenProcessor.hx
View File

@ -1,6 +1,7 @@
class ConstantTokenProcessor extends TokenProcessor {
public static var cachePath : String = "../data/constant_tokens_cache.hxd";
override public function get_cache_path() { return ConstantTokenProcessor.cachePath; }
override public function get_default_token_type() { return ConstantToken; }
public static var source_path : String = "../data";
public static var source_file_pattern : EReg = ~/phpdoc_constants_.*\.xml/;
@ -16,7 +17,7 @@ class ConstantTokenProcessor extends TokenProcessor {
];
#if neko
public function populate_from_files() {
public override function populate_from_file() {
this.tokenHash = new Hash<Token>();
for (file in neko.FileSystem.readDirectory(source_path)) {
if (source_file_pattern.match(file)) {

2
src/FunctionToken.hx
View File

@ -1,5 +1,5 @@
class FunctionToken extends Token {
override public function getTokenType() {
override public function get_token_type() {
return ResultType.Function;
}
}

3
src/FunctionTokenProcessor.hx
View File

@ -2,9 +2,10 @@ class FunctionTokenProcessor extends TokenProcessor {
public static var cachePath : String = "../data/functions_tokens_cache.hxd";
override public function get_cache_path() { return FunctionTokenProcessor.cachePath; }
public static var sourcePath : String = "../data/phpdoc_function_versions.xml";
override public function get_default_token_type() { return FunctionToken; }
#if neko
public function populate_from_file() {
public override function populate_from_file() {
this.populate_from_string(neko.io.File.getContent(sourcePath));
}

8
src/JavaScriptTarget.hx
View File

@ -1,3 +1,8 @@
import FunctionTokenProcessor;
import ConstantTokenProcessor;
import FunctionToken;
import ConstantToken;
/**
The JavaScript functionality of Harmonious Code.
**/
@ -9,9 +14,6 @@ class JavaScriptTarget {
static public var manually_ignored_modules : Hash<Bool>;
static public function main() {
var function_token = new FunctionToken("a","a");
var constant_token = new ConstantToken("a","a");
code_parser = new CodeParser();
code_parser.load_processors_from_resources();

2
src/RegenerateDataFiles.hx
View File

@ -10,7 +10,7 @@ class RegenerateDataFiles {
var constantProcessor = new ConstantTokenProcessor();
if (!constantProcessor.load_from_cache()) {
neko.Lib.print("Regenerating constants cache...\n");
constantProcessor.populate_from_files();
constantProcessor.populate_from_file();
constantProcessor.save_to_cache();
}
}

35
src/TestFunctionTokenProcessor.hx
View File

@ -1,17 +1,34 @@
class TestFunctionTokenProcessor extends haxe.unit.TestCase {
static var functionName : String = "test";
static var functionFrom : String = "5.2";
var testXml : String;
var tokenProcessor : FunctionTokenProcessor;
static var function_name : String = "test";
static var function_from : String = "5.2";
var token_processor : FunctionTokenProcessor;
public override function setup() {
testXml = "<versions> <function name='" + functionName + "' from='" + functionFrom + "'/> </versions>";
tokenProcessor = new FunctionTokenProcessor();
tokenProcessor.populate_from_string(testXml);
var test_xml = "<versions> <function name='" + function_name + "' from='" + function_from + "'/> </versions>";
token_processor = new FunctionTokenProcessor();
token_processor.populate_from_string(test_xml);
}
public function testGenerateSampleToken() {
var testTokenArray = [ new FunctionToken(functionName, functionFrom) ];
assertTrue(tokenProcessor.tokenHash.exists(functionName));
assertTrue(token_processor.tokenHash.exists(function_name));
}
public function testSerializeInfo() {
var test_xml = "<versions> <function name='one' from='PHP 4, PHP 5' /> <function name='two' from='PHP 4, PHP 5' /> </versions>";
token_processor.populate_from_string(test_xml);
var target_token_hash = "{one => { version => PHP 4, PHP 5, token => one }, two => { version => PHP 4, PHP 5, token => two }}";
assertEquals(target_token_hash, token_processor.tokenHash.toString());
var unwound_tokens = token_processor.unwind_tokens();
assertTrue(unwound_tokens.toString().length < target_token_hash.length);
trace(unwound_tokens.toString().length + " bytes vs. " + target_token_hash.length);
token_processor = new FunctionTokenProcessor();
token_processor.populate_from_unwound_tokens(unwound_tokens);
assertEquals(target_token_hash, token_processor.tokenHash.toString());
}
}

12
src/Token.hx
View File

@ -1,16 +1,16 @@
class Token {
public var token(getToken, null) : String;
public var version(getVersion, null) : String;
public var token_type(getTokenType, null) : ResultType;
public var token(get_token, null) : String;
public var version(get_version, null) : String;
public var token_type(get_token_type, null) : ResultType;
public function new(t : String, ?m : String) {
this.token = t;
this.version = m;
}
public function getToken() { return this.token; }
public function getVersion() { return this.version; }
public function getTokenType() { return ResultType.Generic; }
public function get_token() { return this.token; }
public function get_version() { return this.version; }
public function get_token_type() { return ResultType.Generic; }
public function toResult() {
return new Result(this.token_type, this.token, this.version);

19
src/TokenProcessor.hx
View File

@ -4,6 +4,7 @@ class TokenProcessor {
public function new() { this.tokenHash = new Hash<Token>(); }
public function get_cache_path() { return TokenProcessor.cachePath; }
public function get_default_token_type() { return Token; }
#if neko
public function load_from_cache() : Bool {
@ -20,9 +21,27 @@ class TokenProcessor {
fh.writeString(haxe.Serializer.run(this.tokenHash));
fh.close();
}
public function populate_from_file() {}
#end
public function load_from_resource() {
this.tokenHash = haxe.Unserializer.run(haxe.Resource.getString(this.get_cache_path()));
}
public function unwind_tokens() : Hash<String> {
var unwound_tokens = new Hash<String>();
for (token in this.tokenHash.keys()) {
unwound_tokens.set(token, this.tokenHash.get(token).version);
}
return unwound_tokens;
}
public function populate_from_unwound_tokens(unwound_tokens : Hash<String>) {
this.tokenHash = new Hash<Token>();
var token_type = get_default_token_type();
for (token in unwound_tokens.keys()) {
this.tokenHash.set(token, Type.createInstance(token_type, [ token, unwound_tokens.get(token) ]));
}
}
}