Fixed lexer issue with non word characters

This commit is contained in:
Frederic Guillot 2016-07-01 09:47:10 -04:00
parent 4b94714b3d
commit a089cd72de
No known key found for this signature in database
GPG Key ID: 92D77191BA7FBC99
3 changed files with 46 additions and 1 deletions

View File

@ -21,6 +21,7 @@ Bug fixes:
* Fixed broken CSV exports
* Fixed identical background color for LetterAvatar on 32bits platforms (Hash greater than PHP_MAX_INT)
* Fixed lexer issue with non word characters
Version 1.0.30
--------------

View File

@ -30,7 +30,7 @@ class Lexer
'/^([<=>]{1,2}\w+)/u' => 'T_STRING',
'/^([<=>]{1,2}".+")/' => 'T_STRING',
'/^("(.+)")/' => 'T_STRING',
'/^(\w+)/u' => 'T_STRING',
'/^(\S+)/u' => 'T_STRING',
'/^(#\d+)/' => 'T_STRING',
);

View File

@ -214,4 +214,48 @@ class LexerTest extends Base
$this->assertSame($expected, $lexer->tokenize('tag:"tag 1" tag:tag2'));
}
public function testTokenizeWithDash()
{
$lexer = new Lexer();
$lexer->addToken("/^(test:)/", 'T_TEST');
$expected = array(
'T_TEST' => array('PO-123'),
);
$this->assertSame($expected, $lexer->tokenize('test:PO-123'));
$lexer = new Lexer();
$lexer->setDefaultToken('myDefaultToken');
$expected = array(
'myDefaultToken' => array('PO-123'),
);
$this->assertSame($expected, $lexer->tokenize('PO-123'));
}
public function testTokenizeWithUnderscore()
{
$lexer = new Lexer();
$lexer->addToken("/^(test:)/", 'T_TEST');
$expected = array(
'T_TEST' => array('PO_123'),
);
$this->assertSame($expected, $lexer->tokenize('test:PO_123'));
$lexer = new Lexer();
$lexer->addToken("/^(test:)/", 'T_TEST');
$lexer->setDefaultToken('myDefaultToken');
$expected = array(
'T_TEST' => array('ABC-123'),
'myDefaultToken' => array('PO_123'),
);
$this->assertSame($expected, $lexer->tokenize('test:ABC-123 PO_123'));
}
}