Fixed lexer issue with non word characters

This commit is contained in:
Frederic Guillot
2016-07-01 09:47:10 -04:00
parent 4b94714b3d
commit a089cd72de
3 changed files with 46 additions and 1 deletions

View File

@@ -214,4 +214,48 @@ class LexerTest extends Base
$this->assertSame($expected, $lexer->tokenize('tag:"tag 1" tag:tag2'));
}
public function testTokenizeWithDash()
{
$lexer = new Lexer();
$lexer->addToken("/^(test:)/", 'T_TEST');
$expected = array(
'T_TEST' => array('PO-123'),
);
$this->assertSame($expected, $lexer->tokenize('test:PO-123'));
$lexer = new Lexer();
$lexer->setDefaultToken('myDefaultToken');
$expected = array(
'myDefaultToken' => array('PO-123'),
);
$this->assertSame($expected, $lexer->tokenize('PO-123'));
}
public function testTokenizeWithUnderscore()
{
$lexer = new Lexer();
$lexer->addToken("/^(test:)/", 'T_TEST');
$expected = array(
'T_TEST' => array('PO_123'),
);
$this->assertSame($expected, $lexer->tokenize('test:PO_123'));
$lexer = new Lexer();
$lexer->addToken("/^(test:)/", 'T_TEST');
$lexer->setDefaultToken('myDefaultToken');
$expected = array(
'T_TEST' => array('ABC-123'),
'myDefaultToken' => array('PO_123'),
);
$this->assertSame($expected, $lexer->tokenize('test:ABC-123 PO_123'));
}
}