Fixed lexer issue with non word characters
This commit is contained in:
@@ -214,4 +214,48 @@ class LexerTest extends Base
|
||||
|
||||
$this->assertSame($expected, $lexer->tokenize('tag:"tag 1" tag:tag2'));
|
||||
}
|
||||
|
||||
public function testTokenizeWithDash()
|
||||
{
|
||||
$lexer = new Lexer();
|
||||
$lexer->addToken("/^(test:)/", 'T_TEST');
|
||||
|
||||
$expected = array(
|
||||
'T_TEST' => array('PO-123'),
|
||||
);
|
||||
|
||||
$this->assertSame($expected, $lexer->tokenize('test:PO-123'));
|
||||
|
||||
$lexer = new Lexer();
|
||||
$lexer->setDefaultToken('myDefaultToken');
|
||||
|
||||
$expected = array(
|
||||
'myDefaultToken' => array('PO-123'),
|
||||
);
|
||||
|
||||
$this->assertSame($expected, $lexer->tokenize('PO-123'));
|
||||
}
|
||||
|
||||
public function testTokenizeWithUnderscore()
|
||||
{
|
||||
$lexer = new Lexer();
|
||||
$lexer->addToken("/^(test:)/", 'T_TEST');
|
||||
|
||||
$expected = array(
|
||||
'T_TEST' => array('PO_123'),
|
||||
);
|
||||
|
||||
$this->assertSame($expected, $lexer->tokenize('test:PO_123'));
|
||||
|
||||
$lexer = new Lexer();
|
||||
$lexer->addToken("/^(test:)/", 'T_TEST');
|
||||
$lexer->setDefaultToken('myDefaultToken');
|
||||
|
||||
$expected = array(
|
||||
'T_TEST' => array('ABC-123'),
|
||||
'myDefaultToken' => array('PO_123'),
|
||||
);
|
||||
|
||||
$this->assertSame($expected, $lexer->tokenize('test:ABC-123 PO_123'));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user