Fix filters with UTF-8 characters

2016-05-31 15:44:09 -05:00
parent 4987e245bb
commit 6af3b6dfdb
2 changed files with 32 additions and 7 deletions
--- a/tests/units/Core/Filter/LexerTest.php
+++ b/tests/units/Core/Filter/LexerTest.php
@@ -178,4 +178,28 @@ class LexerTest extends Base

        $this->assertSame($expected, $lexer->tokenize('date:<=2016-01-01 something else'));
    }
+
+    public function testTokenizeWithUtf8Letters()
+    {
+        $lexer = new Lexer();
+        $lexer->setDefaultToken('myDefaultToken');
+
+        $expected = array(
+            'myDefaultToken' => array('àa éçùe'),
+        );
+
+        $this->assertSame($expected, $lexer->tokenize('àa éçùe'));
+    }
+
+    public function testTokenizeWithUtf8Numbers()
+    {
+        $lexer = new Lexer();
+        $lexer->setDefaultToken('myDefaultToken');
+
+        $expected = array(
+            'myDefaultToken' => array('६Δↈ五一'),
+        );
+
+        $this->assertSame($expected, $lexer->tokenize('६Δↈ五一'));
+    }
 }