Add user CSV import

ChangeLog

@@ -1,9 +1,14 @@
 Version 1.0.20 (unreleased)
 ---------------------------
 
+New features:
+
+* Add users CSV import
+
 Improvements:
 
 * Allow to change comments sorting
+* Add the possibility to append or not custom filters
 
 Version 1.0.19
 --------------

app/Console/ProjectDailyColumnStatsExport.php

@@ -2,7 +2,7 @@
 
 namespace Console;
 
-use Core\Tool;
+use Core\Csv;
 use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Output\OutputInterface;
@@ -28,7 +28,7 @@ class ProjectDailyColumnStatsExport extends Base
         );
 
         if (is_array($data)) {
-            Tool::csv($data);
+            Csv::output($data);
         }
     }
 }

app/Console/SubtaskExport.php

@@ -2,7 +2,7 @@
 
 namespace Console;
 
-use Core\Tool;
+use Core\Csv;
 use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Output\OutputInterface;
@@ -28,7 +28,7 @@ class SubtaskExport extends Base
         );
 
         if (is_array($data)) {
-            Tool::csv($data);
+            Csv::output($data);
         }
     }
 }

app/Console/TaskExport.php

@@ -2,7 +2,7 @@
 
 namespace Console;
 
-use Core\Tool;
+use Core\Csv;
 use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Output\OutputInterface;
@@ -28,7 +28,7 @@ class TaskExport extends Base
         );
 
         if (is_array($data)) {
-            Tool::csv($data);
+            Csv::output($data);
         }
     }
 }

app/Console/TransitionExport.php

@@ -2,7 +2,7 @@
 
 namespace Console;
 
-use Core\Tool;
+use Core\Csv;
 use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Output\OutputInterface;
@@ -28,7 +28,7 @@ class TransitionExport extends Base
         );
 
         if (is_array($data)) {
-            Tool::csv($data);
+            Csv::output($data);
         }
     }
 }

app/Controller/UserImport.php

@@ -0,0 +1,67 @@
+<?php
+
+namespace Controller;
+
+use Core\Csv;
+
+/**
+ * User Import controller
+ *
+ * @package  controller
+ * @author   Frederic Guillot
+ */
+class UserImport extends Base
+{
+    /**
+     * Upload the file and ask settings
+     *
+     */
+    public function step1(array $values = array(), array $errors = array())
+    {
+        $this->response->html($this->template->layout('user_import/step1', array(
+            'values' => $values,
+            'errors' => $errors,
+            'max_size' => ini_get('upload_max_filesize'),
+            'delimiters' => Csv::getDelimiters(),
+            'enclosures' => Csv::getEnclosures(),
+            'title' => t('Import users from CSV file'),
+        )));
+    }
+
+    /**
+     * Process CSV file
+     *
+     */
+    public function step2()
+    {
+        $values = $this->request->getValues();
+        $filename = $this->request->getFilePath('file');
+
+        if (! file_exists($filename)) {
+            $this->step1($values, array('file' => array(t('Unable to read your file'))));
+        }
+
+        $csv = new Csv($values['delimiter'], $values['enclosure']);
+        $csv->setColumnMapping($this->userImport->getColumnMapping());
+        $csv->read($filename, array($this->userImport, 'import'));
+
+        if ($this->userImport->counter > 0) {
+            $this->session->flash(t('%d user(s) have been imported successfully.', $this->userImport->counter));
+        }
+        else {
+            $this->session->flash(t('Nothing have been imported!'));
+        }
+
+        $this->response->redirect($this->helper->url->to('userImport', 'step1'));
+    }
+
+    /**
+     * Generate template
+     *
+     */
+    public function template()
+    {
+        $this->response->forceDownload('users.csv');
+        $this->response->csv(array($this->userImport->getColumnMapping()));
+    }
+}

app/Core/Csv.php

@@ -0,0 +1,212 @@
+<?php
+
+namespace Core;
+
+use SplFileObject;
+
+/**
+ * CSV Writer/Reader
+ *
+ * @package core
+ * @author  Frederic Guillot
+ */
+class Csv
+{
+    /**
+     * CSV delimiter
+     *
+     * @access private
+     * @var string
+     */
+    private $delimiter = ',';
+
+    /**
+     * CSV enclosure
+     *
+     * @access private
+     * @var string
+     */
+    private $enclosure = '"';
+
+    /**
+     * CSV/SQL columns
+     *
+     * @access private
+     * @var array
+     */
+    private $columns = array();
+
+    /**
+     * Constructor
+     *
+     * @access public
+     * @param  string  $delimiter
+     * @param  string  $enclosure
+     */
+    public function __construct($delimiter = ',', $enclosure = '"')
+    {
+        $this->delimiter = $delimiter;
+        $this->enclosure = $enclosure;
+    }
+
+    /**
+     * Get list of delimiters
+     *
+     * @static
+     * @access public
+     * @return array
+     */
+    public static function getDelimiters()
+    {
+        return array(
+            ',' => t('Comma'),
+            ';' => t('Semi-colon'),
+            '\t' => t('Tab'),
+            '|' => t('Vertical bar'),
+        );
+    }
+
+    /**
+     * Get list of enclosures
+     *
+     * @static
+     * @access public
+     * @return array
+     */
+    public static function getEnclosures()
+    {
+        return array(
+            '"' => t('Double Quote'),
+            "'" => t('Single Quote'),
+            '' => t('None'),
+        );
+    }
+
+    /**
+     * Check boolean field value
+     *
+     * @static
+     * @access public
+     * @return integer
+     */
+    public static function getBooleanValue($value)
+    {
+        if (! empty($value)) {
+            $value = trim(strtolower($value));
+            return $value === '1' || $value{0} === 't' ? 1 : 0;
+        }
+
+        return 0;
+    }
+
+    /**
+     * Output CSV file to standard output
+     *
+     * @static
+     * @access public
+     * @param  array  $rows
+     */
+    public static function output(array $rows)
+    {
+        $csv = new static;
+        $csv->write('php://output', $rows);
+    }
+
+    /**
+     * Define column mapping between CSV and SQL columns
+     *
+     * @access public
+     * @param  array $columns
+     * @return Csv
+     */
+    public function setColumnMapping(array $columns)
+    {
+        $this->columns = $columns;
+        return $this;
+    }
+
+    /**
+     * Read CSV file
+     *
+     * @access public
+     * @param  string    $filename
+     * @param  \Closure  $callback   Example: function(array $row, $line_number)
+     * @return Csv
+     */
+    public function read($filename, $callback)
+    {
+        $file = new SplFileObject($filename);
+        $file->setFlags(SplFileObject::READ_CSV);
+        $file->setCsvControl($this->delimiter, $this->enclosure);
+        $line_number = 0;
+
+        foreach ($file as $row) {
+            $row = $this->filterRow($row);
+
+            if (! empty($row) && $line_number > 0) {
+                call_user_func_array($callback, array($this->associateColumns($row), $line_number));
+            }
+
+            $line_number++;
+        }
+
+        return $this;
+    }
+
+    /**
+     * Write CSV file
+     *
+     * @access public
+     * @param  string    $filename
+     * @param  array     $rows
+     * @return Csv
+     */
+    public function write($filename, array $rows)
+    {
+        $file = new SplFileObject($filename, 'w');
+
+        foreach ($rows as $row) {
+            $file->fputcsv($row, $this->delimiter, $this->enclosure);
+        }
+
+        return $this;
+    }
+
+    /**
+     * Associate columns header with row values
+     *
+     * @access private
+     * @param  array $row
+     * @return array
+     */
+    private function associateColumns(array $row)
+    {
+        $line = array();
+        $index = 0;
+
+        foreach ($this->columns as $sql_name => $csv_name) {
+            if (isset($row[$index])) {
+                $line[$sql_name] = $row[$index];
+            }
+            else {
+                $line[$sql_name] = '';
+            }
+
+            $index++;
+        }
+
+        return $line;
+    }
+
+    /**
+     * Filter empty rows
+     *
+     * @access private
+     * @param  array $row
+     * @return array
+     */
+    private function filterRow(array $row)
+    {
+        return array_filter($row);
+    }
+}

app/Core/Request.php

@@ -102,6 +102,18 @@ class Request
         return '';
     }
 
+    /**
+     * Get the path of an uploaded file
+     *
+     * @access public
+     * @param  string   $name   Form file name
+     * @return string
+     */
+    public function getFilePath($name)
+    {
+        return isset($_FILES[$name]['tmp_name']) ? $_FILES[$name]['tmp_name'] : '';
+    }
+
     /**
      * Return true if the HTTP request is sent with the POST method
      *

app/Core/Response.php

@@ -87,8 +87,9 @@ class Response
     {
         $this->status($status_code);
         $this->nocache();
+
         header('Content-Type: text/csv');
-        Tool::csv($data);
+        Csv::output($data);
         exit;
     }
 

app/Core/Router.php

@@ -197,7 +197,7 @@ class Router extends Base
      */
     public function sanitize($value, $default_value)
     {
-        return ! ctype_alpha($value) || empty($value) ? $default_value : strtolower($value);
+        return ! preg_match('/^[a-zA-Z_0-9]+$/', $value) ? $default_value : $value;
     }
 
     /**
@@ -218,6 +218,7 @@ class Router extends Base
             list($this->controller, $this->action) = $this->findRoute($this->getPath($uri, $query_string)); // TODO: add plugin for routes
             $plugin = '';
         }
+
         $class = empty($plugin) ? '\Controller\\'.ucfirst($this->controller) : '\Plugin\\'.ucfirst($plugin).'\Controller\\'.ucfirst($this->controller);
 
         $instance = new $class($this->container);

app/Core/Tool.php

@@ -12,27 +12,6 @@ use Pimple\Container;
  */
 class Tool
 {
-    /**
-     * Write a CSV file
-     *
-     * @static
-     * @access public
-     * @param  array    $rows       Array of rows
-     * @param  string   $filename   Output filename
-     */
-    public static function csv(array $rows, $filename = 'php://output')
-    {
-        $fp = fopen($filename, 'w');
-
-        if (is_resource($fp)) {
-            foreach ($rows as $fields) {
-                fputcsv($fp, $fields);
-            }
-
-            fclose($fp);
-        }
-    }
-
     /**
      * Get the mailbox hash from an email address
      *

app/Helper/Form.php

@@ -177,6 +177,23 @@ class Form extends \Core\Base
         return $html;
     }
 
+    /**
+     * Display file field
+     *
+     * @access public
+     * @param  string  $name
+     * @param  array   $errors
+     * @param  boolean $multiple
+     * @return string
+     */
+    public function file($name, array $errors = array(), $multiple = false)
+    {
+        $html = '<input type="file" name="'.$name.'" id="form-'.$name.'" '.($multiple ? 'multiple' : '').'>';
+        $html .= $this->errorList($errors, $name);
+
+        return $html;
+    }
+
     /**
      * Display a input field
      *

app/Model/Acl.php

@@ -88,6 +88,7 @@ class Acl extends Base
      */
     private $admin_acl = array(
         'user' => array('index', 'create', 'save', 'remove', 'authentication'),
+        'userimport' => '*',
         'config' => '*',
         'link' => '*',
         'currency' => '*',
@@ -117,6 +118,7 @@ class Acl extends Base
      */
     public function matchAcl(array $acl, $controller, $action)
     {
+        $controller = strtolower($controller);
         $action = strtolower($action);
         return isset($acl[$controller]) && $this->hasAction($action, $acl[$controller]);
     }

app/Model/UserImport.php

@@ -0,0 +1,110 @@
+<?php
+
+namespace Model;
+
+use SimpleValidator\Validator;
+use SimpleValidator\Validators;
+use Core\Csv;
+
+/**
+ * User Import
+ *
+ * @package  model
+ * @author   Frederic Guillot
+ */
+class UserImport extends Base
+{
+    /**
+     * Number of successful import
+     *
+     * @access public
+     * @var integer
+     */
+    public $counter = 0;
+
+    /**
+     * Get mapping between CSV header and SQL columns
+     *
+     * @access public
+     * @return array
+     */
+    public function getColumnMapping()
+    {
+        return array(
+            'username'         => 'Username',
+            'password'         => 'Password',
+            'email'            => 'Email',
+            'name'             => 'Full Name',
+            'is_admin'         => 'Administrator',
+            'is_project_admin' => 'Project Administrator',
+            'is_ldap_user'     => 'Remote User',
+        );
+    }
+
+    /**
+     * Import a single row
+     *
+     * @access public
+     * @param  array   $row
+     * @param  integer $line_number
+     */
+    public function import(array $row, $line_number)
+    {
+        $row = $this->prepare($row);
+
+        if ($this->validateCreation($row)) {
+            if ($this->user->create($row)) {
+                $this->logger->debug('UserImport: imported successfully line '.$line_number);
+                $this->counter++;
+            }
+            else {
+                $this->logger->error('UserImport: creation error at line '.$line_number);
+            }
+        }
+        else {
+            $this->logger->error('UserImport: validation error at line '.$line_number);
+        }
+    }
+
+    /**
+     * Format row before validation
+     *
+     * @access public
+     * @param  array   $data
+     * @return array
+     */
+    public function prepare(array $row)
+    {
+        $row['username'] = strtolower($row['username']);
+
+        foreach (array('is_admin', 'is_project_admin', 'is_ldap_user') as $field) {
+            $row[$field] = csv::getBooleanValue($row[$field]);
+        }
+
+        $this->removeEmptyFields($row, array('password', 'email', 'name'));
+
+        return $row;
+    }
+
+    /**
+     * Validate user creation
+     *
+     * @access public
+     * @param  array   $values
+     * @return boolean
+     */
+    public function validateCreation(array $values)
+    {
+        $v = new Validator($values, array(
+            new Validators\MaxLength('username', t('The maximum length is %d characters', 50), 50),
+            new Validators\Unique('username', t('The username must be unique'), $this->db->getConnection(), User::TABLE, 'id'),
+            new Validators\MinLength('password', t('The minimum length is %d characters', 6), 6),
+            new Validators\Email('email', t('Email address invalid')),
+            new Validators\Integer('is_admin', t('This value must be an integer')),
+            new Validators\Integer('is_project_admin', t('This value must be an integer')),
+            new Validators\Integer('is_ldap_user', t('This value must be an integer')),
+        ));
+
+        return $v->execute();
+    }
+}

app/ServiceProvider/ClassProvider.php

@@ -65,6 +65,7 @@ class ClassProvider implements ServiceProviderInterface
             'TaskValidator',
             'Transition',
             'User',
+            'UserImport',
             'UserSession',
             'Webhook',
         ),

app/Template/user/index.php

@@ -4,10 +4,10 @@
         <ul>
             <li><i class="fa fa-plus fa-fw"></i><?= $this->url->link(t('New local user'), 'user', 'create') ?></li>
             <li><i class="fa fa-plus fa-fw"></i><?= $this->url->link(t('New remote user'), 'user', 'create', array('remote' => 1)) ?></li>
+            <li><i class="fa fa-upload fa-fw"></i><?= $this->url->link(t('Import'), 'userImport', 'step1') ?></li>
         </ul>
         <?php endif ?>
     </div>
-    <section>
     <?php if ($paginator->isEmpty()): ?>
         <p class="alert"><?= t('No user') ?></p>
     <?php else: ?>
@@ -62,5 +62,4 @@
 
         <?= $paginator ?>
     <?php endif ?>
-    </section>
 </section>

app/Template/user_import/step1.php

@@ -0,0 +1,46 @@
+<section id="main">
+    <div class="page-header">
+        <?php if ($this->user->isAdmin()): ?>
+        <ul>
+            <li><i class="fa fa-user fa-fw"></i><?= $this->url->link(t('All users'), 'user', 'index') ?></li>
+            <li><i class="fa fa-plus fa-fw"></i><?= $this->url->link(t('New local user'), 'user', 'create') ?></li>
+            <li><i class="fa fa-plus fa-fw"></i><?= $this->url->link(t('New remote user'), 'user', 'create', array('remote' => 1)) ?></li>
+        </ul>
+        <?php endif ?>
+    </div>
+    <div class="page-header">
+        <h2><?= t('Import') ?></h2>
+    </div>
+    <form action="<?= $this->url->href('userImport', 'step2') ?>" method="post" enctype="multipart/form-data">
+        <?= $this->form->csrf() ?>
+
+        <?= $this->form->label(t('Delimiter'), 'delimiter') ?>
+        <?= $this->form->select('delimiter', $delimiters, $values) ?>
+
+        <?= $this->form->label(t('Enclosure'), 'enclosure') ?>
+        <?= $this->form->select('enclosure', $enclosures, $values) ?>
+
+        <?= $this->form->label(t('CSV File'), 'file') ?>
+        <?= $this->form->file('file', $errors) ?>
+
+        <p class="form-help"><?= t('Maximum size: ') ?><?= is_integer($max_size) ? $this->text->bytes($max_size) : $max_size ?></p>
+
+        <div class="form-actions">
+            <input type="submit" value="<?= t('Import') ?>" class="btn btn-blue">
+        </div>
+    </form>
+    <div class="page-header">
+        <h2><?= t('Instructions') ?></h2>
+    </div>
+    <div class="alert">
+        <ul>
+            <li><?= t('Your file must use the predefined CSV format') ?></li>
+            <li><?= t('Your file must be encoded in UTF-8') ?></li>
+            <li><?= t('The first row must be the header') ?></li>
+            <li><?= t('Duplicates are not imported') ?></li>
+            <li><?= t('Usernames must be lowercase and unique') ?></li>
+            <li><?= t('Passwords will be encrypted if present') ?></li>
+        </ul>
+    </div>
+    <p><i class="fa fa-download fa-fw"></i><?= $this->url->link(t('Download CSV template'), 'userImport', 'template') ?></p>
+</section>

tests/units/Core/CsvTest.php

@@ -0,0 +1,22 @@
+<?php
+
+require_once __DIR__.'/../Base.php';
+
+use Core\Csv;
+
+class CsvTest extends Base
+{
+    public function testGetBooleanValue()
+    {
+        $this->assertEquals(1, Csv::getBooleanValue('1'));
+        $this->assertEquals(1, Csv::getBooleanValue('True'));
+        $this->assertEquals(1, Csv::getBooleanValue('t'));
+        $this->assertEquals(1, Csv::getBooleanValue('TRUE'));
+        $this->assertEquals(1, Csv::getBooleanValue('true'));
+        $this->assertEquals(1, Csv::getBooleanValue('T'));
+
+        $this->assertEquals(0, Csv::getBooleanValue('0'));
+        $this->assertEquals(0, Csv::getBooleanValue('123'));
+        $this->assertEquals(0, Csv::getBooleanValue('anything'));
+    }
+}

tests/units/Core/RouterTest.php

@@ -10,11 +10,13 @@ class RouterTest extends Base
     {
         $r = new Router($this->container);
 
-        $this->assertEquals('plop', $r->sanitize('PloP', 'default'));
+        $this->assertEquals('PloP', $r->sanitize('PloP', 'default'));
         $this->assertEquals('default', $r->sanitize('', 'default'));
         $this->assertEquals('default', $r->sanitize('123-AB', 'default'));
         $this->assertEquals('default', $r->sanitize('R&D', 'default'));
-        $this->assertEquals('default', $r->sanitize('Test123', 'default'));
+        $this->assertEquals('Test123', $r->sanitize('Test123', 'default'));
+        $this->assertEquals('Test_123', $r->sanitize('Test_123', 'default'));
+        $this->assertEquals('userImport', $r->sanitize('userImport', 'default'));
     }
 
     public function testPath()

tests/units/Model/AclTest.php

@@ -17,6 +17,7 @@ class AclTest extends Base
             'controller3' => '*',
             'controller5' => '-',
             'controller6' => array(),
+            'controllera' => '*',
         );
 
         $acl = new Acl($this->container);
@@ -30,6 +31,8 @@ class AclTest extends Base
         $this->assertFalse($acl->matchAcl($acl_rules, 'controller4', 'anything'));
         $this->assertFalse($acl->matchAcl($acl_rules, 'controller5', 'anything'));
         $this->assertFalse($acl->matchAcl($acl_rules, 'controller6', 'anything'));
+        $this->assertTrue($acl->matchAcl($acl_rules, 'ControllerA', 'anything'));
+        $this->assertTrue($acl->matchAcl($acl_rules, 'controllera', 'anything'));
     }
 
     public function testPublicActions()