Improve background workers

app/Core/Http/Client.php

@@ -3,6 +3,7 @@
 namespace Kanboard\Core\Http;
 
 use Kanboard\Core\Base;
+use Kanboard\Job\HttpAsyncJob;
 
 /**
  * HTTP client
@@ -79,6 +80,24 @@ class Client extends Base
         );
     }
 
+    /**
+     * Send a POST HTTP request encoded in JSON (Fire and forget)
+     *
+     * @access public
+     * @param  string     $url
+     * @param  array      $data
+     * @param  string[]   $headers
+     */
+    public function postJsonAsync($url, array $data, array $headers = array())
+    {
+        $this->queueManager->push(HttpAsyncJob::getInstance($this->container)->withParams(
+            'POST',
+            $url,
+            json_encode($data),
+            array_merge(array('Content-type: application/json'), $headers)
+        ));
+    }
+
     /**
      * Send a POST HTTP request encoded in www-form-urlencoded
      *
@@ -98,22 +117,41 @@ class Client extends Base
         );
     }
 
+    /**
+     * Send a POST HTTP request encoded in www-form-urlencoded (fire and forget)
+     *
+     * @access public
+     * @param  string     $url
+     * @param  array      $data
+     * @param  string[]   $headers
+     */
+    public function postFormAsync($url, array $data, array $headers = array())
+    {
+        $this->queueManager->push(HttpAsyncJob::getInstance($this->container)->withParams(
+            'POST',
+            $url,
+            http_build_query($data),
+            array_merge(array('Content-type: application/x-www-form-urlencoded'), $headers)
+        ));
+    }
+
     /**
      * Make the HTTP request
      *
-     * @access private
+     * @access public
      * @param  string     $method
      * @param  string     $url
      * @param  string     $content
      * @param  string[]   $headers
      * @return string
      */
-    private function doRequest($method, $url, $content, array $headers)
+    public function doRequest($method, $url, $content, array $headers)
     {
         if (empty($url)) {
             return '';
         }
 
+        $startTime = microtime(true);
         $stream = @fopen(trim($url), 'r', false, stream_context_create($this->getContext($method, $content, $headers)));
         $response = '';
 
@@ -128,6 +166,7 @@ class Client extends Base
             $this->logger->debug('HttpClient: payload='.$content);
             $this->logger->debug('HttpClient: metadata='.var_export(@stream_get_meta_data($stream), true));
             $this->logger->debug('HttpClient: response='.$response);
+            $this->logger->debug('HttpClient: executionTime='.(microtime(true) - $startTime));
         }
 
         return $response;

app/Core/Queue/JobHandler.php

@@ -26,6 +26,7 @@ class JobHandler extends Base
         return new Job(array(
             'class' => get_class($job),
             'params' => $job->getJobParams(),
+            'user_id' => $this->userSession->getId(),
         ));
     }
 
@@ -39,12 +40,30 @@ class JobHandler extends Base
     {
         $payload = $job->getBody();
         $className = $payload['class'];
+        $this->prepareJobSession($payload['user_id']);
 
         if (DEBUG) {
-            $this->logger->debug(__METHOD__.' Received job => '.$className);
+            $this->logger->debug(__METHOD__.' Received job => '.$className.' ('.getmypid().')');
         }
 
         $worker = new $className($this->container);
         call_user_func_array(array($worker, 'execute'), $payload['params']);
     }
+
+    /**
+     * Create the session for the job
+     *
+     * @access protected
+     * @param integer $user_id
+     */
+    protected function prepareJobSession($user_id)
+    {
+        $session = array();
+        $this->sessionStorage->setStorage($session);
+
+        if ($user_id > 0) {
+            $user = $this->userModel->getById($user_id);
+            $this->userSession->initialize($user);
+        }
+    }
 }

app/Job/HttpAsyncJob.php

@@ -0,0 +1,43 @@
+<?php
+
+namespace Kanboard\Job;
+
+/**
+ * Async HTTP Client (fire and forget)
+ *
+ * @package Kanboard\Job
+ * @author  Frederic Guillot
+ */
+class HttpAsyncJob extends BaseJob
+{
+    /**
+     * Set job parameters
+     *
+     * @access public
+     * @param string $method
+     * @param string $url
+     * @param string $content
+     * @param array  $headers
+     * @return $this
+     */
+    public function withParams($method, $url, $content, array $headers)
+    {
+        $this->jobParams = array($method, $url, $content, $headers);
+        return $this;
+    }
+
+    /**
+     * Set job parameters
+     *
+     * @access public
+     * @param string $method
+     * @param string $url
+     * @param string $content
+     * @param array  $headers
+     * @return $this
+     */
+    public function execute($method, $url, $content, array $headers)
+    {
+        $this->httpClient->doRequest($method, $url, $content, $headers);
+    }
+}

doc/cli.markdown

@@ -31,6 +31,7 @@ Available commands:
   cronjob                            Execute daily cronjob
   help                               Displays help for a command
   list                               Lists commands
+  worker                             Execute queue worker
  export
   export:daily-project-column-stats  Daily project column stats CSV export (number of tasks per column and per day)
   export:subtasks                    Subtasks CSV export
@@ -196,3 +197,9 @@ Note: Installed files will have the same permissions as the current user
 * Updating plugin: Budget Planning
 * Plugin up to date: Github Authentication
 ```
+
+### Run Background worker
+
+```bash
+./kanboard worker
+```

doc/index.markdown

@@ -105,7 +105,9 @@ Technical details
 
 ### Configuration
 
+- [Performances](performances.markdown)
 - [Daily background job](cronjob.markdown)
+- [Background Worker](worker.markdown)
 - [Config file](config.markdown)
 - [Environment variables](env.markdown)
 - [Email configuration](email-configuration.markdown)

doc/installation.markdown

@@ -53,13 +53,14 @@ The `.htaccess` is optional because its content can be included directly in the
 
 You can also define a custom location for the plugins and files folders by changing the [config file](config.markdown).
 
+Optional installation
+---------------------
+
+- Some features of Kanboard require that you run [a daily background job](cronjob.markdown) (Reports and analytics)
+- [Install the background worker](worker.markdown) to improve the performances
+
 Security
 --------
 
 - Don't forget to change the default user/password
-- Don't allow everybody to access to the directory `data` from the URL. There is already a `.htaccess` for Apache but nothing for Nginx.
+- Don't allow everybody to access to the directory `data` from the URL. There is already a `.htaccess` for Apache but nothing for other web servers.
-
-Notes
------
-
-- Some features of Kanboard require that you run [a daily background job](cronjob.markdown)

doc/performances.markdown

@@ -0,0 +1,39 @@
+Kanboard Performances
+=====================
+
+According to your configuration, some features can slow down the usage of Kanboard.
+By default, all operations are synchronous and performed in the same thread as the HTTP request.
+This is a PHP limitation.
+However, it's possible to improve that.
+
+Depending on the plugins you install, communicating to external services can take hundred of milliseconds or even seconds.
+To avoid blocking the main thread, it's possible to delegate these operations to a pool of [background workers](worker.markdown).
+This setup require that you install additional software in your infrastructure.
+
+How to detect the bottleneck?
+-----------------------------
+
+- Enable the debug mode
+- Monitor the log file
+- Do something in Kanboard (drag and drop a task for example)
+- All operations are logged with the execution time (HTTP requests, Email notifications, SQL requests)
+
+Improve Email notifications speed
+---------------------------------
+
+Using the SMTP method with an external server can be very slow.
+
+Possible solutions:
+
+- Use the background workers if you still want to use SMTP
+- Use a local email relay with Postfix and use the "mail" transport
+- Use an email provider that use an HTTP API to send emails (Sendgrid, Mailgun or Postmark)
+
+Improve Sqlite performances
+---------------------------
+
+Possible solutions:
+
+- Do not use Sqlite when you have a lot of concurrency (several users), choose Postgres or Mysql instead
+- Do not use Sqlite on a shared NFS mount
+- Do not use Sqlite on a disk with poor IOPS, it's always preferable to use local SSD drives

doc/worker.markdown

@@ -0,0 +1,35 @@
+Background Workers
+==================
+
+**This feature is experimental**.
+
+Depending on your configuration, some features can slow down the application if they are executed in the same process as the HTTP request.
+Kanboard can delegate these tasks to a background worker that listen for incoming events.
+
+Example of feature that may slow down Kanboard:
+
+- Sending emails via an external SMTP server can take several seconds
+- Sending notifications to external services
+
+This feature is optional and require the installation of a queue daemon on your server.
+
+### Beanstalk
+
+[Beanstalk](http://kr.github.io/beanstalkd/) is a simple, fast work queue.
+
+- To install Beanstalk, you can simply use the package manager of your Linux distribution
+- Install the [Kanboard plugin for Beanstalk](https://kanboard.net/plugin/beanstalk)
+- Start the worker with the Kanboard command line tool: `./kanboard worker`
+
+### RabbitMQ
+
+[RabbitMQ](https://www.rabbitmq.com/) is a robust messaging system that is more suitable for high-availability infrastructure.
+
+- Follow the official documentation of RabbitMQ for the installation and the configuration
+- Install the [Kanboard plugin for RabbitMQ](https://kanboard.net/plugin/rabbitmq)
+- Start the worker with the Kanboard command line tool: `./kanboard worker`
+
+### Notes
+
+- You should start the Kanboard worker with a process supervisor (systemd, upstart or supervisord)
+- The process must be have access to the data folder if you store files on the local filesystem and have Sqlite