Rewrite email parser using ImapEngine, harden processing loop

Replace webklex/php-imap with directorytree/imapengine in the ticket
email parser. ImapEngine is pure PHP over sockets.

Parser improvements:
- Wrap per-message processing in try/catch so one malformed email
  can't abort the run; failures are flagged and logged with UID
- Query unseen + unflagged so previously-failed (flagged) messages
  are no longer re-processed on every cron run
- Skip vacation/auto-responder emails (RFC 3834) to prevent mail
  loops with the ticket auto-reply
- Cap messages per run (50) and attachment size (15MB); inline
  images over 2MB are stored as attachments instead of base64-embedded
  in ticket details
- Atomic lock file creation
- preg_quote() the ticket prefix in subject matching
- Dedupe CC watchers and exclude the sender
- Map webklex 'tls' encryption setting to STARTTLS for compatibility

NDR/DSN parsing now walks MIME parts via the underlying
zbateson parser instead of relying on attachment extraction.

plugins/vendor/zbateson/mail-mime-parser/src/Stream/HeaderStream.php

@@ -0,0 +1,101 @@
+<?php
+/**
+ * This file is part of the ZBateson\MailMimeParser project.
+ *
+ * @license http://opensource.org/licenses/bsd-license.php BSD
+ */
+
+namespace ZBateson\MailMimeParser\Stream;
+
+use ArrayIterator;
+use GuzzleHttp\Psr7;
+use Psr\Http\Message\StreamInterface;
+use SplObserver;
+use SplSubject;
+use Traversable;
+use ZBateson\MailMimeParser\Header\HeaderConsts;
+use ZBateson\MailMimeParser\Message\IMessagePart;
+use ZBateson\MailMimeParser\Message\IMimePart;
+
+/**
+ * Psr7 stream decorator implementation providing a readable stream for a part's
+ * headers.
+ *
+ * HeaderStream is only used by a MimePart parent.  It can accept any
+ * MessagePart - for non-MimeParts, only type headers are generated based on
+ * available information.
+ *
+ * @author Zaahid Bateson
+ */
+class HeaderStream extends MessagePartStreamDecorator implements SplObserver, StreamInterface
+{
+    /**
+     * @var IMessagePart the part to read from.
+     */
+    protected IMessagePart $part;
+
+    public function __construct(IMessagePart $part)
+    {
+        parent::__construct($part);
+        $part->attach($this);
+
+        // unsetting the property forces the first access to go through
+        // __get().
+        unset($this->stream);
+    }
+
+    public function __destruct()
+    {
+        $this->part->detach($this);
+    }
+
+    public function update(SplSubject $subject) : void
+    {
+        if ($this->stream !== null) {
+            $this->stream = $this->createStream();
+        }
+    }
+
+    /**
+     * Returns a header array for the current part.
+     *
+     * If the part is not a MimePart, Content-Type, Content-Disposition and
+     * Content-Transfer-Encoding headers are generated manually.
+     */
+    private function getPartHeadersIterator() : Traversable
+    {
+        if ($this->part instanceof IMimePart) {
+            return $this->part->getRawHeaderIterator();
+        } elseif ($this->part->getParent() !== null && $this->part->getParent()->isMime()) {
+            return new ArrayIterator([
+                [HeaderConsts::CONTENT_TYPE, $this->part->getContentType()],
+                [HeaderConsts::CONTENT_DISPOSITION, $this->part->getContentDisposition()],
+                [HeaderConsts::CONTENT_TRANSFER_ENCODING, $this->part->getContentTransferEncoding()]
+            ]);
+        }
+        return new ArrayIterator();
+    }
+
+    /**
+     * Writes out headers for $this->part and follows them with an empty line.
+     */
+    public function writePartHeadersTo(StreamInterface $stream) : static
+    {
+        foreach ($this->getPartHeadersIterator() as $header) {
+            $stream->write("{$header[0]}: {$header[1]}\r\n");
+        }
+        $stream->write("\r\n");
+        return $this;
+    }
+
+    /**
+     * Creates the underlying stream lazily when required.
+     */
+    protected function createStream() : StreamInterface
+    {
+        $stream = Psr7\Utils::streamFor();
+        $this->writePartHeadersTo($stream);
+        $stream->rewind();
+        return $stream;
+    }
+}

plugins/vendor/zbateson/mail-mime-parser/src/Stream/MessagePartStream.php

@@ -0,0 +1,188 @@
+<?php
+/**
+ * This file is part of the ZBateson\MailMimeParser project.
+ *
+ * @license http://opensource.org/licenses/bsd-license.php BSD
+ */
+
+namespace ZBateson\MailMimeParser\Stream;
+
+use GuzzleHttp\Psr7;
+use GuzzleHttp\Psr7\AppendStream;
+use Psr\Http\Message\StreamInterface;
+use SplObserver;
+use SplSubject;
+use ZBateson\MailMimeParser\Header\HeaderConsts;
+use ZBateson\MailMimeParser\MailMimeParser;
+use ZBateson\MailMimeParser\Message\IMessagePart;
+use ZBateson\MailMimeParser\Message\IMimePart;
+use ZBateson\MbWrapper\UnsupportedCharsetException;
+
+/**
+ * Provides a readable stream for a MessagePart.
+ *
+ * @author Zaahid Bateson
+ */
+class MessagePartStream extends MessagePartStreamDecorator implements SplObserver, StreamInterface
+{
+    /**
+     * @var StreamFactory For creating needed stream decorators.
+     */
+    protected StreamFactory $streamFactory;
+
+    /**
+     * @var IMessagePart The part to read from.
+     */
+    protected IMessagePart $part;
+
+    /**
+     * @var bool if false, saving a content stream with an unsupported charset
+     *      will be written in the default charset, otherwise the stream will be
+     *      created with the unsupported charset, and an exception will be
+     *      thrown when read from.
+     */
+    protected bool $throwExceptionReadingPartContentFromUnsupportedCharsets;
+
+    /**
+     * @var ?AppendStream
+     */
+    protected ?AppendStream $appendStream = null;
+
+    public function __construct(StreamFactory $sdf, IMessagePart $part, bool $throwExceptionReadingPartContentFromUnsupportedCharsets)
+    {
+        parent::__construct($part);
+        $this->streamFactory = $sdf;
+        $this->part = $part;
+        $this->throwExceptionReadingPartContentFromUnsupportedCharsets = $throwExceptionReadingPartContentFromUnsupportedCharsets;
+        $part->attach($this);
+
+        // unsetting the property forces the first access to go through
+        // __get().
+        unset($this->stream);
+    }
+
+    public function __destruct()
+    {
+        $this->part->detach($this);
+    }
+
+    public function update(SplSubject $subject) : void
+    {
+        if ($this->appendStream !== null) {
+            // unset forces recreation in StreamDecoratorTrait with a call to __get
+            unset($this->stream);
+            $this->appendStream = null;
+        }
+    }
+
+    /**
+     * Attaches and returns a CharsetStream decorator to the passed $stream.
+     *
+     * If the current attached IMessagePart doesn't specify a charset, $stream
+     * is returned as-is.
+     */
+    private function getCharsetDecoratorForStream(StreamInterface $stream) : StreamInterface
+    {
+        $charset = $this->part->getCharset();
+        if (!empty($charset)) {
+            if (!$this->throwExceptionReadingPartContentFromUnsupportedCharsets) {
+                $test = $this->streamFactory->newCharsetStream(
+                    Psr7\Utils::streamFor(),
+                    $charset,
+                    MailMimeParser::DEFAULT_CHARSET
+                );
+                try {
+                    $test->write('t');
+                } catch (UnsupportedCharsetException $e) {
+                    return $stream;
+                } finally {
+                    $test->close();
+                }
+            }
+            $stream = $this->streamFactory->newCharsetStream(
+                $stream,
+                $charset,
+                MailMimeParser::DEFAULT_CHARSET
+            );
+        }
+        return $stream;
+    }
+
+    /**
+     * Creates an array of streams based on the attached part's mime boundary
+     * and child streams.
+     *
+     * @param IMimePart $part passed in because $this->part is declared
+     *        as IMessagePart
+     * @return StreamInterface[]
+     */
+    protected function getBoundaryAndChildStreams(IMimePart $part) : array
+    {
+        $boundary = $part->getHeaderParameter(HeaderConsts::CONTENT_TYPE, 'boundary');
+        if ($boundary === null) {
+            return \array_map(
+                function($child) {
+                    return $child->getStream();
+                },
+                $part->getChildParts()
+            );
+        }
+        $streams = [];
+        foreach ($part->getChildParts() as $i => $child) {
+            if ($i !== 0 || $part->hasContent()) {
+                $streams[] = Psr7\Utils::streamFor("\r\n");
+            }
+            $streams[] = Psr7\Utils::streamFor("--$boundary\r\n");
+            $streams[] = $child->getStream();
+        }
+        $streams[] = Psr7\Utils::streamFor("\r\n--$boundary--\r\n");
+
+        return $streams;
+    }
+
+    /**
+     * Returns an array of Psr7 Streams representing the attached part and it's
+     * direct children.
+     *
+     * @return StreamInterface[]
+     */
+    protected function getStreamsArray() : array
+    {
+        $contentStream = $this->part->getContentStream();
+        if ($contentStream !== null) {
+            // wrapping in a SeekingLimitStream because the underlying
+            // ContentStream could be rewound, etc...
+            $contentStream = $this->streamFactory->newDecoratedCachingStream(
+                $this->streamFactory->newSeekingStream($contentStream),
+                function($stream) {
+                    $es = $this->streamFactory->getTransferEncodingDecoratedStream(
+                        $stream,
+                        $this->part->getContentTransferEncoding(),
+                        $this->part->getFilename()
+                    );
+                    $cs = $this->getCharsetDecoratorForStream($es);
+                    return $cs;
+                }
+            );
+        }
+
+        $streams = [$this->streamFactory->newHeaderStream($this->part), $contentStream ?: Psr7\Utils::streamFor()];
+
+        if ($this->part instanceof IMimePart && $this->part->getChildCount() > 0) {
+            $streams = \array_merge($streams, $this->getBoundaryAndChildStreams($this->part));
+        }
+
+        return $streams;
+    }
+
+    /**
+     * Creates the underlying stream lazily when required.
+     */
+    protected function createStream() : StreamInterface
+    {
+        if ($this->appendStream === null) {
+            $this->appendStream = new AppendStream($this->getStreamsArray());
+        }
+        return $this->appendStream;
+    }
+}

plugins/vendor/zbateson/mail-mime-parser/src/Stream/MessagePartStreamDecorator.php

@@ -0,0 +1,61 @@
+<?php
+/**
+ * This file is part of the ZBateson\MailMimeParser project.
+ *
+ * @license http://opensource.org/licenses/bsd-license.php BSD
+ */
+
+namespace ZBateson\MailMimeParser\Stream;
+
+use GuzzleHttp\Psr7\StreamDecoratorTrait;
+use Psr\Http\Message\StreamInterface;
+use RuntimeException;
+use ZBateson\MailMimeParser\Message\IMessagePart;
+
+/**
+ * Provides a readable stream for a MessagePart.
+ *
+ * @author Zaahid Bateson
+ */
+class MessagePartStreamDecorator implements StreamInterface
+{
+    use StreamDecoratorTrait {
+        read as private decoratorRead;
+    }
+
+    /**
+     * @var IMessagePart The part to read from.
+     */
+    protected IMessagePart $part;
+
+    protected ?StreamInterface $stream;
+
+    public function __construct(IMessagePart $part, ?StreamInterface $stream = null)
+    {
+        $this->part = $part;
+        $this->stream = $stream;
+    }
+
+    /**
+     * Overridden to wrap exceptions in MessagePartReadException which provides
+     * 'getPart' to inspect the part the error occurs on.
+     *
+     * @throws MessagePartStreamReadException
+     */
+    public function read(int $length) : string
+    {
+        try {
+            return $this->decoratorRead($length);
+        } catch (MessagePartStreamReadException $me) {
+            throw $me;
+        } catch (RuntimeException $e) {
+            throw new MessagePartStreamReadException(
+                $this->part,
+                'Exception occurred reading a part stream: cid=' . $this->part->getContentId()
+                . ' type=' . $this->part->getContentType() . ', message: ' . $e->getMessage(),
+                $e->getCode(),
+                $e
+            );
+        }
+    }
+}

plugins/vendor/zbateson/mail-mime-parser/src/Stream/MessagePartStreamReadException.php

@@ -0,0 +1,36 @@
+<?php
+/**
+ * This file is part of the ZBateson\MailMimeParser project.
+ *
+ * @license http://opensource.org/licenses/bsd-license.php BSD
+ */
+
+namespace ZBateson\MailMimeParser\Stream;
+
+use RuntimeException;
+use ZBateson\MailMimeParser\Message\IMessagePart;
+
+/**
+ * Thrown for exceptions on MessagePartStream::read so a $part can be used to
+ * determine where the exception occurred.
+ *
+ * @author Zaahid Bateson
+ */
+class MessagePartStreamReadException extends RuntimeException
+{
+    /**
+     * @var IMessagePart the IMessagePart the error was caused on.
+     */
+    protected IMessagePart $part;
+
+    public function __construct(IMessagePart $part, string $message = '', int $code = 0, ?\Throwable $previous = null)
+    {
+        parent::__construct($message, $code, $previous);
+        $this->part = $part;
+    }
+
+    public function getPart() : IMessagePart
+    {
+        return $this->part;
+    }
+}

plugins/vendor/zbateson/mail-mime-parser/src/Stream/StreamFactory.php

@@ -0,0 +1,198 @@
+<?php
+/**
+ * This file is part of the ZBateson\MailMimeParser project.
+ *
+ * @license http://opensource.org/licenses/bsd-license.php BSD
+ */
+
+namespace ZBateson\MailMimeParser\Stream;
+
+use Psr\Http\Message\StreamInterface;
+use ZBateson\MailMimeParser\Message\IMessagePart;
+use ZBateson\MailMimeParser\Parser\PartBuilder;
+use ZBateson\StreamDecorators\Base64Stream;
+use ZBateson\StreamDecorators\CharsetStream;
+use ZBateson\StreamDecorators\ChunkSplitStream;
+use ZBateson\StreamDecorators\DecoratedCachingStream;
+use ZBateson\StreamDecorators\NonClosingStream;
+use ZBateson\StreamDecorators\PregReplaceFilterStream;
+use ZBateson\StreamDecorators\QuotedPrintableStream;
+use ZBateson\StreamDecorators\SeekingLimitStream;
+use ZBateson\StreamDecorators\UUStream;
+
+/**
+ * Factory class for Psr7 stream decorators used in MailMimeParser.
+ *
+ * @author Zaahid Bateson
+ */
+class StreamFactory
+{
+    /**
+     * @var bool if true, saving a content stream with an unsupported charset
+     *      will be written in the default charset.
+     */
+    protected bool $throwExceptionReadingPartContentFromUnsupportedCharsets;
+
+    public function __construct(bool $throwExceptionReadingPartContentFromUnsupportedCharsets)
+    {
+        $this->throwExceptionReadingPartContentFromUnsupportedCharsets = $throwExceptionReadingPartContentFromUnsupportedCharsets;
+    }
+
+    /**
+     * Returns a SeekingLimitStream using $part->getStreamPartLength() and
+     * $part->getStreamPartStartPos()
+     */
+    public function getLimitedPartStream(PartBuilder $part) : StreamInterface
+    {
+        return $this->newLimitStream(
+            $part->getStream(),
+            $part->getStreamPartLength(),
+            $part->getStreamPartStartPos()
+        );
+    }
+
+    /**
+     * Returns a SeekingLimitStream using $part->getStreamContentLength() and
+     * $part->getStreamContentStartPos()
+     */
+    public function getLimitedContentStream(PartBuilder $part) : ?StreamInterface
+    {
+        $length = $part->getStreamContentLength();
+        if ($length !== 0) {
+            return $this->newLimitStream(
+                $part->getStream(),
+                $part->getStreamContentLength(),
+                $part->getStreamContentStartPos()
+            );
+        }
+        return null;
+    }
+
+    /**
+     * Creates and returns a SeekingLimitedStream.
+     */
+    private function newLimitStream(StreamInterface $stream, int $length, int $start) : StreamInterface
+    {
+        return new SeekingLimitStream(
+            $this->newNonClosingStream($stream),
+            $length,
+            $start
+        );
+    }
+
+    /**
+     * Creates and returns a SeekingLimitedStream without limits, so it's a
+     * stream that preserves its current position on the underlying stream it
+     * reads from.
+     */
+    public function newSeekingStream(StreamInterface $stream) : StreamInterface
+    {
+        return new SeekingLimitStream($this->newNonClosingStream($stream));
+    }
+
+    /**
+     * Creates a non-closing stream that doesn't close it's internal stream when
+     * closing/detaching.
+     */
+    public function newNonClosingStream(StreamInterface $stream) : StreamInterface
+    {
+        return new NonClosingStream($stream);
+    }
+
+    /**
+     * Creates a ChunkSplitStream.
+     */
+    public function newChunkSplitStream(StreamInterface $stream) : StreamInterface
+    {
+        return new ChunkSplitStream($stream);
+    }
+
+    /**
+     * Creates and returns a Base64Stream with an internal
+     * PregReplaceFilterStream that filters out non-base64 characters.
+     */
+    public function newBase64Stream(StreamInterface $stream) : StreamInterface
+    {
+        return new Base64Stream(
+            new PregReplaceFilterStream($stream, '/[^a-zA-Z0-9\/\+=]/', '')
+        );
+    }
+
+    /**
+     * Creates and returns a QuotedPrintableStream.
+     */
+    public function newQuotedPrintableStream(StreamInterface $stream) : StreamInterface
+    {
+        return new QuotedPrintableStream($stream);
+    }
+
+    /**
+     * Creates and returns a UUStream
+     */
+    public function newUUStream(StreamInterface $stream) : StreamInterface
+    {
+        return new UUStream($stream);
+    }
+
+    public function getTransferEncodingDecoratedStream(StreamInterface $stream, ?string $transferEncoding, ?string $filename = null) : StreamInterface
+    {
+        $decorated = null;
+        switch ($transferEncoding) {
+            case 'quoted-printable':
+                $decorated = $this->newQuotedPrintableStream($stream);
+                break;
+            case 'base64':
+                $decorated = $this->newBase64Stream(
+                    $this->newChunkSplitStream($stream)
+                );
+                break;
+            case 'x-uuencode':
+                $decorated = $this->newUUStream($stream);
+                if ($filename !== null) {
+                    $decorated->setFilename($filename);
+                }
+                break;
+            default:
+                return $stream;
+        }
+        return $decorated;
+    }
+
+    /**
+     * Creates and returns a CharsetStream
+     */
+    public function newCharsetStream(StreamInterface $stream, string $streamCharset, string $stringCharset) : StreamInterface
+    {
+        return new CharsetStream($stream, $streamCharset, $stringCharset);
+    }
+
+    /**
+     * Creates and returns a MessagePartStream
+     */
+    public function newMessagePartStream(IMessagePart $part) : MessagePartStreamDecorator
+    {
+        return new MessagePartStream($this, $part, $this->throwExceptionReadingPartContentFromUnsupportedCharsets);
+    }
+
+    /**
+     * Creates and returns a DecoratedCachingStream
+     */
+    public function newDecoratedCachingStream(StreamInterface $stream, callable $decorator) : StreamInterface
+    {
+        // seems to perform best locally, would be good to test this out more
+        return new DecoratedCachingStream($stream, $decorator, 204800);
+    }
+
+    /**
+     * Creates and returns a HeaderStream
+     */
+    public function newHeaderStream(IMessagePart $part) : StreamInterface
+    {
+        return new HeaderStream($part);
+    }
+
+    public function newDecoratedMessagePartStream(IMessagePart $part, StreamInterface $stream) : MessagePartStreamDecorator
+    {
+        return new MessagePartStreamDecorator($part, $stream);
+    }
+}