Merge pull request #8 from jubbs/StatusEvents

Status events and Update to Cakephp5

Author: narendravaghela

README.md

@@ -8,14 +8,14 @@
 
 This plugin provides email delivery using [SendGrid](https://sendgrid.com/).
 
-This branch is for use with CakePHP 4.0+. For CakePHP 3, please use cake-3.x branch.
+This branch is for use with CakePHP 5.0+. For CakePHP 4, please use cake-4.x branch.
 
 ## Requirements
 
 This plugin has the following requirements:
 
-* CakePHP 4.0 or greater.
-* PHP 7.2 or greater.
+* CakePHP 5.0 or greater.
+* PHP 8.1 or greater.
 
 ## Installation
 
@@ -137,6 +137,69 @@ $email->setTo('[email protected]')
     ->setSendAt(1649500630)
     ->deliver();
 ```
+## Webhooks
+You can receive status events from SendGrid. This allows you ensure that SendGrid was able to send the email recording bounces etc. 
+
+### Webhook Config
+You will require a Table in the database to record the emails sent. You can use the lorenzo/cakephp-email-queue plugin to queue the emails and in that case you would 
+use the email_queue table. However you can create your own table/Model as long as it has at least three columns. They can be called anything but they must have the correct types.
+
+When you send the email the deliver function will return an array with a 'messageId' element if it successfully connected to SendGrid. This needs to be recorded in the status_id field.
+
+* status_id VARCHAR(100)
+* status VARCHAR(100)
+* status_message TEXT
+
+You need to map this table and these fields in you app_local.php config
+
+```php
+
+    'sendgridWebhook' => [
+        'tableClass' => 'EmailQueue', // The table name that stores email data
+        'uniqueIdField' => 'status_id', // The field name that stores the unique message ID VARCHAR(100)
+        'statusField' => 'status', // The field name that stores the status of the email status VARCHAR(100)
+        'statusMessageField' => 'status_message', // The field name that stores the status messages TEXT
+        'debug' => 'true', // write incoming requests to debug log
+        'secure' => 'true', // enable SendGrid signed webhook security. You should enable this in production
+        'verification_key' => '<YOUR VERIFICATION KEY>', // The verification key from SendGrid
+    ],
+
+```
+
+You will need to login to your SendGrid Account and configure your domain and the events that you want to track
+
+ https://app.sendgrid.com/settings/mail_settings/webhook_settings
+
+The return url needs to be set to 
+* https://YOUR DOMAIN/send-grid/webhook
+
+
+The CSRF protection middleware needs to allow posts to the webhooks controller in Application.php
+Remove the current CSRF protection middleware and replace it with the following. If you already have CSRF exceptions then add the Webhooks one
+  
+  ```php
+    $csrf = new CsrfProtectionMiddleware();
+
+    $csrf->skipCheckCallback(function ($request) {
+           // Skip token check for API URLs.
+          if ($request->getParam('controller') === 'Webhooks') {
+             return true;
+            }
+    });
+ 
+      // Ensure routing middleware is added to the queue before CSRF protection middleware.
+    $middlewareQueue->add($csrf);
+ 
+    return $middlewareQueue;
+  
+  ```
+
+If the authentication plugin (https://book.cakephp.org/authentication/3/en/index.html) is used for authentication the webhook action should work OK. If you have a different authentication method then you will need to add an exception for the webhook action. /send-grid/webhooks/index 
+
+#### Webhook Signature Verification
+SendGrid allows you to sign the webhook requests. This is a good idea in production to keep the webhook secure. You will need to enable this in your SendGrid account and then set secure to true and add your verification key to your app_local.php config file.
+
+https://docs.sendgrid.com/for-developers/tracking-events/getting-started-event-webhook-security-features. Enable signed event webhook and follow the instructions to get the verification key.
 
 ## Reporting Issues
 

composer.json

@@ -29,8 +29,8 @@
         "issues": "https://github.com/sprintcube/cakephp-sendgrid/issues"
     },
     "require": {
-        "php": ">=7.2",
-        "cakephp/cakephp": "^4.0.0"
+        "php": ">=8.1",
+        "cakephp/cakephp": "^5.0.0"
     },
     "require-dev": {
         "phpunit/phpunit": "^8.5 || ^9.3",

src/Controller/AppController.php

@@ -0,0 +1,10 @@
+<?php
+declare(strict_types=1);
+
+namespace SendGrid\Controller;
+
+use App\Controller\AppController as BaseController;
+
+class AppController extends BaseController
+{
+}

src/Controller/WebhooksController.php

@@ -0,0 +1,130 @@
+<?php
+
+declare(strict_types=1);
+
+// Docs
+//https://docs.sendgrid.com/for-developers/tracking-events/event
+//Config
+//https://app.sendgrid.com/settings/mail_settings
+//Key Verification
+//https://docs.sendgrid.com/for-developers/tracking-events/event#security-features
+
+/**
+ * test 
+ * curl -X POST http://localhost:8765/send-grid/webhooks -H 'Content-Type: application/json' -d '[{"timestamp": 1700762652,  "event": "processed", "sg_message_id": "14c5d75ce93.dfd.64b469.filter0001.16648.5515E0B88.0"}]'
+ *
+ * security test 
+ * curl -X POST http://localhost:8765/send-grid/webhooks -H 'Content-Type: application/json' -H 'X-Twilio-Email-Event-Webhook-Signature: MFk..........2mg==' -d '[{"timestamp": 1700762652,  "event": "processed", "sg_message_id": "14c5d75ce93.dfd.64b469.filter0001.16648.5515E0B88.0"}]'
+ * 
+ * 
+ */
+
+
+namespace SendGrid\Controller;
+
+use SendGrid\Controller\AppController;
+
+use Cake\View\JsonView;
+use Cake\Core\Configure;
+use Cake\I18n\DateTime;
+use Cake\Event\EventInterface;
+use Cake\Log\Log;
+use SendGrid\Util\EllipticCurve\Ecdsa;
+use SendGrid\Util\EllipticCurve\PublicKey;
+use SendGrid\Util\EllipticCurve\Signature;
+
+class WebHooksController extends AppController
+{
+
+    const SIGNATURE = "X-Twilio-Email-Event-Webhook-Signature";
+    const TIMESTAMP = "X-Twilio-Email-Event-Webhook-Timestamp";
+    /**
+     * beforeFilter callback.
+     *
+     * @param \Cake\Event\EventInterface<\Cake\Controller\Controller> $event Event.
+     * @return \Cake\Http\Response|null|void
+     */
+    public function beforeFilter(EventInterface $event)
+    {
+        // If the authentication plugin is loaded then open up the index action
+        if (isset($this->Authentication)) {
+            $this->Authentication->allowUnauthenticated(['index']);
+        }
+    }
+
+    public function viewClasses(): array
+    {
+        return [JsonView::class];
+    }
+
+
+    public function index()
+    {
+        $this->request->allowMethod(['post']);
+        $this->viewBuilder()->setClassName("Json");
+        $result = $this->request->getData();
+
+        $this->set('result', $result);
+        $config = Configure::read('sendgridWebhook');
+
+        if (isset($config['debug']) && $config['debug'] == 'true') {
+            Log::debug(json_encode($result));
+            Log::debug(json_encode($this->request->getHeaders()));
+        }
+
+        if (isset($config['secure']) && $config['secure'] == 'true') {
+            $this->request->getBody()->rewind();
+            $payload = $this->request->getBody()->getContents();
+           // Log::debug($payload);
+
+            if (!isset($config['verification_key'])) {
+                if (isset($config['debug']) && $config['debug'] == 'true') {
+                    Log::debug("Verfication Failed: Webhook Signature verification key not set in app_local.php");
+                }
+                $this->set('error', "Invalid Signature");
+                $this->viewBuilder()->setOption('serialize', "error");
+                return;
+            }
+            
+            $publicKey = PublicKey::fromString($config['verification_key']);
+
+            $timestampedPayload = $this->request->getHeaderLine($this::TIMESTAMP) . $payload;
+            $decodedSignature = Signature::fromBase64($this->request->getHeaderLine($this::SIGNATURE));
+
+            if (!Ecdsa::verify($timestampedPayload, $decodedSignature, $publicKey)) {
+                if (isset($config['debug']) && $config['debug'] == 'true') {
+                    Log::debug("Verfication Failed: Webhook Signature does not verify against the verification key");
+                }
+                $this->set('error', "Invalid Signature");
+                $this->viewBuilder()->setOption('serialize', "error");
+                return;
+            }
+        }
+
+        $emailTable = $this->getTableLocator()->get($config['tableClass']);
+        $count = 0;
+        foreach ($result as $event) {
+            $message_id = explode(".", $event['sg_message_id'])[0];
+            $email_record = $emailTable->find('all')->select([
+                "id",
+                $config['uniqueIdField'],
+                $config['statusField'],
+                $config['statusMessageField'],
+            ])->where([$config['uniqueIdField'] => $message_id])->first();
+            if ($email_record) {
+                $count++;
+                $email_record->set($config['statusMessageField'], $email_record->get($config['statusMessageField']) . (new DateTime())->format('d/m/Y H:i:s') . ": " . $event['event'] . " " .
+                    (isset($event['response']) ? $event['response'] : "") . " " .
+                    (isset($event['reason']) ? $event['reason'] : "<br>"));
+                $email_record->set($config['statusField'], $event['event']);
+                $emailTable->save($email_record);
+            }
+        }
+        if (isset($config['debug']) && $config['debug'] == 'true') {
+            Log::debug("Updated $count Email records");
+        }
+        $this->set('OK', "OK");
+        $this->viewBuilder()->setOption('serialize', "OK");
+    }
+
+}

src/Mailer/Exception/SendGridApiException.php

@@ -1,6 +1,7 @@
 <?php
 declare(strict_types=1);
 
+use Cake\Core\Exception\Exception;
 /**
  * SendGrid Plugin for CakePHP
  * Copyright (c) SprintCube (https://www.sprintcube.com)
@@ -17,7 +18,7 @@
 
 namespace SendGrid\Mailer\Exception;
 
-use Cake\Core\Exception\Exception;
+use Exception;
 
 /**
  * SendGrid Api exception

src/Mailer/Transport/SendGridTransport.php

@@ -1,4 +1,5 @@
 <?php
+
 declare(strict_types=1);
 
 /**
@@ -33,9 +34,17 @@ class SendGridTransport extends AbstractTransport
      *
      * @var array
      */
-    protected $_defaultConfig = [
+    protected array $_defaultConfig = [
         'apiEndpoint' => 'https://api.sendgrid.com/v3',
         'apiKey' => '',
+        'enableWebhooks' => false, // If you want to use webhooks to monitor delivery ane error reports
+        'webhookConfig' => [
+            'domain' => 'example.com', // The domain used for Message IDs does not need to be 
+            'tableName' => 'email_queue', // The table name that stores email data
+            'uniqueIdField' => 'id', // The field name that stores the unique message ID char(36) uid
+            'statusField' => 'status', // The field name that stores the status of the email char(50) status
+            'statusMessageField' => 'status_message', // The field name that stores the status message TEXT status_message
+        ]
     ];
 
     /**
@@ -110,19 +119,21 @@ public function send(Message $message): array
         $this->_reqParams['subject'] = $message->getSubject();
 
         $emailFormat = $message->getEmailFormat();
-        if (!empty($message->getBodyHtml())) {
-            $this->_reqParams['content'][] = (object)[
-                'type' => 'text/html',
-                'value' => trim($message->getBodyHtml()),
-            ];
-        }
+
         if ($emailFormat == 'both' || $emailFormat == 'text') {
             $this->_reqParams['content'][] = (object)[
                 'type' => 'text/plain',
                 'value' => trim($message->getBodyText()),
             ];
         }
 
+        if (!empty($message->getBodyHtml())) {
+            $this->_reqParams['content'][] = (object)[
+                'type' => 'text/html',
+                'value' => trim($message->getBodyHtml()),
+            ];
+        }
+
         $this->_processHeaders($message);
 
         $this->_processAttachments($message);
@@ -178,6 +189,16 @@ protected function _prepareEmailAddresses(Message $message)
             $this->_reqParams['from'] = (object)['email' => key($from)];
         }
 
+        $replyTo = $message->getReplyTo();
+        if (!empty($replyTo)) {
+            if (key($replyTo) != $replyTo[key($replyTo)]) {
+                $this->_reqParams['reply_to'] = (object)['email' => key($replyTo), 'name' => $replyTo[key($replyTo)]];
+                
+            } else {
+                $this->_reqParams['reply_to'] = (object)['email' => key($replyTo)];
+            }        
+        }
+
         $emails = [];
         foreach ($message->getTo() as $toEmail => $toName) {
             $emails['to'][] = [
@@ -253,6 +274,7 @@ protected function _processAttachments(Message $message)
      */
     protected function _sendEmail()
     {
+
         $options = [
             'type' => 'json',
             'headers' => [
@@ -261,12 +283,12 @@ protected function _sendEmail()
             ],
         ];
 
-        $response = $this->Client
-            ->post("{$this->getConfig('apiEndpoint')}/mail/send", json_encode($this->_reqParams), $options);
+        $response = $this->Client->post("{$this->getConfig('apiEndpoint')}/mail/send", json_encode($this->_reqParams), $options);
 
         $result = [];
         $result['apiResponse'] = $response->getJson();
         $result['responseCode'] = $response->getStatusCode();
+        $result['messageId'] = $response->getHeader('X-Message-Id') ?? '';
         $result['status'] = $result['responseCode'] == 202 ? 'OK' : 'ERROR';
         if (Configure::read('debug')) {
             $result['reqParams'] = $this->_reqParams;