Ability for messages to auto-detect if they need to be Unicode

mheap · mheap · commit 610fba3a8a8a · 2017-08-28T10:59:43.000+01:00
This is an initial attempt at solving unicode auto-detection. It is
*disabled* by default and needs to be enabled explicitly by calling
`$message-&gt;enableEncodingDetection()`
diff --git a/src/Entity/RequestArrayTrait.php b/src/Entity/RequestArrayTrait.php
@@ -40,6 +40,13 @@ public function getRequestData($sent = true)
             return $query;
         }
 
+        // Trigger a pre-getRequestData() hook for any last minute
+        // decision making that needs to be done, but only if
+        // it hasn't been sent already
+        if (method_exists($this, 'preGetRequestDataHook')) {
+            $this->preGetRequestDataHook();
+        }
+
         return $this->requestData;
     }    
     
diff --git a/src/Message/AutoDetect.php b/src/Message/AutoDetect.php
@@ -0,0 +1,38 @@
+<?php
+/**
+ * Nexmo Client Library for PHP
+ *
+ * @copyright Copyright (c) 2016 Nexmo, Inc. (http://nexmo.com)
+ * @license   https://github.com/Nexmo/nexmo-php/blob/master/LICENSE.txt MIT License
+ */
+
+namespace Nexmo\Message;
+
+/**
+ * SMS Text Message
+ */
+class AutoDetect extends Message
+{
+    const TYPE = 'text';
+    
+    /**
+     * Message Body
+     * @var string
+     */
+    protected $text;
+    
+    /**
+     * Create a new SMS text message.
+     * 
+     * @param string $to
+     * @param string $from
+     * @param string $text
+     * @param array  $additional
+     */
+    public function __construct($to, $from, $text, $additional = [])
+    {
+        parent::__construct($to, $from, $additional);
+        $this->enableEncodingDetection();
+        $this->requestData['text'] = (string) $text;
+    }
+}
diff --git a/src/Message/EncodingDetector.php b/src/Message/EncodingDetector.php
@@ -0,0 +1,48 @@
+<?php
+/**
+ * Nexmo Client Library for PHP
+ *
+ * @copyright Copyright (c) 2016 Nexmo, Inc. (http://nexmo.com)
+ * @license   https://github.com/Nexmo/nexmo-php/blob/master/LICENSE.txt MIT License
+ */
+
+namespace Nexmo\Message;
+
+class EncodingDetector {
+
+    public function requiresUnicodeEncoding($content)
+    {
+
+        $gsmCodePoints = [
+            0x0040, 0x00A3, 0x0024, 0x00A5, 0x00E8, 0x00E9, 0x00F9, 0x00EC, 0x00F2, 0x00E7, 0x000A, 0x00D8, 0x00F8, 0x000D, 0x00C5, 0x00E5, 0x0394,
+            0x005F, 0x03A6, 0x0393, 0x039B, 0x03A9, 0x03A0, 0x03A8, 0x03A3, 0x0398, 0x039E, 0x00A0, 0x000C, 0x005E, 0x007B, 0x007D, 0x005C, 0x005B,
+            0x007E, 0x005D, 0x007C, 0x20AC, 0x00C6, 0x00E6, 0x00DF, 0x00C9, 0x0020, 0x0021, 0x0022, 0x0023, 0x00A4, 0x0025, 0x0026, 0x0027, 0x0028,
+            0x0029, 0x002A, 0x002B, 0x002C, 0x002D, 0x002E, 0x002F, 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039,
+            0x003A, 0x003B, 0x003C, 0x003D, 0x003E, 0x003F, 0x00A1, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004A,
+            0x004B, 0x004C, 0x004D, 0x004E, 0x004F, 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, 0x005A, 0x00C4,
+            0x00D6, 0x00D1, 0x00DC, 0x00A7, 0x00BF, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006A, 0x006B, 0x006C,
+            0x006D, 0x006E, 0x006F, 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007A, 0x00E4, 0x00F6, 0x00F1,
+            0x00FC, 0x00E0
+        ];
+
+        // Split $text into an array in a way that respects multibyte characters.
+        $textChars = preg_split('//u', $content, null, PREG_SPLIT_NO_EMPTY);
+
+        // Array of codepoint values for characters in $text.
+        $textCodePoints = array_map(function ($char) {
+            $k = mb_convert_encoding($char, 'UTF-16LE', 'UTF-8');
+            $k1 = ord(substr($k, 0, 1));
+            $k2 = ord(substr($k, 1, 1));
+            return $k2 * 256 + $k1;
+        }, $textChars);
+
+        // Filter the array to contain only codepoints from $text that are not in the set of valid GSM codepoints.
+        $nonGsmCodePoints = array_diff($textCodePoints, $gsmCodePoints);
+
+        // The text contains unicode if the result is not empty.
+        return !empty($nonGsmCodePoints);
+    }
+
+
+}
+
diff --git a/src/Message/Message.php b/src/Message/Message.php
@@ -7,6 +7,7 @@
  */
 
 namespace Nexmo\Message;
+use Nexmo\Message\EncodingDetector;
 use Nexmo\Entity\JsonResponseTrait;
 use Nexmo\Entity\Psr7Trait;
 use Nexmo\Entity\RequestArrayTrait;
@@ -40,6 +41,8 @@ class Message implements MessageInterface, \Countable, \ArrayAccess, \Iterator
 
     protected $id;
 
+    protected $autodetectEncoding = false;
+
     /**
      * @param string $idOrTo Message ID or E.164 (international) formatted number to send the message
      * @param null|string $from Number or name the message is from
@@ -90,6 +93,16 @@ public function setClass($class)
         return $this->setRequestData('message-class', $class);
     }
 
+    public function enableEncodingDetection()
+    {
+        $this->autodetectEncoding = true;
+    }
+
+    public function disableEncodingDetection()
+    {
+        $this->autodetectEncoding = false;
+    }
+
     public function count()
     {
         $data = $this->getResponseData();
@@ -188,6 +201,11 @@ public function getDeliveryLabel()
         return $this['error-code-label'];
     }
 
+    public function isEncodingDetectionEnabled()
+    {
+        return $this->autodetectEncoding;
+    }
+
     protected function getMessageData($name, $index = null)
     {
         if(!isset($this->response)){
@@ -206,6 +224,30 @@ protected function getMessageData($name, $index = null)
         return $data[$name];
     }
 
+    protected function preGetRequestDataHook()
+    {
+        // If $autodetectEncoding is true, we want to set the `type`
+        // field in our payload
+        if ($this->isEncodingDetectionEnabled()) {
+            $this->requestData['type'] = $this->detectEncoding();
+        }
+    }
+
+    protected function detectEncoding()
+    {
+        if (!isset($this->requestData['text'])) {
+            return static::TYPE;
+        }
+
+        // Auto detect unicode messages
+        $detector = new EncodingDetector;
+        if ($detector->requiresUnicodeEncoding($this->requestData['text'])){
+            return Unicode::TYPE;
+        }
+
+        return static::TYPE;
+    }
+
     public function offsetExists($offset)
     {
         $response = $this->getResponseData();
diff --git a/test/Message/AutoDetectTest.php b/test/Message/AutoDetectTest.php
@@ -0,0 +1,24 @@
+<?php
+/**
+ * Nexmo Client Library for PHP
+ *
+ * @copyright Copyright (c) 2016 Nexmo, Inc. (http://nexmo.com)
+ * @license   https://github.com/Nexmo/nexmo-php/blob/master/LICENSE.txt MIT License
+ */
+
+namespace NexmoTest\Message;
+use Nexmo\Message\AutoDetect;
+
+class AutoDetectTest extends \PHPUnit_Framework_TestCase
+{
+    /**
+     * When creating a message, it should not auto-detect encoding by default
+     */
+    public function testAutoDetectEnabledByDefault()
+    {
+        $message = new AutoDetect('to', 'from', 'Example Message');
+        $this->assertTrue($message->isEncodingDetectionEnabled());
+    }
+
+
+}
diff --git a/test/Message/EncodingDetectorTest.php b/test/Message/EncodingDetectorTest.php
@@ -0,0 +1,44 @@
+<?php
+/**
+ * Nexmo Client Library for PHP
+ *
+ * @copyright Copyright (c) 2016 Nexmo, Inc. (http://nexmo.com)
+ * @license   https://github.com/Nexmo/nexmo-php/blob/master/LICENSE.txt MIT License
+ */
+
+namespace NexmoTest\Message;
+use Nexmo\Message\EncodingDetector;
+
+class EncodingDetectorTest extends \PHPUnit_Framework_TestCase
+{
+
+    /**
+     * @dataProvider unicodeProvider
+     */
+    public function testDetectsUnicode($content, $expected) {
+        $d = new EncodingDetector;
+        $this->assertEquals($expected, $d->requiresUnicodeEncoding($content));
+    }
+
+    public function unicodeProvider() {
+        $r = [];
+
+        $r['ascii'] = ['Hello World', false];
+        $r['emoji'] = ['Testing 💪 👌', true];
+        $r['danish'] = ['Quizdeltagerne spiste jordbær med fløde, mens cirkusklovnen Wolther spillede på xylofon.', false];
+        $r['german'] = ['Heizölrückstoßabdämpfung', false];
+        $r['greek'] = ['  Γαζέες καὶ μυρτιὲς δὲν θὰ βρῶ πιὰ στὸ χρυσαφὶ ξέφωτο', true];
+        $r['spanish'] = ['El pingüino Wenceslao hizo kilómetros bajo exhaustiva lluvia y frío, añoraba a su querido cachorro.', true];
+        $r['french'] = ['Le cœur déçu mais l\'âme plutôt naïve, Louÿs rêva de crapaüter en canoë au delà des îles, près du mälström où brûlent les novæ.', true];
+        $r['icelandic'] = ['Kæmi ný öxi hér ykist þjófum nú bæði víl og ádrepa ', true];
+        $r['japanese-hiragana'] = ['いろはにほへとちりぬるを', true];
+        $r['japanese-katakana'] = ['イロハニホヘト チリヌルヲ ワカヨタレソ ツネナラム', true];
+        $r['hebrew'] = ['  ? דג סקרן שט בים מאוכזב ולפתע מצא לו חברה איך הקליטה', true];
+        $r['polish'] = ['Pchnąć w tę łódź jeża lub ośm skrzyń fig', true];
+        $r['russian'] = ['В чащах юга жил бы цитрус? Да, но фальшивый экземпляр!', true];
+        $r['thai'] = ['กว่าบรรดาฝูงสัตว์เดรัจฉาน', true];
+        $r['turkish'] = ['Pijamalı hasta, yağız şoföre çabucak güvendi.', true];
+
+        return $r;
+    }
+}
diff --git a/test/Message/MessageTest.php b/test/Message/MessageTest.php
@@ -8,6 +8,7 @@
 
 namespace NexmoTest\Message;
 use Nexmo\Message\Message;
+use Nexmo\Message\Text;
 use Zend\Diactoros\Response;
 use Zend\Diactoros\ServerRequest;
 
@@ -69,6 +70,41 @@ public function testCanCreateWithId()
         $this->assertEquals('00000123', $message->getMessageId());
     }
 
+    /**
+     * When creating a message, it should not auto-detect encoding by default
+     * @dataProvider messageEncodingProvider
+     */
+    public function testDoesNotAutodetectByDefault($msg, $encoding)
+    {
+        $message = new Text('to', 'from', $msg);
+        $this->assertFalse($message->isEncodingDetectionEnabled());
+        $d = $message->getRequestData(false);
+        $this->assertEquals($d['type'], 'text');
+    }
+
+    /**
+     * When creating a message, it should not auto-detect encoding by default
+     * @dataProvider messageEncodingProvider
+     */
+    public function testDoesAutodetectWhenEnabled($msg, $encoding)
+    {
+        $message = new Text('to', 'from', $msg);
+        $message->enableEncodingDetection();
+        $this->assertTrue($message->isEncodingDetectionEnabled());
+
+        $d = $message->getRequestData(false);
+        $this->assertEquals($d['type'], $encoding);
+    }
+
+    public function messageEncodingProvider() {
+
+        $r = [];
+        $r['text'] = ['Hello World', 'text'];
+        $r['emoji'] = ['Testing 💪', 'unicode'];
+        $r['kanji'] = ['漢字', 'unicode'];
+        return $r;
+    }
+
     /**
      * Get the API response we'd expect for a call to the API. Message API currently returns 200 all the time, so only
      * change between success / fail is body of the message.

Original file line number	Diff line number	Diff line change
`@@ -40,6 +40,13 @@ public function getRequestData($sent = true)`
`40`	`40`	`return $query;`
`41`	`41`	`}`
`42`	`42`
	`43`	`+ // Trigger a pre-getRequestData() hook for any last minute`
	`44`	`+ // decision making that needs to be done, but only if`
	`45`	`+ // it hasn't been sent already`
	`46`	`+ if (method_exists($this, 'preGetRequestDataHook')) {`
	`47`	`+ $this->preGetRequestDataHook();`
	`48`	`+ }`
	`49`	`+`
`43`	`50`	`return $this->requestData;`
`44`	`51`	`}`
`45`	`52`