fix: add support for multibyte characters (#2)

* test(unicode): add tests for complicated characters

if the length of a character is more than one (here tested with 2 and 7), it should be counted as that length, but definitely shouldn't cut in the middle of that character.
This does not work as expected yet, so the test is skipped

* use an array to have single characters

* add another failing case

* comp++

* fix tests by using runes

Author: Haroenv

package.json

@@ -30,5 +30,8 @@
     "eslint-plugin-prettier": "^2.1.2",
     "jest": "^20.0.4",
     "prettier": "^1.4.4"
+  },
+  "dependencies": {
+    "runes": "^0.4.1"
   }
 }

src/__tests__/index.js

@@ -39,3 +39,26 @@ it('should truncate a word if longer than size', () => {
   const pieces = chunk('hello you', 4);
   expect(pieces).toEqual(['hell', 'o', 'you']);
 });
+
+it('should count double width characters as single characters', () => {
+  // each of these characters is two bytes
+  const chineseText = '𤻪𬜬𬜯';
+  const camembert = '🧀🧀🧀🧀 🧀🧀🧀🧀';
+
+  expect(chunk(chineseText, 2)).toEqual(['𤻪𬜬', '𬜯']);
+  expect(chunk(chineseText, 1)).toEqual(['𤻪', '𬜬', '𬜯']);
+  expect(chunk(camembert, 4)).toEqual(['🧀🧀🧀🧀', '🧀🧀🧀🧀']);
+});
+
+// this test does not pass yet
+it('should not cut combined characters', () => {
+  // one woman runner emoji with a colour is seven bytes, or five characters
+  // RUNNER(2) + COLOUR(2) + ZJW + GENDER + VS15
+  const runners = '🏃🏽‍♀️🏃🏽‍♀️🏃🏽‍♀️';
+  // FLAG + RAINBOW
+  const flags = '🏳️‍🌈🏳️‍🌈';
+
+  expect(chunk(runners, 3)).toEqual(['🏃🏽‍♀️🏃🏽‍♀️🏃🏽‍♀️']);
+  expect(chunk(runners, 1)).toEqual(['🏃🏽‍♀️', '🏃🏽‍♀️', '🏃🏽‍♀️']);
+  expect(chunk(flags, 1)).toEqual(['🏳️‍🌈', '🏳️‍🌈']);
+});

src/index.js

@@ -1,3 +1,6 @@
+/* eslint-disable import/no-commonjs */
+const runes = require('runes');
+
 const assertIsValidText = function(text) {
   if (typeof text !== 'string') {
     throw new TypeError(
@@ -19,19 +22,24 @@ export default function(text, chunkSize) {
   assertIsValidChunkSize(chunkSize);
 
   const chunks = [];
-  while (text.length > chunkSize) {
-    const splitAt = text.lastIndexOf(' ', chunkSize);
+  let characters = runes(text);
+
+  while (characters.length > chunkSize) {
+    const splitAt = characters.lastIndexOf(' ', chunkSize);
+
     if (splitAt === -1) {
       // No whitespace found, we need to truncate the word in that case.
-      chunks.push(text.substr(0, chunkSize));
-      text = text.substr(chunkSize); // eslint-disable-line no-param-reassign
+      const chunk = characters.slice(0, chunkSize).join('');
+      chunks.push(chunk);
+      characters = characters.slice(chunkSize); // eslint-disable-line no-param-reassign
     } else {
-      chunks.push(text.substr(0, splitAt));
-      text = text.substr(splitAt + 1); // eslint-disable-line no-param-reassign
+      const chunk = characters.slice(0, splitAt).join('');
+      chunks.push(chunk);
+      characters = characters.slice(splitAt + 1); // eslint-disable-line no-param-reassign
     }
   }
 
-  chunks.push(text);
+  chunks.push(characters.join(''));
 
   return chunks;
 }

yarn.lock

@@ -2809,6 +2809,10 @@ run-async@^2.2.0:
   dependencies:
     is-promise "^2.1.0"
 
+runes@^0.4.1:
+  version "0.4.1"
+  resolved "https://registry.yarnpkg.com/runes/-/runes-0.4.1.tgz#8fe5ca2cf7d7a384c6456beeeff6c4e9dc3e8502"
+
 rx-lite-aggregates@^4.0.8:
   version "4.0.8"
   resolved "https://registry.yarnpkg.com/rx-lite-aggregates/-/rx-lite-aggregates-4.0.8.tgz#753b87a89a11c95467c4ac1626c4efc4e05c67be"