Fix typeahead suggestions with nonlatin chars

app/assets/javascripts/app/views/search_base_view.js

@@ -10,13 +10,18 @@ app.views.SearchBase = app.views.Base.extend({
     if(options.autoselect) { this.setupAutoselect(); }
   },
 
+  bloodhoundTokenizer: function(str) {
+    if(typeof str !== "string") { return []; }
+    return str.split(/[\s\.:,;\?\!#@\-_\[\]\{\}\(\)]+/).filter(function(s) { return s !== ""; });
+  },
+
   setupBloodhound: function(options) {
     var bloodhoundOptions = {
       datumTokenizer: function(datum) {
-        var nameTokens = Bloodhound.tokenizers.nonword(datum.name);
+        var nameTokens = this.bloodhoundTokenizer(datum.name);
-        var handleTokens = datum.handle ? Bloodhound.tokenizers.nonword(datum.name) : [];
+        var handleTokens = datum.handle ? this.bloodhoundTokenizer(datum.handle) : [];
         return nameTokens.concat(handleTokens);
-      },
+      }.bind(this),
       queryTokenizer: Bloodhound.tokenizers.whitespace,
       prefetch: {
         url: "/contacts.json",

spec/javascripts/app/views/search_base_view_spec.js

@@ -65,6 +65,37 @@ describe("app.views.SearchBase", function() {
     });
   });
 
+  describe("bloodhoundTokenizer", function() {
+    beforeEach(function() {
+      this.view = new app.views.SearchBase({ el: "#search_people_form", typeaheadInput: $("#q") });
+    });
+
+    it("splits the string at whitespaces and punctuation chars", function() {
+      expect(this.view.bloodhoundTokenizer("ab.c-d_ef g;h,i  #jkl?mnopq!rstu[vwx]::y(z){}")).toEqual(
+        ["ab", "c", "d", "ef", "g", "h", "i", "jkl", "mnopq", "rstu", "vwx", "y", "z"]
+      );
+    });
+
+    it("doesn't split the string at Cyrillic chars", function() {
+      expect(this.view.bloodhoundTokenizer("АаБбВвГгДдЕеЁё ЖжЗзИиЙйКкЛлМмНнОоПпРрСсТтУуФф")).toEqual(
+        ["АаБбВвГгДдЕеЁё", "ЖжЗзИиЙйКкЛлМмНнОоПпРрСсТтУуФф"]
+      );
+    });
+
+    it("doesn't split the string at Malayalam chars", function() {
+      expect(this.view.bloodhoundTokenizer("ബിപിൻദാസ്")).toEqual(
+        ["ബിപിൻദാസ്"]
+      );
+    });
+
+    it("returns an empty array inputs which are not a string", function() {
+      expect(this.view.bloodhoundTokenizer(undefined)).toEqual([]);
+      expect(this.view.bloodhoundTokenizer(null)).toEqual([]);
+      expect(this.view.bloodhoundTokenizer(23)).toEqual([]);
+      expect(this.view.bloodhoundTokenizer({foo: "bar"})).toEqual([]);
+    });
+  });
+
   describe("setupCustomSearch", function() {
     it("sets bloodhound.customSearch", function() {
       this.view = new app.views.SearchBase({el: "#search_people_form", typeaheadInput: $("#q")});