Add support for japanese and chinese search

package.json

@@ -14,9 +14,11 @@
     "@types/glob": "^7.1.2",
     "@types/i18n": "^0.12.0",
     "@types/js-md5": "^0.4.2",
+    "@types/jskana": "^0.1.0",
     "@types/mathjs": "^6.0.5",
     "@types/node": "^14.0.19",
     "@types/node-fetch": "^2.5.7",
+    "@types/pinyin": "^2.10.0",
     "@types/redis": "^2.8.25",
     "@types/request-promise": "^4.1.46",
     "@types/validator": "^13.0.0",
@@ -27,6 +29,7 @@
     "chalk": "^4.1.0",
     "chance": "^1.1.7",
     "cheerio": "^1.0.0-rc.3",
+    "cld": "^2.7.1",
     "cookie-parser": "^1.4.5",
     "cross-fetch": "^3.1.4",
     "date-fns": "^2.16.1",
@@ -40,12 +43,14 @@
     "hangul-romanization": "^1.0.0",
     "http-errors": "^1.8.0",
     "js-md5": "^0.7.3",
+    "jskana": "^0.1.1",
     "mathjs": "^7.1.0",
     "morgan": "^1.10.0",
     "node-fetch": "^2.6.0",
     "npm": "^6.14.8",
     "pg": "^8.3.2",
     "pg-hstore": "^2.3.3",
+    "pinyin": "^2.10.2",
     "pm2": "^4.5.0",
     "prom-client": "^14.0.0",
     "pug": "^3.0.0",

src/commands/Lastfm/Library/SearchAlbum.ts

@@ -37,9 +37,21 @@ export default class SearchAlbum extends SearchCommand {
       concurrent: this.variationWasUsed("deep"),
     });
 
-    const filtered = topAlbums.albums.filter((a) =>
-      this.clean(a.name).includes(this.clean(keywords))
-    );
+    const filteredKeywords = {
+      whitespace: (await this.clean(keywords, true)).text,
+      noWhitespace: (await this.clean(keywords, true)).text.replace(/\s+/g, "")
+    }
+
+    const filtered = await this.asyncFilter(topAlbums.albums, async (a) => {
+      const currentString = (await this.clean(a.name, false));
+      let currentKeywords = "";
+      if (currentString.noWhitespace) {
+        currentKeywords = filteredKeywords.noWhitespace;
+      } else {
+        currentKeywords = filteredKeywords.whitespace;
+      }
+      return currentString.text.includes(currentKeywords);
+    });
 
     if (filtered.length !== 0 && filtered.length === topAlbums.albums.length) {
       throw new LogicError(

src/commands/Lastfm/Library/SearchArtist.ts

@@ -37,9 +37,22 @@ export default class SearchArtist extends SearchCommand {
       concurrent: this.variationWasUsed("deep"),
     });
 
-    const filtered = topArtists.artists.filter((a) =>
-      this.clean(a.name).includes(this.clean(keywords))
-    );
+    const filteredKeywords = {
+      whitespace: (await this.clean(keywords, true)).text,
+      noWhitespace: (await this.clean(keywords, true)).text.replace(/\s+/g, "")
+    }
+
+    const filtered = await this.asyncFilter(topArtists.artists, async (a) => {
+
+      const currentString = (await this.clean(a.name, false));
+      let currentKeywords = "";
+      if (currentString.noWhitespace) {
+        currentKeywords = filteredKeywords.noWhitespace;
+      } else {
+        currentKeywords = filteredKeywords.whitespace;
+      }
+      return currentString.text.includes(currentKeywords);
+    });
 
     if (
       filtered.length !== 0 &&

src/commands/Lastfm/Library/SearchCommand.ts

@@ -1,8 +1,12 @@
 import { convert as romanizeHangeul } from "hangul-romanization";
+import pinyin from "pinyin";
+import cld from "cld";
 import { Arguments } from "../../../lib/arguments/arguments";
 import { standardMentions } from "../../../lib/arguments/mentions/mentions";
 import { Validation } from "../../../lib/validation/ValidationChecker";
 import { validators } from "../../../lib/validation/validators";
+import { MecabService } from "../../../services/mecab/MecabService";
+import { ServiceRegistry } from "../../../services/ServicesRegistry";
 import { LastFMBaseCommand } from "../LastFMBaseCommand";
 
 const args = {
@@ -32,12 +36,79 @@ export abstract class SearchCommand extends LastFMBaseCommand<typeof args> {
     ],
   };
 
-  clean(string: string): string {
-    return romanizeHangeul(string)
+  async clean(string: string, isKeyword:boolean): Promise<{text:string, noWhitespace:boolean}> {
+    let processedInput = await this.processLanguage(string, isKeyword);
+    processedInput.text = processedInput.text
       .replace(/[\-_'"‘’”“`「」『』«»―~‐⁓,.]+/g, "")
       .replace("&", " and ")
       .toLowerCase()
       .normalize("NFD")
       .replace(/[\u0300-\u036f]/g, "");
+    return processedInput;
+  }
+
+  private processChinese(input:string) {
+    return [pinyin(input).join(""), pinyin(input, {style: pinyin.STYLE_NORMAL}).join(""), input].join("").replace(/\s+/g, "");
+  }
+
+  private async processText(input:string, language:string, isKeyword:boolean):Promise<{text:string, noWhitespace:boolean}> {
+    switch(language) {
+      case "zh":
+        if (isKeyword) {
+          return {
+            text: input,
+            noWhitespace: true
+          };
+        } else {
+          return {
+            text: this.processChinese(input),
+            noWhitespace: true
+          };
+        }
+
+      case "ja":
+        if (isKeyword) {
+          return {
+            text: input,
+            noWhitespace: true
+          };
+        } else {
+          return {
+            text: await ServiceRegistry.get(MecabService).processJapanese(input),
+            noWhitespace: true
+          };
+        }
+      case "kr":
+        return {
+          text: romanizeHangeul(input),
+          noWhitespace: false
+        };
+      default:
+        return {
+          text: input,
+          noWhitespace: false
+        };
+    }
+  }
+
+  private async processLanguage(input:string, isKeyword:boolean) {
+    try {
+      const language = (await cld.detect(input)).languages[0];
+      return await this.processText(input, language.code, isKeyword);
+    } catch (err) {
+      return await this.processText(input, "en", isKeyword); //fallback to english if no language detected
+    }
+
+  }
+
+  protected async asyncFilter (arr:any[], predicate:(a: any) => Promise<boolean>) {
+
+    let res:any[] = [];
+    for (let entry of arr) {
+      if (await predicate(entry)) {
+        res.push(entry)
+      }
+    }
+    return res;
   }
 }

src/commands/Lastfm/Library/SearchTrack.ts

@@ -37,9 +37,21 @@ export default class SearchTrack extends SearchCommand {
       concurrent: this.variationWasUsed("deep"),
     });
 
-    const filtered = topTracks.tracks.filter((t) =>
-      this.clean(t.name).includes(this.clean(keywords))
-    );
+    const filteredKeywords = {
+      whitespace: (await this.clean(keywords, true)).text,
+      noWhitespace: (await this.clean(keywords, true)).text.replace(/\s+/g, "")
+    }
+
+    const filtered = await this.asyncFilter(topTracks.tracks, async (t) => {
+      const currentString = (await this.clean(t.name, false));
+      let currentKeywords = "";
+      if (currentString.noWhitespace) {
+        currentKeywords = filteredKeywords.noWhitespace;
+      } else {
+        currentKeywords = filteredKeywords.whitespace;
+      }
+      return currentString.text.includes(currentKeywords);
+    });
 
     if (filtered.length !== 0 && filtered.length === topTracks.tracks.length) {
       throw new LogicError(

src/customTypings/cld.d.ts

@@ -0,0 +1,21 @@
+interface Language {
+  readonly name: string;
+  readonly code: string;
+  readonly percent: number;
+  readonly score: number;
+}
+interface Chunk {
+  readonly name: string;
+  readonly code: string;
+  readonly offset: number;
+  readonly bytes: number;
+}
+interface DetectLanguage {
+  readonly reliable: boolean;
+  readonly textBytes: number;
+  readonly languages: Language[];
+  readonly chunks: Chunk[];
+}
+declare module "cld" {
+  function detect(text: string):Promise<DetectLanguage>;
+}

src/services/ServicesRegistry.ts

@@ -35,6 +35,7 @@ import { WordBlacklistService } from "./WordBlacklistService";
 import { RollbarService } from "./Rollbar/RollbarService";
 import { NowPlayingEmbedParsingService } from "./NowPlayingEmbedParsingService";
 import { ChartService } from "./pantomime/ChartService";
+import { MecabService } from "./mecab/MecabService";
 
 type Service<T = any> = { new (): T };
 
@@ -59,6 +60,7 @@ const services: Service[] = [
   GuildEventService,
   LastFMService,
   LastFMArguments,
+  MecabService,
   MetaService,
   MirrorballService,
   MirrorballCacheService,

src/services/mecab/MecabService.ts

@@ -0,0 +1,68 @@
+import { ChildProcessWithoutNullStreams, spawn } from "child_process";
+import { BaseService } from "../BaseService";
+import jskana from "jskana";
+
+export class MecabService extends BaseService {
+
+  mecabProcess:ChildProcessWithoutNullStreams;
+
+  constructor() {
+    super();
+    this.mecabProcess = spawn("mecab");
+  }
+
+  //currently unused, when bot shutdown script is added, call this method for more graceful exit. Not necessary.
+  public close() {
+    this.mecabProcess.kill("SIGINT");
+  }
+
+  private segmentText(input:string):Promise<string> {
+    this.mecabProcess.stdin.write(input);
+
+    return new Promise((resolve, reject) => {
+
+      const resolver = (data:string, resolve:(value: string | PromiseLike<string>) => void) => {
+        resolve(data.toString());
+        clearHandlers(resolver, rejecter);
+      }
+
+      const rejecter = (error:string, reject:(reason?: any) => void) => {
+        reject(error);
+        clearHandlers(resolver, rejecter);
+      }
+
+      const clearHandlers = (resolver:(data: string, resolve: (value: string | PromiseLike<string>) => void) => void, rejecter:(error: string, reject: (reason?: any) => void) => void) => {
+        this.mecabProcess.stdout.removeListener('data', resolver);
+        this.mecabProcess.stderr.removeListener('data', rejecter);
+      }
+
+      this.mecabProcess.stdout.on('data', (data:string) => { resolver(data, resolve) });
+      this.mecabProcess.stderr.on('data', (error:string) => { rejecter(error, reject) });
+    });
+  }
+
+  public async processJapanese(input:string) {
+    const unprocessed = await this.segmentText(input + "\n");
+    const processed = unprocessed.split("\n").slice(0,-2).map(word => {
+      const commasplit = word.split(",");
+      const kanji = word.split("\t")[0];
+      if (commasplit[commasplit.length - 2] === "*") {
+        return [kanji,kanji,kanji,kanji,kanji];
+      } else {
+        return [kanji,
+                jskana.katakanaToHiragana(commasplit[commasplit.length - 2]),
+                commasplit[commasplit.length - 2],
+                commasplit[commasplit.length - 1]];
+      }
+
+    }).reduce((acc, cur) => {
+      acc[0] += cur[0];
+      acc[1] += cur[1];
+      acc[2] += cur[2];
+      acc[3] += cur[3];
+      return acc;
+    }, ["", "", "", "", ""]);
+    return [...processed.slice(0,3), jskana.kanaToRomaji(processed[3]), jskana.kanaToRomaji(processed[2]), input].join("").replace(/\s+/g, "");
+  }
+
+}