refactor: change the message ids cache in chatwoot to use a in memory cache

Remove use of disc cache for optimize performance.
To make this, we need change to use only one instance of ChatwootService in entire application.

src/whatsapp/controllers/chatwoot.controller.ts

@@ -7,7 +7,7 @@ import { ChatwootDto } from '../dto/chatwoot.dto';
 import { InstanceDto } from '../dto/instance.dto';
 import { RepositoryBroker } from '../repository/repository.manager';
 import { ChatwootService } from '../services/chatwoot.service';
-import { waMonitor } from '../whatsapp.module';
+import { instanceController } from '../whatsapp.module';
 
 const logger = new Logger('ChatwootController');
 
@@ -94,7 +94,7 @@ export class ChatwootController {
 
   public async receiveWebhook(instance: InstanceDto, data: any) {
     logger.verbose('requested receiveWebhook from ' + instance.instanceName + ' instance');
-    const chatwootService = new ChatwootService(waMonitor, this.configService, this.repository);
+    const chatwootService = instanceController.getChatwootService();
 
     return chatwootService.receiveWebhook(instance, data);
   }

src/whatsapp/controllers/instance.controller.ts

@@ -659,4 +659,10 @@ export class InstanceController {
     this.logger.verbose('requested refreshToken');
     return await this.authService.refreshToken(oldToken);
   }
+
+  public getChatwootService() {
+    this.logger.verbose('getting chatwootService object instance');
+
+    return this.chatwootService;
+  }
 }

src/whatsapp/services/chatwoot.service.ts

@@ -1,14 +1,13 @@
 import ChatwootClient from '@figuro/chatwoot-sdk';
 import axios from 'axios';
 import FormData from 'form-data';
-import { createReadStream, readFileSync, unlinkSync, writeFileSync } from 'fs';
+import { createReadStream, unlinkSync, writeFileSync } from 'fs';
 import Jimp from 'jimp';
 import mimeTypes from 'mime-types';
 import path from 'path';
 
 import { ConfigService, HttpServer } from '../../config/env.config';
 import { Logger } from '../../config/logger.config';
-import { ROOT_DIR } from '../../config/path.config';
 import { ChatwootDto } from '../dto/chatwoot.dto';
 import { InstanceDto } from '../dto/instance.dto';
 import { Options, Quoted, SendAudioDto, SendMediaDto, SendTextDto } from '../dto/sendMessage.dto';
@@ -18,8 +17,7 @@ import { Events } from '../types/wa.types';
 import { WAMonitoringService } from './monitor.service';
 
 export class ChatwootService {
-  private messageCacheFile: string;
+  private messageCache: Record<string, Set<number>>;
-  private messageCache: Set<string>;
 
   private readonly logger = new Logger(ChatwootService.name);
 
@@ -30,31 +28,25 @@ export class ChatwootService {
     private readonly configService: ConfigService,
     private readonly repository: RepositoryBroker,
   ) {
-    this.messageCache = new Set();
+    this.messageCache = {};
   }
 
-  private loadMessageCache(): Set<string> {
+  private isMessageInCache(instance: InstanceDto, id: number) {
-    this.logger.verbose('load message cache');
+    this.logger.verbose('check if message is in cache');
-    try {
+    if (!this.messageCache[instance.instanceName]) {
-      const cacheData = readFileSync(this.messageCacheFile, 'utf-8');
+      return false;
-      const cacheArray = cacheData.split('\n');
-      return new Set(cacheArray);
-    } catch (error) {
-      return new Set();
     }
+
+    return this.messageCache[instance.instanceName].has(id);
   }
 
-  private saveMessageCache() {
+  private addMessageToCache(instance: InstanceDto, id: number) {
-    this.logger.verbose('save message cache');
+    this.logger.verbose('add message to cache');
-    const cacheData = Array.from(this.messageCache).join('\n');
-    writeFileSync(this.messageCacheFile, cacheData, 'utf-8');
-    this.logger.verbose('message cache saved');
-  }
 
-  private clearMessageCache() {
+    if (!this.messageCache[instance.instanceName]) {
-    this.logger.verbose('clear message cache');
+      this.messageCache[instance.instanceName] = new Set();
-    this.messageCache.clear();
+    }
-    this.saveMessageCache();
+    this.messageCache[instance.instanceName].add(id);
   }
 
   private async getProvider(instance: InstanceDto) {
@@ -1105,22 +1097,11 @@ export class ChatwootService {
       if (body.message_type === 'outgoing' && body?.conversation?.messages?.length && chatId !== '123456') {
         this.logger.verbose('check if is group');
 
-        this.messageCacheFile = path.join(ROOT_DIR, 'store', 'chatwoot', `${instance.instanceName}_cache.txt`);
+        if (this.isMessageInCache(instance, body.id)) {
-        this.logger.verbose('cache file path: ' + this.messageCacheFile);
-
-        this.messageCache = this.loadMessageCache();
-        this.logger.verbose('cache file loaded');
-        this.logger.verbose(this.messageCache);
-
-        this.logger.verbose('check if message is cached');
-        if (this.messageCache.has(body.id.toString())) {
           this.logger.verbose('message is cached');
           return { message: 'bot' };
         }
 
-        this.logger.verbose('clear cache');
-        this.clearMessageCache();
-
         this.logger.verbose('Format message to send');
         let formatText: string;
         if (senderName === null || senderName === undefined) {
@@ -1597,14 +1578,7 @@ export class ChatwootService {
               return;
             }
 
-            this.messageCacheFile = path.join(ROOT_DIR, 'store', 'chatwoot', `${instance.instanceName}_cache.txt`);
+            this.addMessageToCache(instance, send.id);
-
-            this.messageCache = this.loadMessageCache();
-
-            this.messageCache.add(send.id.toString());
-
-            this.logger.verbose('save message cache');
-            this.saveMessageCache();
 
             return send;
           } else {
@@ -1618,14 +1592,7 @@ export class ChatwootService {
               return;
             }
 
-            this.messageCacheFile = path.join(ROOT_DIR, 'store', 'chatwoot', `${instance.instanceName}_cache.txt`);
+            this.addMessageToCache(instance, send.id);
-
-            this.messageCache = this.loadMessageCache();
-
-            this.messageCache.add(send.id.toString());
-
-            this.logger.verbose('save message cache');
-            this.saveMessageCache();
 
             return send;
           }
@@ -1650,11 +1617,7 @@ export class ChatwootService {
               this.logger.warn('message not sent');
               return;
             }
-            this.messageCacheFile = path.join(ROOT_DIR, 'store', 'chatwoot', `${instance.instanceName}_cache.txt`);
+            this.addMessageToCache(instance, send.id);
-            this.messageCache = this.loadMessageCache();
-            this.messageCache.add(send.id.toString());
-            this.logger.verbose('save message cache');
-            this.saveMessageCache();
           }
 
           return;
@@ -1711,14 +1674,7 @@ export class ChatwootService {
             return;
           }
 
-          this.messageCacheFile = path.join(ROOT_DIR, 'store', 'chatwoot', `${instance.instanceName}_cache.txt`);
+          this.addMessageToCache(instance, send.id);
-
-          this.messageCache = this.loadMessageCache();
-
-          this.messageCache.add(send.id.toString());
-
-          this.logger.verbose('save message cache');
-          this.saveMessageCache();
 
           return send;
         }
@@ -1746,14 +1702,7 @@ export class ChatwootService {
             return;
           }
 
-          this.messageCacheFile = path.join(ROOT_DIR, 'store', 'chatwoot', `${instance.instanceName}_cache.txt`);
+          this.addMessageToCache(instance, send.id);
-
-          this.messageCache = this.loadMessageCache();
-
-          this.messageCache.add(send.id.toString());
-
-          this.logger.verbose('save message cache');
-          this.saveMessageCache();
 
           return send;
         } else {
@@ -1767,14 +1716,7 @@ export class ChatwootService {
             return;
           }
 
-          this.messageCacheFile = path.join(ROOT_DIR, 'store', 'chatwoot', `${instance.instanceName}_cache.txt`);
+          this.addMessageToCache(instance, send.id);
-
-          this.messageCache = this.loadMessageCache();
-
-          this.messageCache.add(send.id.toString());
-
-          this.logger.verbose('save message cache');
-          this.saveMessageCache();
 
           return send;
         }

src/whatsapp/services/whatsapp.service.ts

@@ -75,6 +75,7 @@ import { getIO } from '../../libs/socket.server';
 import { getSQS, removeQueues as removeQueuesSQS } from '../../libs/sqs.server';
 import { useMultiFileAuthStateDb } from '../../utils/use-multi-file-auth-state-db';
 import { useMultiFileAuthStateRedisDb } from '../../utils/use-multi-file-auth-state-redis-db';
+import { instanceController } from '../../whatsapp/whatsapp.module';
 import {
   ArchiveChatDto,
   DeleteMessage,
@@ -131,7 +132,6 @@ import { RepositoryBroker } from '../repository/repository.manager';
 import { Events, MessageSubtype, TypeMediaMessage, wa } from '../types/wa.types';
 import { waMonitor } from '../whatsapp.module';
 import { ChamaaiService } from './chamaai.service';
-import { ChatwootService } from './chatwoot.service';
 import { TypebotService } from './typebot.service';
 
 const retryCache = {};
@@ -169,7 +169,7 @@ export class WAStartupService {
 
   private phoneNumber: string;
 
-  private chatwootService = new ChatwootService(waMonitor, this.configService, this.repository);
+  private chatwootService = instanceController.getChatwootService();
 
   private typebotService = new TypebotService(waMonitor, this.configService, this.eventEmitter);