diff --git a/src/config/env.config.ts b/src/config/env.config.ts index 511f4ebc..71a1a497 100644 --- a/src/config/env.config.ts +++ b/src/config/env.config.ts @@ -132,11 +132,22 @@ export type GlobalWebhook = { ENABLED: boolean; WEBHOOK_BY_EVENTS: boolean; }; +export type CacheConfRedis = { + ENABLED: boolean; + URI: string; + PREFIX_KEY: string; + TTL: number; +}; +export type CacheConfLocal = { + ENABLED: boolean; + TTL: number; +}; export type SslConf = { PRIVKEY: string; FULLCHAIN: string }; export type Webhook = { GLOBAL?: GlobalWebhook; EVENTS: EventsWebhook }; export type ConfigSessionPhone = { CLIENT: string; NAME: string }; export type QrCode = { LIMIT: number; COLOR: string }; export type Typebot = { API_VERSION: string; KEEP_OPEN: boolean }; +export type CacheConf = { REDIS: CacheConfRedis; LOCAL: CacheConfLocal }; export type Production = boolean; export interface Env { @@ -156,6 +167,7 @@ export interface Env { CONFIG_SESSION_PHONE: ConfigSessionPhone; QRCODE: QrCode; TYPEBOT: Typebot; + CACHE: CacheConf; AUTHENTICATION: Auth; PRODUCTION?: Production; } @@ -318,6 +330,18 @@ export class ConfigService { API_VERSION: process.env?.TYPEBOT_API_VERSION || 'old', KEEP_OPEN: process.env.TYPEBOT_KEEP_OPEN === 'true', }, + CACHE: { + REDIS: { + ENABLED: process.env?.CACHE_REDIS_ENABLED === 'true', + URI: process.env.CACHE_REDIS_URI || '', + PREFIX_KEY: process.env.CACHE_REDIS_PREFIX_KEY || 'evolution-cache', + TTL: Number.parseInt(process.env.CACHE_REDIS_TTL) || 604800, + }, + LOCAL: { + ENABLED: process.env?.CACHE_LOCAL_ENABLED === 'true', + TTL: Number.parseInt(process.env.CACHE_REDIS_TTL) || 86400, + }, + }, AUTHENTICATION: { TYPE: process.env.AUTHENTICATION_TYPE as 'apikey', API_KEY: { diff --git a/src/dev-env.yml b/src/dev-env.yml index 80c7e376..42438aff 100644 --- a/src/dev-env.yml +++ b/src/dev-env.yml @@ -153,6 +153,17 @@ TYPEBOT: API_VERSION: 'old' # old | latest KEEP_OPEN: false +# Cache to optimize application performance +CACHE: + REDIS: + ENABLED: false + URI: "redis://localhost:6379" + PREFIX_KEY: "evolution-cache" + TTL: 604800 + LOCAL: + ENABLED: false + TTL: 86400 + # Defines an authentication type for the api # We recommend using the apikey because it will allow you to use a custom token, # if you use jwt, a random token will be generated and may be expired and you will have to generate a new token diff --git a/src/libs/cacheengine.ts b/src/libs/cacheengine.ts new file mode 100644 index 00000000..a22d7e68 --- /dev/null +++ b/src/libs/cacheengine.ts @@ -0,0 +1,22 @@ +import { CacheConf, ConfigService } from '../config/env.config'; +import { ICache } from '../whatsapp/abstract/abstract.cache'; +import { LocalCache } from './localcache'; +import { RedisCache } from './rediscache'; + +export class CacheEngine { + private engine: ICache; + + constructor(private readonly configService: ConfigService, module: string) { + const cacheConf = configService.get('CACHE'); + + if (cacheConf?.REDIS?.ENABLED && cacheConf?.REDIS?.URI !== '') { + this.engine = new RedisCache(configService, module); + } else if (cacheConf?.LOCAL?.ENABLED) { + this.engine = new LocalCache(configService, module); + } + } + + public getEngine() { + return this.engine; + } +} diff --git a/src/libs/localcache.ts b/src/libs/localcache.ts new file mode 100644 index 00000000..fe1f295f --- /dev/null +++ b/src/libs/localcache.ts @@ -0,0 +1,48 @@ +import NodeCache from 'node-cache'; + +import { CacheConf, CacheConfLocal, ConfigService } from '../config/env.config'; +import { ICache } from '../whatsapp/abstract/abstract.cache'; + +export class LocalCache implements ICache { + private conf: CacheConfLocal; + static localCache = new NodeCache(); + + constructor(private readonly configService: ConfigService, private readonly module: string) { + this.conf = this.configService.get('CACHE')?.LOCAL; + } + + async get(key: string): Promise { + return LocalCache.localCache.get(this.buildKey(key)); + } + + async set(key: string, value: any, ttl?: number) { + return LocalCache.localCache.set(this.buildKey(key), value, ttl || this.conf.TTL); + } + + async has(key: string) { + return LocalCache.localCache.has(this.buildKey(key)); + } + + async delete(key: string) { + return LocalCache.localCache.del(this.buildKey(key)); + } + + async deleteAll(appendCriteria?: string) { + const keys = await this.keys(appendCriteria); + if (!keys?.length) { + return 0; + } + + return LocalCache.localCache.del(keys); + } + + async keys(appendCriteria?: string) { + const filter = `${this.buildKey('')}${appendCriteria ? `${appendCriteria}:` : ''}`; + + return LocalCache.localCache.keys().filter((key) => key.substring(0, filter.length) === filter); + } + + buildKey(key: string) { + return `${this.module}:${key}`; + } +} diff --git a/src/libs/rediscache.client.ts b/src/libs/rediscache.client.ts new file mode 100644 index 00000000..b3f8dead --- /dev/null +++ b/src/libs/rediscache.client.ts @@ -0,0 +1,59 @@ +import { createClient, RedisClientType } from 'redis'; + +import { CacheConf, CacheConfRedis, configService } from '../config/env.config'; +import { Logger } from '../config/logger.config'; + +class Redis { + private logger = new Logger(Redis.name); + private client: RedisClientType = null; + private conf: CacheConfRedis; + private connected = false; + + constructor() { + this.conf = configService.get('CACHE')?.REDIS; + } + + getConnection(): RedisClientType { + if (this.connected) { + return this.client; + } else { + this.client = createClient({ + url: this.conf.URI, + }); + + this.client.on('connect', () => { + this.logger.verbose('redis connecting'); + }); + + this.client.on('ready', () => { + this.logger.verbose('redis ready'); + this.connected = true; + }); + + this.client.on('error', () => { + this.logger.error('redis disconnected'); + this.connected = false; + }); + + this.client.on('end', () => { + this.logger.verbose('redis connection ended'); + this.connected = false; + }); + + try { + this.logger.verbose('connecting new redis client'); + this.client.connect(); + this.connected = true; + this.logger.verbose('connected to new redis client'); + } catch (e) { + this.connected = false; + this.logger.error('redis connect exception caught: ' + e); + return null; + } + + return this.client; + } + } +} + +export const redisClient = new Redis(); diff --git a/src/libs/rediscache.ts b/src/libs/rediscache.ts new file mode 100644 index 00000000..cd0b1283 --- /dev/null +++ b/src/libs/rediscache.ts @@ -0,0 +1,83 @@ +import { RedisClientType } from 'redis'; + +import { CacheConf, CacheConfRedis, ConfigService } from '../config/env.config'; +import { Logger } from '../config/logger.config'; +import { ICache } from '../whatsapp/abstract/abstract.cache'; +import { redisClient } from './rediscache.client'; + +export class RedisCache implements ICache { + private readonly logger = new Logger(RedisCache.name); + private client: RedisClientType; + private conf: CacheConfRedis; + + constructor(private readonly configService: ConfigService, private readonly module: string) { + this.conf = this.configService.get('CACHE')?.REDIS; + this.client = redisClient.getConnection(); + } + + async get(key: string): Promise { + try { + return JSON.parse(await this.client.get(this.buildKey(key))); + } catch (error) { + this.logger.error(error); + } + } + + async set(key: string, value: any, ttl?: number) { + try { + await this.client.setEx(this.buildKey(key), ttl || this.conf?.TTL, JSON.stringify(value)); + } catch (error) { + this.logger.error(error); + } + } + + async has(key: string) { + try { + return (await this.client.exists(this.buildKey(key))) > 0; + } catch (error) { + this.logger.error(error); + } + } + + async delete(key: string) { + try { + return await this.client.del(this.buildKey(key)); + } catch (error) { + this.logger.error(error); + } + } + + async deleteAll(appendCriteria?: string) { + try { + const keys = await this.keys(appendCriteria); + if (!keys?.length) { + return 0; + } + + return await this.client.del(keys); + } catch (error) { + this.logger.error(error); + } + } + + async keys(appendCriteria?: string) { + try { + const match = `${this.buildKey('')}${appendCriteria ? `${appendCriteria}:` : ''}*`; + const keys = []; + for await (const key of this.client.scanIterator({ + MATCH: match, + COUNT: 100, + })) { + keys.push(key); + } + + return [...new Set(keys)]; + } catch (error) { + this.logger.error(error); + } + } + + buildKey(key: string) { + return `${this.conf?.PREFIX_KEY}:${this.module}:${key}`; + } +} diff --git a/src/whatsapp/abstract/abstract.cache.ts b/src/whatsapp/abstract/abstract.cache.ts new file mode 100644 index 00000000..caad2691 --- /dev/null +++ b/src/whatsapp/abstract/abstract.cache.ts @@ -0,0 +1,13 @@ +export interface ICache { + get(key: string): Promise; + + set(key: string, value: any, ttl?: number): void; + + has(key: string): Promise; + + keys(appendCriteria?: string): Promise; + + delete(key: string | string[]): Promise; + + deleteAll(appendCriteria?: string): Promise; +} diff --git a/src/whatsapp/controllers/chatwoot.controller.ts b/src/whatsapp/controllers/chatwoot.controller.ts index 8f59ccac..2de472ff 100644 --- a/src/whatsapp/controllers/chatwoot.controller.ts +++ b/src/whatsapp/controllers/chatwoot.controller.ts @@ -3,9 +3,11 @@ import { isURL } from 'class-validator'; import { ConfigService, HttpServer } from '../../config/env.config'; import { Logger } from '../../config/logger.config'; import { BadRequestException } from '../../exceptions'; +import { CacheEngine } from '../../libs/cacheengine'; import { ChatwootDto } from '../dto/chatwoot.dto'; import { InstanceDto } from '../dto/instance.dto'; import { RepositoryBroker } from '../repository/repository.manager'; +import { CacheService } from '../services/cache.service'; import { ChatwootService } from '../services/chatwoot.service'; import { waMonitor } from '../whatsapp.module'; @@ -94,7 +96,9 @@ export class ChatwootController { public async receiveWebhook(instance: InstanceDto, data: any) { logger.verbose('requested receiveWebhook from ' + instance.instanceName + ' instance'); - const chatwootService = new ChatwootService(waMonitor, this.configService, this.repository); + + const chatwootCache = new CacheService(new CacheEngine(this.configService, ChatwootService.name).getEngine()); + const chatwootService = new ChatwootService(waMonitor, this.configService, this.repository, chatwootCache); return chatwootService.receiveWebhook(instance, data); } diff --git a/src/whatsapp/controllers/instance.controller.ts b/src/whatsapp/controllers/instance.controller.ts index 3c125efc..8bba80aa 100644 --- a/src/whatsapp/controllers/instance.controller.ts +++ b/src/whatsapp/controllers/instance.controller.ts @@ -10,6 +10,7 @@ import { RedisCache } from '../../libs/redis.client'; import { InstanceDto } from '../dto/instance.dto'; import { RepositoryBroker } from '../repository/repository.manager'; import { AuthService, OldToken } from '../services/auth.service'; +import { CacheService } from '../services/cache.service'; import { ChatwootService } from '../services/chatwoot.service'; import { WAMonitoringService } from '../services/monitor.service'; import { RabbitmqService } from '../services/rabbitmq.service'; @@ -36,6 +37,7 @@ export class InstanceController { private readonly sqsService: SqsService, private readonly typebotService: TypebotService, private readonly cache: RedisCache, + private readonly chatwootCache: CacheService, ) {} private readonly logger = new Logger(InstanceController.name); @@ -82,7 +84,13 @@ export class InstanceController { await this.authService.checkDuplicateToken(token); this.logger.verbose('creating instance'); - const instance = new WAStartupService(this.configService, this.eventEmitter, this.repository, this.cache); + const instance = new WAStartupService( + this.configService, + this.eventEmitter, + this.repository, + this.cache, + this.chatwootCache, + ); instance.instanceName = instanceName; const instanceId = v4(); diff --git a/src/whatsapp/services/cache.service.ts b/src/whatsapp/services/cache.service.ts index 8a77b79b..0db39a44 100644 --- a/src/whatsapp/services/cache.service.ts +++ b/src/whatsapp/services/cache.service.ts @@ -1,39 +1,62 @@ -import NodeCache from 'node-cache'; - import { Logger } from '../../config/logger.config'; +import { ICache } from '../abstract/abstract.cache'; export class CacheService { private readonly logger = new Logger(CacheService.name); - constructor(private module: string) {} - - static localCache = new NodeCache({ - stdTTL: 12 * 60 * 60, - }); - - public get(key: string) { - return CacheService.localCache.get(`${this.module}-${key}`); + constructor(private readonly cache: ICache) { + if (cache) { + this.logger.verbose(`cacheservice created using cache engine: ${cache.constructor?.name}`); + } else { + this.logger.verbose(`cacheservice disabled`); + } } - public set(key: string, value) { - return CacheService.localCache.set(`${this.module}-${key}`, value); + async get(key: string): Promise { + if (!this.cache) { + return; + } + this.logger.verbose(`cacheservice getting key: ${key}`); + return this.cache.get(key); } - public has(key: string) { - return CacheService.localCache.has(`${this.module}-${key}`); + async set(key: string, value: any) { + if (!this.cache) { + return; + } + this.logger.verbose(`cacheservice setting key: ${key}`); + this.cache.set(key, value); } - public delete(key: string) { - return CacheService.localCache.del(`${this.module}-${key}`); + async has(key: string) { + if (!this.cache) { + return; + } + this.logger.verbose(`cacheservice has key: ${key}`); + return this.cache.has(key); } - public deleteAll() { - const keys = CacheService.localCache.keys().filter((key) => key.substring(0, this.module.length) === this.module); - - return CacheService.localCache.del(keys); + async delete(key: string) { + if (!this.cache) { + return; + } + this.logger.verbose(`cacheservice deleting key: ${key}`); + return this.cache.delete(key); } - public keys() { - return CacheService.localCache.keys(); + async deleteAll(appendCriteria?: string) { + if (!this.cache) { + return; + } + this.logger.verbose(`cacheservice deleting all keys`); + return this.cache.deleteAll(appendCriteria); + } + + async keys(appendCriteria?: string) { + if (!this.cache) { + return; + } + this.logger.verbose(`cacheservice getting all keys`); + return this.cache.keys(appendCriteria); } } diff --git a/src/whatsapp/services/chatwoot.service.ts b/src/whatsapp/services/chatwoot.service.ts index bf22892b..ad1e9ef5 100644 --- a/src/whatsapp/services/chatwoot.service.ts +++ b/src/whatsapp/services/chatwoot.service.ts @@ -8,31 +8,31 @@ import path from 'path'; import { ConfigService, HttpServer } from '../../config/env.config'; import { Logger } from '../../config/logger.config'; +import { ICache } from '../abstract/abstract.cache'; import { ChatwootDto } from '../dto/chatwoot.dto'; import { InstanceDto } from '../dto/instance.dto'; import { Options, Quoted, SendAudioDto, SendMediaDto, SendTextDto } from '../dto/sendMessage.dto'; import { ChatwootRaw, MessageRaw } from '../models'; import { RepositoryBroker } from '../repository/repository.manager'; import { Events } from '../types/wa.types'; -import { CacheService } from './cache.service'; import { WAMonitoringService } from './monitor.service'; export class ChatwootService { private readonly logger = new Logger(ChatwootService.name); private provider: any; - private cache = new CacheService(ChatwootService.name); constructor( private readonly waMonitor: WAMonitoringService, private readonly configService: ConfigService, private readonly repository: RepositoryBroker, + private readonly cache: ICache, ) {} private async getProvider(instance: InstanceDto) { - const cacheKey = `getProvider-${instance.instanceName}`; - if (this.cache.has(cacheKey)) { - return this.cache.get(cacheKey) as ChatwootRaw; + const cacheKey = `${instance.instanceName}:getProvider`; + if (await this.cache.has(cacheKey)) { + return (await this.cache.get(cacheKey)) as ChatwootRaw; } this.logger.verbose('get provider to instance: ' + instance.instanceName); @@ -69,11 +69,6 @@ export class ChatwootService { this.provider = provider; - const cacheKey = `clientCw-${instance.instanceName}`; - if (this.cache.has(cacheKey)) { - return this.cache.get(cacheKey) as ChatwootClient; - } - this.logger.verbose('create client to instance: ' + instance.instanceName); const client = new ChatwootClient({ config: { @@ -86,8 +81,6 @@ export class ChatwootService { this.logger.verbose('client created'); - this.cache.set(cacheKey, client); - return client; } @@ -409,9 +402,9 @@ export class ChatwootService { return null; } - const cacheKey = `createConversation-${instance.instanceName}-${body.key.remoteJid}`; - if (this.cache.has(cacheKey)) { - const conversationId = this.cache.get(cacheKey) as number; + const cacheKey = `${instance.instanceName}:createConversation-${body.key.remoteJid}`; + if (await this.cache.has(cacheKey)) { + const conversationId = (await this.cache.get(cacheKey)) as number; let conversationExists: conversation | boolean; try { conversationExists = await client.conversations.get({ @@ -615,9 +608,9 @@ export class ChatwootService { public async getInbox(instance: InstanceDto) { this.logger.verbose('get inbox to instance: ' + instance.instanceName); - const cacheKey = `getInbox-${instance.instanceName}`; - if (this.cache.has(cacheKey)) { - return this.cache.get(cacheKey) as inbox; + const cacheKey = `${instance.instanceName}:getInbox`; + if (await this.cache.has(cacheKey)) { + return (await this.cache.get(cacheKey)) as inbox; } const client = await this.clientCw(instance); @@ -1044,7 +1037,7 @@ export class ChatwootService { body.status === 'resolved' && body.meta?.sender?.identifier ) { - const keyToDelete = `createConversation-${instance.instanceName}-${body.meta.sender.identifier}`; + const keyToDelete = `${instance.instanceName}:createConversation-${body.meta.sender.identifier}`; this.cache.delete(keyToDelete); } diff --git a/src/whatsapp/services/monitor.service.ts b/src/whatsapp/services/monitor.service.ts index ecf70586..3c3e8881 100644 --- a/src/whatsapp/services/monitor.service.ts +++ b/src/whatsapp/services/monitor.service.ts @@ -26,6 +26,7 @@ import { WebsocketModel, } from '../models'; import { RepositoryBroker } from '../repository/repository.manager'; +import { CacheService } from './cache.service'; import { WAStartupService } from './whatsapp.service'; export class WAMonitoringService { @@ -34,6 +35,7 @@ export class WAMonitoringService { private readonly configService: ConfigService, private readonly repository: RepositoryBroker, private readonly cache: RedisCache, + private readonly chatwootCache: CacheService, ) { this.logger.verbose('instance created'); @@ -359,7 +361,13 @@ export class WAMonitoringService { } private async setInstance(name: string) { - const instance = new WAStartupService(this.configService, this.eventEmitter, this.repository, this.cache); + const instance = new WAStartupService( + this.configService, + this.eventEmitter, + this.repository, + this.cache, + this.chatwootCache, + ); instance.instanceName = name; this.logger.verbose('Instance loaded: ' + name); await instance.connectToWhatsapp(); diff --git a/src/whatsapp/services/whatsapp.service.ts b/src/whatsapp/services/whatsapp.service.ts index c6aa06b6..efe4e2a3 100644 --- a/src/whatsapp/services/whatsapp.service.ts +++ b/src/whatsapp/services/whatsapp.service.ts @@ -131,6 +131,7 @@ import { MessageUpQuery } from '../repository/messageUp.repository'; import { RepositoryBroker } from '../repository/repository.manager'; import { Events, MessageSubtype, TypeMediaMessage, wa } from '../types/wa.types'; import { waMonitor } from '../whatsapp.module'; +import { CacheService } from './cache.service'; import { ChamaaiService } from './chamaai.service'; import { ChatwootService } from './chatwoot.service'; import { TypebotService } from './typebot.service'; @@ -143,6 +144,7 @@ export class WAStartupService { private readonly eventEmitter: EventEmitter2, private readonly repository: RepositoryBroker, private readonly cache: RedisCache, + private readonly chatwootCache: CacheService, ) { this.logger.verbose('WAStartupService initialized'); this.cleanStore(); @@ -170,7 +172,7 @@ export class WAStartupService { private phoneNumber: string; - private chatwootService = new ChatwootService(waMonitor, this.configService, this.repository); + private chatwootService = new ChatwootService(waMonitor, this.configService, this.repository, this.chatwootCache); private typebotService = new TypebotService(waMonitor, this.configService, this.eventEmitter); @@ -408,7 +410,7 @@ export class WAStartupService { this.logger.verbose('Removing cache from chatwoot'); if (this.localChatwoot.enabled) { - this.chatwootService.getCache().deleteAll(); + this.chatwootService.getCache()?.deleteAll(this.instanceName); } } diff --git a/src/whatsapp/whatsapp.module.ts b/src/whatsapp/whatsapp.module.ts index d459bf6a..0b5da554 100644 --- a/src/whatsapp/whatsapp.module.ts +++ b/src/whatsapp/whatsapp.module.ts @@ -1,6 +1,7 @@ import { configService } from '../config/env.config'; import { eventEmitter } from '../config/event.config'; import { Logger } from '../config/logger.config'; +import { CacheEngine } from '../libs/cacheengine'; import { dbserver } from '../libs/db.connect'; import { RedisCache } from '../libs/redis.client'; import { ChamaaiController } from './controllers/chamaai.controller'; @@ -48,6 +49,7 @@ import { TypebotRepository } from './repository/typebot.repository'; import { WebhookRepository } from './repository/webhook.repository'; import { WebsocketRepository } from './repository/websocket.repository'; import { AuthService } from './services/auth.service'; +import { CacheService } from './services/cache.service'; import { ChamaaiService } from './services/chamaai.service'; import { ChatwootService } from './services/chatwoot.service'; import { WAMonitoringService } from './services/monitor.service'; @@ -97,7 +99,9 @@ export const repository = new RepositoryBroker( export const cache = new RedisCache(); -export const waMonitor = new WAMonitoringService(eventEmitter, configService, repository, cache); +const chatwootCache = new CacheService(new CacheEngine(configService, ChatwootService.name).getEngine()); + +export const waMonitor = new WAMonitoringService(eventEmitter, configService, repository, cache, chatwootCache); const authService = new AuthService(configService, waMonitor, repository); @@ -129,7 +133,7 @@ const sqsService = new SqsService(waMonitor); export const sqsController = new SqsController(sqsService); -const chatwootService = new ChatwootService(waMonitor, configService, repository); +const chatwootService = new ChatwootService(waMonitor, configService, repository, chatwootCache); export const chatwootController = new ChatwootController(chatwootService, configService, repository); @@ -151,6 +155,7 @@ export const instanceController = new InstanceController( sqsService, typebotService, cache, + chatwootCache, ); export const sendMessageController = new SendMessageController(waMonitor); export const chatController = new ChatController(waMonitor);