From 696261d74981052918f0b001562b6527143f93a7 Mon Sep 17 00:00:00 2001 From: Davidson Gomes Date: Sat, 1 Jun 2024 09:54:56 -0300 Subject: [PATCH 01/13] New method of saving sessions to a file using worker --- CHANGELOG.md | 5 + Docker/.env.example | 7 +- src/api/controllers/instance.controller.ts | 4 + src/api/provider/sessions.ts | 140 ++++++++++++++++++ src/api/server.module.ts | 4 + .../channels/whatsapp.baileys.service.ts | 21 ++- .../channels/whatsapp.business.service.ts | 2 + src/api/services/monitor.service.ts | 4 + src/config/env.config.ts | 14 ++ src/dev-env.yml | 8 + src/main.ts | 7 +- ...se-multi-file-auth-state-provider-files.ts | 140 ++++++++++++++++++ 12 files changed, 349 insertions(+), 7 deletions(-) create mode 100644 src/api/provider/sessions.ts create mode 100644 src/utils/use-multi-file-auth-state-provider-files.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 06379d80..0b96f10d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +# 1.8.1 (develop) + +### Feature +* New method of saving sessions to a file using worker, made in partnership with [codechat](https://github.com/code-chat-br/whatsapp-api) + # 1.8.0 (2024-05-27 16:10) ### Feature diff --git a/Docker/.env.example b/Docker/.env.example index e735d8de..1af5e1cd 100644 --- a/Docker/.env.example +++ b/Docker/.env.example @@ -33,10 +33,7 @@ CLEAN_STORE_CHATS=true # Permanent data storage DATABASE_ENABLED=false -DATABASE_CONNECTION_URI=mongodb://root:root@mongodb:27017/?authSource=admin & -readPreference=primary & -ssl=false & -directConnection=true +DATABASE_CONNECTION_URI=mongodb://root:root@mongodb:27017/?authSource=admin&readPreference=primary&ssl=false&directConnection=true DATABASE_CONNECTION_DB_PREFIX_NAME=evdocker # Choose the data you want to save in the application's database or store @@ -137,7 +134,7 @@ CONFIG_SESSION_PHONE_NAME=Chrome # Set qrcode display limit QRCODE_LIMIT=30 -QRCODE_COLOR=#198754 +QRCODE_COLOR='#198754' # old | latest TYPEBOT_API_VERSION=latest diff --git a/src/api/controllers/instance.controller.ts b/src/api/controllers/instance.controller.ts index 0ead0be1..90d9622a 100644 --- a/src/api/controllers/instance.controller.ts +++ b/src/api/controllers/instance.controller.ts @@ -12,6 +12,7 @@ import { RabbitmqService } from '../integrations/rabbitmq/services/rabbitmq.serv import { SqsService } from '../integrations/sqs/services/sqs.service'; import { TypebotService } from '../integrations/typebot/services/typebot.service'; import { WebsocketService } from '../integrations/websocket/services/websocket.service'; +import { ProviderFiles } from '../provider/sessions'; import { RepositoryBroker } from '../repository/repository.manager'; import { AuthService, OldToken } from '../services/auth.service'; import { CacheService } from '../services/cache.service'; @@ -43,6 +44,7 @@ export class InstanceController { private readonly cache: CacheService, private readonly chatwootCache: CacheService, private readonly messagesLostCache: CacheService, + private readonly providerFiles: ProviderFiles, ) {} private readonly logger = new Logger(InstanceController.name); @@ -111,6 +113,7 @@ export class InstanceController { this.cache, this.chatwootCache, this.messagesLostCache, + this.providerFiles, ); } else { instance = new BaileysStartupService( @@ -120,6 +123,7 @@ export class InstanceController { this.cache, this.chatwootCache, this.messagesLostCache, + this.providerFiles, ); } diff --git a/src/api/provider/sessions.ts b/src/api/provider/sessions.ts new file mode 100644 index 00000000..e9628823 --- /dev/null +++ b/src/api/provider/sessions.ts @@ -0,0 +1,140 @@ +import axios from 'axios'; +import { execSync } from 'child_process'; + +import { ConfigService, ProviderSession } from '../../config/env.config'; +import { Logger } from '../../config/logger.config'; + +type ResponseSuccess = { status: number; data?: any }; +type ResponseProvider = Promise<[ResponseSuccess?, Error?]>; + +export class ProviderFiles { + constructor(private readonly configService: ConfigService) { + this.baseUrl = `http://${this.config.HOST}:${this.config.PORT}/session`; + } + + private readonly logger = new Logger(ProviderFiles.name); + + private baseUrl: string; + + private readonly config = Object.freeze(this.configService.get('PROVIDER')); + + private readonly prefix = Object.freeze(this.configService.get('PROVIDER').PREFIX); + + get isEnabled() { + return !!this.config?.ENABLED; + } + + public async onModuleInit() { + if (this.config.ENABLED) { + const client = axios.create({ + baseURL: this.baseUrl, + }); + try { + const response = await client.options('/ping'); + if (!response?.data?.pong) { + throw new Error('Offline file provider.'); + } + } catch (error) { + this.logger.error(['Failed to connect to the file server', error?.message, error?.stack]); + const pid = process.pid; + execSync(`kill -9 ${pid}`); + } + } + } + + public async onModuleDestroy() { + // + } + + public async create(instance: string): ResponseProvider { + try { + const response = await axios.post(`${this.baseUrl}/${this.prefix}`, { + instance, + }); + return [{ status: response.status, data: response?.data }]; + } catch (error) { + return [ + { + status: error?.response?.status, + data: error?.response?.data, + }, + error, + ]; + } + } + + public async write(instance: string, key: string, data: any): ResponseProvider { + try { + const response = await axios.post(`${this.baseUrl}/${this.prefix}/${instance}/${key}`, data); + return [{ status: response.status, data: response?.data }]; + } catch (error) { + return [ + { + status: error?.response?.status, + data: error?.response?.data, + }, + error, + ]; + } + } + + public async read(instance: string, key: string): ResponseProvider { + try { + const response = await axios.get(`${this.baseUrl}/${this.prefix}/${instance}/${key}`); + return [{ status: response.status, data: response?.data }]; + } catch (error) { + return [ + { + status: error?.response?.status, + data: error?.response?.data, + }, + error, + ]; + } + } + + public async delete(instance: string, key: string): ResponseProvider { + try { + const response = await axios.delete(`${this.baseUrl}/${this.prefix}/${instance}/${key}`); + return [{ status: response.status, data: response?.data }]; + } catch (error) { + return [ + { + status: error?.response?.status, + data: error?.response?.data, + }, + error, + ]; + } + } + + public async allInstances(): ResponseProvider { + try { + const response = await axios.get(`${this.baseUrl}/list-instances/${this.prefix}`); + return [{ status: response.status, data: response?.data as string[] }]; + } catch (error) { + return [ + { + status: error?.response?.status, + data: error?.response?.data, + }, + error, + ]; + } + } + + public async removeSession(instance: string): ResponseProvider { + try { + const response = await axios.delete(`${this.baseUrl}/${this.prefix}/${instance}`); + return [{ status: response.status, data: response?.data }]; + } catch (error) { + return [ + { + status: error?.response?.status, + data: error?.response?.data, + }, + error, + ]; + } + } +} diff --git a/src/api/server.module.ts b/src/api/server.module.ts index 97df81a3..e282be1c 100644 --- a/src/api/server.module.ts +++ b/src/api/server.module.ts @@ -47,6 +47,7 @@ import { WebsocketModel, } from './models'; import { LabelModel } from './models/label.model'; +import { ProviderFiles } from './provider/sessions'; import { AuthRepository } from './repository/auth.repository'; import { ChatRepository } from './repository/chat.repository'; import { ContactRepository } from './repository/contact.repository'; @@ -109,6 +110,7 @@ export const repository = new RepositoryBroker( export const cache = new CacheService(new CacheEngine(configService, 'instance').getEngine()); const chatwootCache = new CacheService(new CacheEngine(configService, ChatwootService.name).getEngine()); const messagesLostCache = new CacheService(new CacheEngine(configService, 'baileys').getEngine()); +const providerFiles = new ProviderFiles(configService); export const waMonitor = new WAMonitoringService( eventEmitter, @@ -117,6 +119,7 @@ export const waMonitor = new WAMonitoringService( cache, chatwootCache, messagesLostCache, + providerFiles, ); const authService = new AuthService(configService, waMonitor, repository); @@ -168,6 +171,7 @@ export const instanceController = new InstanceController( cache, chatwootCache, messagesLostCache, + providerFiles, ); export const sendMessageController = new SendMessageController(waMonitor); export const chatController = new ChatController(waMonitor); diff --git a/src/api/services/channels/whatsapp.baileys.service.ts b/src/api/services/channels/whatsapp.baileys.service.ts index 2840d9a7..4e9ef4f6 100644 --- a/src/api/services/channels/whatsapp.baileys.service.ts +++ b/src/api/services/channels/whatsapp.baileys.service.ts @@ -55,12 +55,21 @@ import qrcode, { QRCodeToDataURLOptions } from 'qrcode'; import qrcodeTerminal from 'qrcode-terminal'; import sharp from 'sharp'; -import { CacheConf, ConfigService, ConfigSessionPhone, Database, Log, QrCode } from '../../../config/env.config'; +import { + CacheConf, + ConfigService, + ConfigSessionPhone, + Database, + Log, + ProviderSession, + QrCode, +} from '../../../config/env.config'; import { INSTANCE_DIR } from '../../../config/path.config'; import { BadRequestException, InternalServerErrorException, NotFoundException } from '../../../exceptions'; import { dbserver } from '../../../libs/db.connect'; import { makeProxyAgent } from '../../../utils/makeProxyAgent'; import { useMultiFileAuthStateDb } from '../../../utils/use-multi-file-auth-state-db'; +import { AuthStateProvider } from '../../../utils/use-multi-file-auth-state-provider-files'; import { useMultiFileAuthStateRedisDb } from '../../../utils/use-multi-file-auth-state-redis-db'; import { ArchiveChatDto, @@ -114,6 +123,7 @@ import { SettingsRaw } from '../../models'; import { ChatRaw } from '../../models/chat.model'; import { ContactRaw } from '../../models/contact.model'; import { MessageRaw, MessageUpdateRaw } from '../../models/message.model'; +import { ProviderFiles } from '../../provider/sessions'; import { RepositoryBroker } from '../../repository/repository.manager'; import { waMonitor } from '../../server.module'; import { Events, MessageSubtype, TypeMediaMessage, wa } from '../../types/wa.types'; @@ -128,6 +138,7 @@ export class BaileysStartupService extends ChannelStartupService { public readonly cache: CacheService, public readonly chatwootCache: CacheService, public readonly messagesLostCache: CacheService, + private readonly providerFiles: ProviderFiles, ) { super(configService, eventEmitter, repository, chatwootCache); this.logger.verbose('BaileysStartupService initialized'); @@ -135,8 +146,10 @@ export class BaileysStartupService extends ChannelStartupService { this.instance.qrcode = { count: 0 }; this.mobile = false; this.recoveringMessages(); + this.authStateProvider = new AuthStateProvider(this.configService, this.providerFiles); } + private authStateProvider: AuthStateProvider; private readonly msgRetryCounterCache: CacheStore = new NodeCache(); private readonly userDevicesCache: CacheStore = new NodeCache(); private endSession = false; @@ -461,6 +474,12 @@ export class BaileysStartupService extends ChannelStartupService { const db = this.configService.get('DATABASE'); const cache = this.configService.get('CACHE'); + const provider = this.configService.get('PROVIDER'); + + if (provider?.ENABLED) { + return await this.authStateProvider.authStateProvider(this.instance.name); + } + if (cache?.REDIS.ENABLED && cache?.REDIS.SAVE_INSTANCES) { this.logger.info('Redis enabled'); return await useMultiFileAuthStateRedisDb(this.instance.name, this.cache); diff --git a/src/api/services/channels/whatsapp.business.service.ts b/src/api/services/channels/whatsapp.business.service.ts index 09ddd2a0..e44f36c7 100644 --- a/src/api/services/channels/whatsapp.business.service.ts +++ b/src/api/services/channels/whatsapp.business.service.ts @@ -23,6 +23,7 @@ import { SendTextDto, } from '../../dto/sendMessage.dto'; import { ContactRaw, MessageRaw, MessageUpdateRaw, SettingsRaw } from '../../models'; +import { ProviderFiles } from '../../provider/sessions'; import { RepositoryBroker } from '../../repository/repository.manager'; import { Events, wa } from '../../types/wa.types'; import { CacheService } from './../cache.service'; @@ -36,6 +37,7 @@ export class BusinessStartupService extends ChannelStartupService { public readonly cache: CacheService, public readonly chatwootCache: CacheService, public readonly messagesLostCache: CacheService, + private readonly providerFiles: ProviderFiles, ) { super(configService, eventEmitter, repository, chatwootCache); this.logger.verbose('BusinessStartupService initialized'); diff --git a/src/api/services/monitor.service.ts b/src/api/services/monitor.service.ts index af93fa74..51ccc201 100644 --- a/src/api/services/monitor.service.ts +++ b/src/api/services/monitor.service.ts @@ -22,6 +22,7 @@ import { WebhookModel, WebsocketModel, } from '../models'; +import { ProviderFiles } from '../provider/sessions'; import { RepositoryBroker } from '../repository/repository.manager'; import { Integration } from '../types/wa.types'; import { CacheService } from './cache.service'; @@ -36,6 +37,7 @@ export class WAMonitoringService { private readonly cache: CacheService, private readonly chatwootCache: CacheService, private readonly messagesLostCache: CacheService, + private readonly providerFiles: ProviderFiles, ) { this.logger.verbose('instance created'); @@ -349,6 +351,7 @@ export class WAMonitoringService { this.cache, this.chatwootCache, this.messagesLostCache, + this.providerFiles, ); instance.instanceName = name; @@ -360,6 +363,7 @@ export class WAMonitoringService { this.cache, this.chatwootCache, this.messagesLostCache, + this.providerFiles, ); instance.instanceName = name; diff --git a/src/config/env.config.ts b/src/config/env.config.ts index eab883f7..ddd5ce9f 100644 --- a/src/config/env.config.ts +++ b/src/config/env.config.ts @@ -28,6 +28,13 @@ export type Log = { BAILEYS: LogBaileys; }; +export type ProviderSession = { + ENABLED: boolean; + HOST: string; + PORT: string; + PREFIX: string; +}; + export type SaveData = { INSTANCE: boolean; NEW_MESSAGE: boolean; @@ -209,6 +216,7 @@ export interface Env { SERVER: HttpServer; CORS: Cors; SSL_CONF: SslConf; + PROVIDER: ProviderSession; STORE: StoreConf; CLEAN_STORE: CleanStoreConf; DATABASE: Database; @@ -274,6 +282,12 @@ export class ConfigService { PRIVKEY: process.env?.SSL_CONF_PRIVKEY || '', FULLCHAIN: process.env?.SSL_CONF_FULLCHAIN || '', }, + PROVIDER: { + ENABLED: process.env?.PROVIDER_ENABLED === 'true', + HOST: process.env.PROVIDER_HOST, + PORT: process.env?.PROVIDER_PORT || '5656', + PREFIX: process.env?.PROVIDER_PREFIX || 'evolution', + }, STORE: { MESSAGES: process.env?.STORE_MESSAGES === 'true', MESSAGE_UP: process.env?.STORE_MESSAGE_UP === 'true', diff --git a/src/dev-env.yml b/src/dev-env.yml index 23e1b479..42573ef3 100644 --- a/src/dev-env.yml +++ b/src/dev-env.yml @@ -49,6 +49,14 @@ LOG: DEL_INSTANCE: false # or false DEL_TEMP_INSTANCES: true # Delete instances with status closed on start +# Seesion Files Providers +# Provider responsible for managing credentials files and WhatsApp sessions. +PROVIDER: + ENABLED: true + HOST: 127.0.0.1 + PORT: 5656 + PREFIX: evolution + # Temporary data storage STORE: MESSAGES: true diff --git a/src/main.ts b/src/main.ts index 815e2a11..2cc9e280 100644 --- a/src/main.ts +++ b/src/main.ts @@ -9,6 +9,7 @@ import { join } from 'path'; import { initAMQP, initGlobalQueues } from './api/integrations/rabbitmq/libs/amqp.server'; import { initSQS } from './api/integrations/sqs/libs/sqs.server'; import { initIO } from './api/integrations/websocket/libs/socket.server'; +import { ProviderFiles } from './api/provider/sessions'; import { HttpStatus, router } from './api/routes/index.router'; import { waMonitor } from './api/server.module'; import { Auth, configService, Cors, HttpServer, Rabbitmq, Sqs, Webhook } from './config/env.config'; @@ -22,10 +23,14 @@ function initWA() { waMonitor.loadInstance(); } -function bootstrap() { +async function bootstrap() { const logger = new Logger('SERVER'); const app = express(); + const providerFiles = new ProviderFiles(configService); + await providerFiles.onModuleInit(); + logger.info('Provider:Files - ON'); + app.use( cors({ origin(requestOrigin, callback) { diff --git a/src/utils/use-multi-file-auth-state-provider-files.ts b/src/utils/use-multi-file-auth-state-provider-files.ts new file mode 100644 index 00000000..89ea4767 --- /dev/null +++ b/src/utils/use-multi-file-auth-state-provider-files.ts @@ -0,0 +1,140 @@ +/** + * ┌──────────────────────────────────────────────────────────────────────────────┐ + * │ @author jrCleber │ + * │ @filename use-multi-file-auth-state-redis-db.ts │ + * │ Developed by: Cleber Wilson │ + * │ Creation date: Apr 09, 2023 │ + * │ Contact: contato@codechat.dev │ + * ├──────────────────────────────────────────────────────────────────────────────┤ + * │ @copyright © Cleber Wilson 2023. All rights reserved. │ + * │ Licensed under the Apache License, Version 2.0 │ + * │ │ + * │ @license "https://github.com/code-chat-br/whatsapp-api/blob/main/LICENSE" │ + * │ │ + * │ You may not use this file except in compliance with the License. │ + * │ You may obtain a copy of the License at │ + * │ │ + * │ http://www.apache.org/licenses/LICENSE-2.0 │ + * │ │ + * │ Unless required by applicable law or agreed to in writing, software │ + * │ distributed under the License is distributed on an "AS IS" BASIS, │ + * │ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. │ + * │ │ + * │ See the License for the specific language governing permissions and │ + * │ limitations under the License. │ + * │ │ + * │ @type {AuthState} │ + * │ @function useMultiFileAuthStateRedisDb │ + * │ @returns {Promise} │ + * ├──────────────────────────────────────────────────────────────────────────────┤ + * │ @important │ + * │ For any future changes to the code in this file, it is recommended to │ + * │ contain, together with the modification, the information of the developer │ + * │ who changed it and the date of modification. │ + * └──────────────────────────────────────────────────────────────────────────────┘ + */ + +import { + AuthenticationCreds, + AuthenticationState, + BufferJSON, + initAuthCreds, + proto, + SignalDataTypeMap, +} from '@whiskeysockets/baileys'; +import { isNotEmpty } from 'class-validator'; + +import { ProviderFiles } from '../api/provider/sessions'; +import { ConfigService } from '../config/env.config'; +import { Logger } from '../config/logger.config'; + +export type AuthState = { state: AuthenticationState; saveCreds: () => Promise }; + +export class AuthStateProvider { + constructor(private readonly configService: ConfigService, private readonly providerFiles: ProviderFiles) {} + + private readonly logger = new Logger(AuthStateProvider.name); + + public async authStateProvider(instance: string): Promise { + const [, error] = await this.providerFiles.create(instance); + if (error) { + this.logger.error(['Failed to create folder on file server', error?.message, error?.stack]); + return; + } + + const writeData = async (data: any, key: string): Promise => { + const json = JSON.stringify(data, BufferJSON.replacer); + const [response, error] = await this.providerFiles.write(instance, key, { + data: json, + }); + if (error) { + this.logger.error([error?.message, error?.stack]); + return; + } + return response; + }; + + const readData = async (key: string): Promise => { + const [response, error] = await this.providerFiles.read(instance, key); + if (error) { + this.logger.error([error?.message, error?.stack]); + return; + } + if (isNotEmpty(response?.data)) { + return JSON.parse(JSON.stringify(response.data), BufferJSON.reviver); + } + }; + + const removeData = async (key: string) => { + const [response, error] = await this.providerFiles.delete(instance, key); + if (error) { + this.logger.error([error?.message, error?.stack]); + return; + } + + return response; + }; + + const creds: AuthenticationCreds = (await readData('creds')) || initAuthCreds(); + + return { + state: { + creds, + keys: { + get: async (type, ids: string[]) => { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + const data: { [_: string]: SignalDataTypeMap[type] } = {}; + await Promise.all( + ids.map(async (id) => { + let value = await readData(`${type}-${id}`); + if (type === 'app-state-sync-key' && value) { + value = proto.Message.AppStateSyncKeyData.fromObject(value); + } + + data[id] = value; + }), + ); + + return data; + }, + set: async (data: any) => { + const tasks: Promise[] = []; + for (const category in data) { + for (const id in data[category]) { + const value = data[category][id]; + const key = `${category}-${id}`; + tasks.push(value ? await writeData(value, key) : await removeData(key)); + } + } + + await Promise.all(tasks); + }, + }, + }, + saveCreds: async () => { + return await writeData(creds, 'creds'); + }, + }; + } +} From f48f331d43b1dfcde6ad736964ca7e7d5ca86fc0 Mon Sep 17 00:00:00 2001 From: Davidson Gomes Date: Sat, 1 Jun 2024 13:28:29 -0300 Subject: [PATCH 02/13] New method of saving sessions to a file using worker --- src/api/provider/sessions.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/provider/sessions.ts b/src/api/provider/sessions.ts index e9628823..d3e63291 100644 --- a/src/api/provider/sessions.ts +++ b/src/api/provider/sessions.ts @@ -30,7 +30,7 @@ export class ProviderFiles { baseURL: this.baseUrl, }); try { - const response = await client.options('/ping'); + const response = await client.options(`/${this.prefix}/ping`); if (!response?.data?.pong) { throw new Error('Offline file provider.'); } @@ -110,7 +110,7 @@ export class ProviderFiles { public async allInstances(): ResponseProvider { try { - const response = await axios.get(`${this.baseUrl}/list-instances/${this.prefix}`); + const response = await axios.get(`${this.baseUrl}/${this.prefix}/list-instances`); return [{ status: response.status, data: response?.data as string[] }]; } catch (error) { return [ From 9354af3bc71a03b0d4d30dd9b4a42c8529e8e794 Mon Sep 17 00:00:00 2001 From: Davidson Gomes Date: Sat, 1 Jun 2024 19:20:59 -0300 Subject: [PATCH 03/13] Adjust to repository from session worker --- package.json | 2 +- src/api/provider/sessions.ts | 18 ++++----- .../channels/whatsapp.baileys.service.ts | 4 +- src/api/services/monitor.service.ts | 40 +++++++++++++++++-- ...se-multi-file-auth-state-provider-files.ts | 13 +++--- 5 files changed, 53 insertions(+), 24 deletions(-) diff --git a/package.json b/package.json index b6f78480..8b847e18 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,7 @@ "amqplib": "^0.10.3", "@aws-sdk/client-sqs": "^3.569.0", "axios": "^1.6.5", - "@whiskeysockets/baileys": "^6.7.2", + "@whiskeysockets/baileys": "6.6.0", "class-validator": "^0.14.1", "compression": "^1.7.4", "cors": "^2.8.5", diff --git a/src/api/provider/sessions.ts b/src/api/provider/sessions.ts index d3e63291..6997d9ff 100644 --- a/src/api/provider/sessions.ts +++ b/src/api/provider/sessions.ts @@ -9,7 +9,7 @@ type ResponseProvider = Promise<[ResponseSuccess?, Error?]>; export class ProviderFiles { constructor(private readonly configService: ConfigService) { - this.baseUrl = `http://${this.config.HOST}:${this.config.PORT}/session`; + this.baseUrl = `http://${this.config.HOST}:${this.config.PORT}/session/${this.config.PREFIX}`; } private readonly logger = new Logger(ProviderFiles.name); @@ -18,8 +18,6 @@ export class ProviderFiles { private readonly config = Object.freeze(this.configService.get('PROVIDER')); - private readonly prefix = Object.freeze(this.configService.get('PROVIDER').PREFIX); - get isEnabled() { return !!this.config?.ENABLED; } @@ -30,7 +28,7 @@ export class ProviderFiles { baseURL: this.baseUrl, }); try { - const response = await client.options(`/${this.prefix}/ping`); + const response = await client.options('/ping'); if (!response?.data?.pong) { throw new Error('Offline file provider.'); } @@ -48,7 +46,7 @@ export class ProviderFiles { public async create(instance: string): ResponseProvider { try { - const response = await axios.post(`${this.baseUrl}/${this.prefix}`, { + const response = await axios.post(`${this.baseUrl}`, { instance, }); return [{ status: response.status, data: response?.data }]; @@ -65,7 +63,7 @@ export class ProviderFiles { public async write(instance: string, key: string, data: any): ResponseProvider { try { - const response = await axios.post(`${this.baseUrl}/${this.prefix}/${instance}/${key}`, data); + const response = await axios.post(`${this.baseUrl}/${instance}/${key}`, data); return [{ status: response.status, data: response?.data }]; } catch (error) { return [ @@ -80,7 +78,7 @@ export class ProviderFiles { public async read(instance: string, key: string): ResponseProvider { try { - const response = await axios.get(`${this.baseUrl}/${this.prefix}/${instance}/${key}`); + const response = await axios.get(`${this.baseUrl}/${instance}/${key}`); return [{ status: response.status, data: response?.data }]; } catch (error) { return [ @@ -95,7 +93,7 @@ export class ProviderFiles { public async delete(instance: string, key: string): ResponseProvider { try { - const response = await axios.delete(`${this.baseUrl}/${this.prefix}/${instance}/${key}`); + const response = await axios.delete(`${this.baseUrl}/${instance}/${key}`); return [{ status: response.status, data: response?.data }]; } catch (error) { return [ @@ -110,7 +108,7 @@ export class ProviderFiles { public async allInstances(): ResponseProvider { try { - const response = await axios.get(`${this.baseUrl}/${this.prefix}/list-instances`); + const response = await axios.get(`${this.baseUrl}/list-instances`); return [{ status: response.status, data: response?.data as string[] }]; } catch (error) { return [ @@ -125,7 +123,7 @@ export class ProviderFiles { public async removeSession(instance: string): ResponseProvider { try { - const response = await axios.delete(`${this.baseUrl}/${this.prefix}/${instance}`); + const response = await axios.delete(`${this.baseUrl}/${instance}`); return [{ status: response.status, data: response?.data }]; } catch (error) { return [ diff --git a/src/api/services/channels/whatsapp.baileys.service.ts b/src/api/services/channels/whatsapp.baileys.service.ts index 4e9ef4f6..911f4c66 100644 --- a/src/api/services/channels/whatsapp.baileys.service.ts +++ b/src/api/services/channels/whatsapp.baileys.service.ts @@ -146,7 +146,7 @@ export class BaileysStartupService extends ChannelStartupService { this.instance.qrcode = { count: 0 }; this.mobile = false; this.recoveringMessages(); - this.authStateProvider = new AuthStateProvider(this.configService, this.providerFiles); + this.authStateProvider = new AuthStateProvider(this.providerFiles); } private authStateProvider: AuthStateProvider; @@ -1486,7 +1486,7 @@ export class BaileysStartupService extends ChannelStartupService { }); const chat = chats.find((c) => c.id === data.association.chatId); if (chat) { - let labels = [...chat.labels]; + let labels = [...chat?.labels]; if (data.type === 'remove') { labels = labels.filter((label) => label !== data.association.labelId); } else if (data.type === 'add') { diff --git a/src/api/services/monitor.service.ts b/src/api/services/monitor.service.ts index 51ccc201..c197b843 100644 --- a/src/api/services/monitor.service.ts +++ b/src/api/services/monitor.service.ts @@ -5,7 +5,15 @@ import { Db } from 'mongodb'; import { Collection } from 'mongoose'; import { join } from 'path'; -import { Auth, CacheConf, ConfigService, Database, DelInstance, HttpServer } from '../../config/env.config'; +import { + Auth, + CacheConf, + ConfigService, + Database, + DelInstance, + HttpServer, + ProviderSession, +} from '../../config/env.config'; import { Logger } from '../../config/logger.config'; import { INSTANCE_DIR, STORE_DIR } from '../../config/path.config'; import { NotFoundException } from '../../exceptions'; @@ -60,6 +68,8 @@ export class WAMonitoringService { private readonly logger = new Logger(WAMonitoringService.name); public readonly waInstances: Record = {}; + private readonly providerSession = Object.freeze(this.configService.get('PROVIDER')); + public delInstanceTime(instance: string) { const time = this.configService.get('DEL_INSTANCE'); if (typeof time === 'number' && time > 0) { @@ -259,13 +269,21 @@ export class WAMonitoringService { } this.logger.verbose('cleaning up instance in files: ' + instanceName); - rmSync(join(INSTANCE_DIR, instanceName), { recursive: true, force: true }); + if (this.providerSession?.ENABLED) { + await this.providerFiles.removeSession(instanceName); + } else { + rmSync(join(INSTANCE_DIR, instanceName), { recursive: true, force: true }); + } } public async cleaningStoreFiles(instanceName: string) { if (!this.db.ENABLED) { this.logger.verbose('cleaning store files instance: ' + instanceName); - rmSync(join(INSTANCE_DIR, instanceName), { recursive: true, force: true }); + if (this.providerSession?.ENABLED) { + await this.providerFiles.removeSession(instanceName); + } else { + rmSync(join(INSTANCE_DIR, instanceName), { recursive: true, force: true }); + } execSync(`rm -rf ${join(STORE_DIR, 'chats', instanceName)}`); execSync(`rm -rf ${join(STORE_DIR, 'contacts', instanceName)}`); @@ -307,7 +325,9 @@ export class WAMonitoringService { this.logger.verbose('Loading instances'); try { - if (this.redis.REDIS.ENABLED && this.redis.REDIS.SAVE_INSTANCES) { + if (this.providerSession.ENABLED) { + await this.loadInstancesFromProvider(); + } else if (this.redis.REDIS.ENABLED && this.redis.REDIS.SAVE_INSTANCES) { await this.loadInstancesFromRedis(); } else if (this.db.ENABLED && this.db.SAVE_DATA.INSTANCE) { await this.loadInstancesFromDatabase(); @@ -405,6 +425,18 @@ export class WAMonitoringService { } } + private async loadInstancesFromProvider() { + this.logger.verbose('Provider in files enabled'); + const [instances] = await this.providerFiles.allInstances(); + + if (!instances?.data) { + this.logger.verbose('No instances found'); + return; + } + + await Promise.all(instances?.data?.map(async (instanceName: string) => this.setInstance(instanceName))); + } + private async loadInstancesFromFiles() { this.logger.verbose('Store in files enabled'); const dir = opendirSync(INSTANCE_DIR, { encoding: 'utf-8' }); diff --git a/src/utils/use-multi-file-auth-state-provider-files.ts b/src/utils/use-multi-file-auth-state-provider-files.ts index 89ea4767..54100463 100644 --- a/src/utils/use-multi-file-auth-state-provider-files.ts +++ b/src/utils/use-multi-file-auth-state-provider-files.ts @@ -1,9 +1,9 @@ /** * ┌──────────────────────────────────────────────────────────────────────────────┐ * │ @author jrCleber │ - * │ @filename use-multi-file-auth-state-redis-db.ts │ + * │ @filename use-multi-file-auth-state-provider-files.ts │ * │ Developed by: Cleber Wilson │ - * │ Creation date: Apr 09, 2023 │ + * │ Creation date: May 31, 2024 │ * │ Contact: contato@codechat.dev │ * ├──────────────────────────────────────────────────────────────────────────────┤ * │ @copyright © Cleber Wilson 2023. All rights reserved. │ @@ -45,13 +45,12 @@ import { import { isNotEmpty } from 'class-validator'; import { ProviderFiles } from '../api/provider/sessions'; -import { ConfigService } from '../config/env.config'; import { Logger } from '../config/logger.config'; export type AuthState = { state: AuthenticationState; saveCreds: () => Promise }; export class AuthStateProvider { - constructor(private readonly configService: ConfigService, private readonly providerFiles: ProviderFiles) {} + constructor(private readonly providerFiles: ProviderFiles) {} private readonly logger = new Logger(AuthStateProvider.name); @@ -68,7 +67,7 @@ export class AuthStateProvider { data: json, }); if (error) { - this.logger.error([error?.message, error?.stack]); + this.logger.error(['writeData', error?.message, error?.stack]); return; } return response; @@ -77,7 +76,7 @@ export class AuthStateProvider { const readData = async (key: string): Promise => { const [response, error] = await this.providerFiles.read(instance, key); if (error) { - this.logger.error([error?.message, error?.stack]); + this.logger.error(['readData', error?.message, error?.stack]); return; } if (isNotEmpty(response?.data)) { @@ -88,7 +87,7 @@ export class AuthStateProvider { const removeData = async (key: string) => { const [response, error] = await this.providerFiles.delete(instance, key); if (error) { - this.logger.error([error?.message, error?.stack]); + this.logger.error(['removeData', error?.message, error?.stack]); return; } From 5b2a0fdcb14187d48d966ce7a6751ed0060bc9d7 Mon Sep 17 00:00:00 2001 From: Davidson Gomes Date: Sat, 1 Jun 2024 19:29:37 -0300 Subject: [PATCH 04/13] Adjust to repository from session worker --- src/api/controllers/instance.controller.ts | 2 +- src/api/services/channels/whatsapp.baileys.service.ts | 4 ++-- src/api/services/monitor.service.ts | 6 ++---- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/api/controllers/instance.controller.ts b/src/api/controllers/instance.controller.ts index 90d9622a..9c4f358c 100644 --- a/src/api/controllers/instance.controller.ts +++ b/src/api/controllers/instance.controller.ts @@ -753,7 +753,7 @@ export class InstanceController { this.logger.verbose('deleting instance: ' + instanceName); try { - this.waMonitor.waInstances[instanceName].sendDataWebhook(Events.INSTANCE_DELETE, { + this.waMonitor.waInstances[instanceName]?.sendDataWebhook(Events.INSTANCE_DELETE, { instanceName, instanceId: (await this.repository.auth.find(instanceName))?.instanceId, }); diff --git a/src/api/services/channels/whatsapp.baileys.service.ts b/src/api/services/channels/whatsapp.baileys.service.ts index 911f4c66..4dafe2aa 100644 --- a/src/api/services/channels/whatsapp.baileys.service.ts +++ b/src/api/services/channels/whatsapp.baileys.service.ts @@ -1478,7 +1478,7 @@ export class BaileysStartupService extends ChannelStartupService { this.logger.verbose('Sending data to webhook in event LABELS_ASSOCIATION'); // Atualiza labels nos chats - if (database.SAVE_DATA.CHATS) { + if (database.ENABLED && database.SAVE_DATA.CHATS) { const chats = await this.repository.chat.find({ where: { owner: this.instance.name, @@ -1486,7 +1486,7 @@ export class BaileysStartupService extends ChannelStartupService { }); const chat = chats.find((c) => c.id === data.association.chatId); if (chat) { - let labels = [...chat?.labels]; + let labels = [...chat.labels]; if (data.type === 'remove') { labels = labels.filter((label) => label !== data.association.labelId); } else if (data.type === 'add') { diff --git a/src/api/services/monitor.service.ts b/src/api/services/monitor.service.ts index c197b843..c342e977 100644 --- a/src/api/services/monitor.service.ts +++ b/src/api/services/monitor.service.ts @@ -271,9 +271,8 @@ export class WAMonitoringService { this.logger.verbose('cleaning up instance in files: ' + instanceName); if (this.providerSession?.ENABLED) { await this.providerFiles.removeSession(instanceName); - } else { - rmSync(join(INSTANCE_DIR, instanceName), { recursive: true, force: true }); } + rmSync(join(INSTANCE_DIR, instanceName), { recursive: true, force: true }); } public async cleaningStoreFiles(instanceName: string) { @@ -281,9 +280,8 @@ export class WAMonitoringService { this.logger.verbose('cleaning store files instance: ' + instanceName); if (this.providerSession?.ENABLED) { await this.providerFiles.removeSession(instanceName); - } else { - rmSync(join(INSTANCE_DIR, instanceName), { recursive: true, force: true }); } + rmSync(join(INSTANCE_DIR, instanceName), { recursive: true, force: true }); execSync(`rm -rf ${join(STORE_DIR, 'chats', instanceName)}`); execSync(`rm -rf ${join(STORE_DIR, 'contacts', instanceName)}`); From dd123c6a996ad4963a1e2fa2194778764584803b Mon Sep 17 00:00:00 2001 From: Davidson Gomes Date: Sat, 1 Jun 2024 19:47:43 -0300 Subject: [PATCH 05/13] Adjust to repository from session worker --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 8b847e18..9f434a39 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,7 @@ "amqplib": "^0.10.3", "@aws-sdk/client-sqs": "^3.569.0", "axios": "^1.6.5", - "@whiskeysockets/baileys": "6.6.0", + "@whiskeysockets/baileys": "6.7.2", "class-validator": "^0.14.1", "compression": "^1.7.4", "cors": "^2.8.5", From 52230edc5c1113fe9a06be0ef2156c65559fbcc2 Mon Sep 17 00:00:00 2001 From: Davidson Gomes Date: Mon, 3 Jun 2024 19:06:00 -0300 Subject: [PATCH 06/13] session worker compatibility --- package.json | 4 +- src/api/provider/sessions.ts | 40 ++++++++++++------- ...se-multi-file-auth-state-provider-files.ts | 6 +-- 3 files changed, 31 insertions(+), 19 deletions(-) diff --git a/package.json b/package.json index 9f434a39..bcd187d5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "evolution-api", - "version": "1.8.0", + "version": "1.8.1", "description": "Rest api for communication with WhatsApp", "main": "./dist/src/main.js", "scripts": { @@ -49,7 +49,7 @@ "amqplib": "^0.10.3", "@aws-sdk/client-sqs": "^3.569.0", "axios": "^1.6.5", - "@whiskeysockets/baileys": "6.7.2", + "@whiskeysockets/baileys": "6.7.4", "class-validator": "^0.14.1", "compression": "^1.7.4", "cors": "^2.8.5", diff --git a/src/api/provider/sessions.ts b/src/api/provider/sessions.ts index 6997d9ff..7afc9c89 100644 --- a/src/api/provider/sessions.ts +++ b/src/api/provider/sessions.ts @@ -1,7 +1,7 @@ import axios from 'axios'; import { execSync } from 'child_process'; -import { ConfigService, ProviderSession } from '../../config/env.config'; +import { Auth, ConfigService, ProviderSession } from '../../config/env.config'; import { Logger } from '../../config/logger.config'; type ResponseSuccess = { status: number; data?: any }; @@ -10,11 +10,13 @@ type ResponseProvider = Promise<[ResponseSuccess?, Error?]>; export class ProviderFiles { constructor(private readonly configService: ConfigService) { this.baseUrl = `http://${this.config.HOST}:${this.config.PORT}/session/${this.config.PREFIX}`; + this.globalApiToken = this.configService.get('AUTHENTICATION').API_KEY.KEY; } private readonly logger = new Logger(ProviderFiles.name); private baseUrl: string; + private globalApiToken: string; private readonly config = Object.freeze(this.configService.get('PROVIDER')); @@ -24,14 +26,14 @@ export class ProviderFiles { public async onModuleInit() { if (this.config.ENABLED) { - const client = axios.create({ - baseURL: this.baseUrl, - }); + const url = `http://${this.config.HOST}:${this.config.PORT}`; try { - const response = await client.options('/ping'); - if (!response?.data?.pong) { + const response = await axios.options(url + '/ping'); + if (response?.data != 'pong') { throw new Error('Offline file provider.'); } + + await axios.post(`${url}/session`, { group: this.config.PREFIX }, { headers: { apikey: this.globalApiToken } }); } catch (error) { this.logger.error(['Failed to connect to the file server', error?.message, error?.stack]); const pid = process.pid; @@ -46,9 +48,13 @@ export class ProviderFiles { public async create(instance: string): ResponseProvider { try { - const response = await axios.post(`${this.baseUrl}`, { - instance, - }); + const response = await axios.post( + `${this.baseUrl}`, + { + instance, + }, + { headers: { apikey: this.globalApiToken } }, + ); return [{ status: response.status, data: response?.data }]; } catch (error) { return [ @@ -63,7 +69,9 @@ export class ProviderFiles { public async write(instance: string, key: string, data: any): ResponseProvider { try { - const response = await axios.post(`${this.baseUrl}/${instance}/${key}`, data); + const response = await axios.post(`${this.baseUrl}/${instance}/${key}`, data, { + headers: { apikey: this.globalApiToken }, + }); return [{ status: response.status, data: response?.data }]; } catch (error) { return [ @@ -78,7 +86,9 @@ export class ProviderFiles { public async read(instance: string, key: string): ResponseProvider { try { - const response = await axios.get(`${this.baseUrl}/${instance}/${key}`); + const response = await axios.get(`${this.baseUrl}/${instance}/${key}`, { + headers: { apikey: this.globalApiToken }, + }); return [{ status: response.status, data: response?.data }]; } catch (error) { return [ @@ -93,7 +103,9 @@ export class ProviderFiles { public async delete(instance: string, key: string): ResponseProvider { try { - const response = await axios.delete(`${this.baseUrl}/${instance}/${key}`); + const response = await axios.delete(`${this.baseUrl}/${instance}/${key}`, { + headers: { apikey: this.globalApiToken }, + }); return [{ status: response.status, data: response?.data }]; } catch (error) { return [ @@ -108,7 +120,7 @@ export class ProviderFiles { public async allInstances(): ResponseProvider { try { - const response = await axios.get(`${this.baseUrl}/list-instances`); + const response = await axios.get(`${this.baseUrl}/list-instances`, { headers: { apikey: this.globalApiToken } }); return [{ status: response.status, data: response?.data as string[] }]; } catch (error) { return [ @@ -123,7 +135,7 @@ export class ProviderFiles { public async removeSession(instance: string): ResponseProvider { try { - const response = await axios.delete(`${this.baseUrl}/${instance}`); + const response = await axios.delete(`${this.baseUrl}/${instance}`, { headers: { apikey: this.globalApiToken } }); return [{ status: response.status, data: response?.data }]; } catch (error) { return [ diff --git a/src/utils/use-multi-file-auth-state-provider-files.ts b/src/utils/use-multi-file-auth-state-provider-files.ts index 54100463..1051af8f 100644 --- a/src/utils/use-multi-file-auth-state-provider-files.ts +++ b/src/utils/use-multi-file-auth-state-provider-files.ts @@ -67,7 +67,7 @@ export class AuthStateProvider { data: json, }); if (error) { - this.logger.error(['writeData', error?.message, error?.stack]); + // this.logger.error(['writeData', error?.message, error?.stack]); return; } return response; @@ -76,7 +76,7 @@ export class AuthStateProvider { const readData = async (key: string): Promise => { const [response, error] = await this.providerFiles.read(instance, key); if (error) { - this.logger.error(['readData', error?.message, error?.stack]); + // this.logger.error(['readData', error?.message, error?.stack]); return; } if (isNotEmpty(response?.data)) { @@ -87,7 +87,7 @@ export class AuthStateProvider { const removeData = async (key: string) => { const [response, error] = await this.providerFiles.delete(instance, key); if (error) { - this.logger.error(['removeData', error?.message, error?.stack]); + // this.logger.error(['removeData', error?.message, error?.stack]); return; } From 6371773416ef2b7871b25ad0d31ef8a1f5f8de1e Mon Sep 17 00:00:00 2001 From: Carlos Chinchilla Date: Tue, 4 Jun 2024 15:14:02 -0300 Subject: [PATCH 07/13] fix: added validation when sending gif image --- src/api/integrations/chatwoot/services/chatwoot.service.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/api/integrations/chatwoot/services/chatwoot.service.ts b/src/api/integrations/chatwoot/services/chatwoot.service.ts index 4190267e..9a30dc12 100644 --- a/src/api/integrations/chatwoot/services/chatwoot.service.ts +++ b/src/api/integrations/chatwoot/services/chatwoot.service.ts @@ -1092,6 +1092,10 @@ export class ChatwootService { return messageSent; } + if (type === 'image' && parsedMedia && parsedMedia?.ext === '.gif') { + type = 'document'; + } + this.logger.verbose('send media to instance: ' + waInstance.instanceName); const data: SendMediaDto = { number: number, From 8034e7f5878c28317ab0f8ed4900ff6669665750 Mon Sep 17 00:00:00 2001 From: Davidson Gomes Date: Wed, 5 Jun 2024 21:25:55 -0300 Subject: [PATCH 08/13] cacheGroupMetadata --- src/api/controllers/instance.controller.ts | 6 +- src/api/server.module.ts | 6 +- .../channels/whatsapp.baileys.service.ts | 88 ++++++++++++++++--- .../channels/whatsapp.business.service.ts | 2 +- src/api/services/monitor.service.ts | 6 +- 5 files changed, 87 insertions(+), 21 deletions(-) diff --git a/src/api/controllers/instance.controller.ts b/src/api/controllers/instance.controller.ts index 9c4f358c..bc3ace61 100644 --- a/src/api/controllers/instance.controller.ts +++ b/src/api/controllers/instance.controller.ts @@ -43,7 +43,7 @@ export class InstanceController { private readonly proxyService: ProxyController, private readonly cache: CacheService, private readonly chatwootCache: CacheService, - private readonly messagesLostCache: CacheService, + private readonly baileysCache: CacheService, private readonly providerFiles: ProviderFiles, ) {} @@ -112,7 +112,7 @@ export class InstanceController { this.repository, this.cache, this.chatwootCache, - this.messagesLostCache, + this.baileysCache, this.providerFiles, ); } else { @@ -122,7 +122,7 @@ export class InstanceController { this.repository, this.cache, this.chatwootCache, - this.messagesLostCache, + this.baileysCache, this.providerFiles, ); } diff --git a/src/api/server.module.ts b/src/api/server.module.ts index e282be1c..9b469ea7 100644 --- a/src/api/server.module.ts +++ b/src/api/server.module.ts @@ -109,7 +109,7 @@ export const repository = new RepositoryBroker( export const cache = new CacheService(new CacheEngine(configService, 'instance').getEngine()); const chatwootCache = new CacheService(new CacheEngine(configService, ChatwootService.name).getEngine()); -const messagesLostCache = new CacheService(new CacheEngine(configService, 'baileys').getEngine()); +const baileysCache = new CacheService(new CacheEngine(configService, 'baileys').getEngine()); const providerFiles = new ProviderFiles(configService); export const waMonitor = new WAMonitoringService( @@ -118,7 +118,7 @@ export const waMonitor = new WAMonitoringService( repository, cache, chatwootCache, - messagesLostCache, + baileysCache, providerFiles, ); @@ -170,7 +170,7 @@ export const instanceController = new InstanceController( proxyController, cache, chatwootCache, - messagesLostCache, + baileysCache, providerFiles, ); export const sendMessageController = new SendMessageController(waMonitor); diff --git a/src/api/services/channels/whatsapp.baileys.service.ts b/src/api/services/channels/whatsapp.baileys.service.ts index 4dafe2aa..b61be675 100644 --- a/src/api/services/channels/whatsapp.baileys.service.ts +++ b/src/api/services/channels/whatsapp.baileys.service.ts @@ -137,7 +137,7 @@ export class BaileysStartupService extends ChannelStartupService { public readonly repository: RepositoryBroker, public readonly cache: CacheService, public readonly chatwootCache: CacheService, - public readonly messagesLostCache: CacheService, + public readonly baileysCache: CacheService, private readonly providerFiles: ProviderFiles, ) { super(configService, eventEmitter, repository, chatwootCache); @@ -146,6 +146,8 @@ export class BaileysStartupService extends ChannelStartupService { this.instance.qrcode = { count: 0 }; this.mobile = false; this.recoveringMessages(); + this.forceUpdateGroupMetadataCache(); + this.authStateProvider = new AuthStateProvider(this.providerFiles); } @@ -166,9 +168,9 @@ export class BaileysStartupService extends ChannelStartupService { if ((cacheConf?.REDIS?.ENABLED && cacheConf?.REDIS?.URI !== '') || cacheConf?.LOCAL?.ENABLED) { setInterval(async () => { - this.messagesLostCache.keys().then((keys) => { + this.baileysCache.keys().then((keys) => { keys.forEach(async (key) => { - const message = await this.messagesLostCache.get(key.split(':')[2]); + const message = await this.baileysCache.get(key.split(':')[2]); if (message.messageStubParameters && message.messageStubParameters[0] === 'Message absent from node') { this.logger.info('Message absent from node, retrying to send, key: ' + key.split(':')[2]); @@ -180,6 +182,17 @@ export class BaileysStartupService extends ChannelStartupService { } } + private async forceUpdateGroupMetadataCache() { + setInterval(async () => { + this.logger.verbose('Forcing update group metadata cache'); + const groups = await this.fetchAllGroups({ getParticipants: 'false' }); + + for (const group of groups) { + await this.updateGroupMetadataCache(group.id); + } + }, 60000); + } + public get connectionStatus() { this.logger.verbose('Getting connection status'); return this.stateConnection; @@ -1124,15 +1137,15 @@ export class BaileysStartupService extends ChannelStartupService { if (received.messageStubParameters && received.messageStubParameters[0] === 'Message absent from node') { this.logger.info('Recovering message lost'); - await this.messagesLostCache.set(received.key.id, received); + await this.baileysCache.set(received.key.id, received); continue; } - const retryCache = (await this.messagesLostCache.get(received.key.id)) || null; + const retryCache = (await this.baileysCache.get(received.key.id)) || null; if (retryCache) { this.logger.info('Recovered message lost'); - await this.messagesLostCache.delete(received.key.id); + await this.baileysCache.delete(received.key.id); } if ( @@ -1421,6 +1434,12 @@ export class BaileysStartupService extends ChannelStartupService { this.logger.verbose('Sending data to webhook in event GROUPS_UPDATE'); this.sendDataWebhook(Events.GROUPS_UPDATE, groupMetadataUpdate); + + groupMetadataUpdate.forEach((group) => { + if (isJidGroup(group.id)) { + this.updateGroupMetadataCache(group.id); + } + }); }, 'group-participants.update': (participantsUpdate: { @@ -1857,7 +1876,8 @@ export class BaileysStartupService extends ChannelStartupService { let mentions: string[]; if (isJidGroup(sender)) { try { - const group = await this.findGroup({ groupJid: sender }, 'inner'); + // const group = await this.findGroup({ groupJid: sender }, 'inner'); + const group = await this.getGroupMetadataCache(sender); if (!group) { throw new NotFoundException('Group not found'); @@ -1910,7 +1930,10 @@ export class BaileysStartupService extends ChannelStartupService { key: message['reactionMessage']['key'], }, } as unknown as AnyMessageContent, - option as unknown as MiscMessageGenerationOptions, + { + ...option, + cachedGroupMetadata: this.getGroupMetadataCache, + } as unknown as MiscMessageGenerationOptions, ); } } @@ -1923,7 +1946,10 @@ export class BaileysStartupService extends ChannelStartupService { mentions, linkPreview: linkPreview, } as unknown as AnyMessageContent, - option as unknown as MiscMessageGenerationOptions, + { + ...option, + cachedGroupMetadata: this.getGroupMetadataCache, + } as unknown as MiscMessageGenerationOptions, ); } @@ -1938,7 +1964,10 @@ export class BaileysStartupService extends ChannelStartupService { }, mentions, }, - option as unknown as MiscMessageGenerationOptions, + { + ...option, + cachedGroupMetadata: this.getGroupMetadataCache, + } as unknown as MiscMessageGenerationOptions, ); } @@ -1959,7 +1988,10 @@ export class BaileysStartupService extends ChannelStartupService { return await this.client.sendMessage( sender, message as unknown as AnyMessageContent, - option as unknown as MiscMessageGenerationOptions, + { + ...option, + cachedGroupMetadata: this.getGroupMetadataCache, + } as unknown as MiscMessageGenerationOptions, ); })(); @@ -3164,6 +3196,40 @@ export class BaileysStartupService extends ChannelStartupService { } // Group + private async updateGroupMetadataCache(groupJid: string) { + try { + const meta = await this.client.groupMetadata(groupJid); + console.log('updateGroupMetadataCache', groupJid); + await this.baileysCache.set(`group-metadata-${groupJid}`, { + timestamp: Date.now(), + data: meta, + }); + + return meta; + } catch (error) { + this.logger.error(error); + } + } + + private async getGroupMetadataCache(groupJid: string) { + if (!isJidGroup(groupJid)) return null; + + console.log('getGroupMetadataCache', groupJid); + if (this.baileysCache.has(`group-metadata-${groupJid}`)) { + console.log('has cache'); + const meta = await this.baileysCache.get(`group-metadata-${groupJid}`); + + if (Date.now() - meta.timestamp > 60000) { + await this.updateGroupMetadataCache(groupJid); + } + + console.log('meta.data', meta.data); + return meta.data; + } + + return await this.updateGroupMetadataCache(groupJid); + } + public async createGroup(create: CreateGroupDto) { this.logger.verbose('Creating group: ' + create.subject); try { diff --git a/src/api/services/channels/whatsapp.business.service.ts b/src/api/services/channels/whatsapp.business.service.ts index e44f36c7..86178659 100644 --- a/src/api/services/channels/whatsapp.business.service.ts +++ b/src/api/services/channels/whatsapp.business.service.ts @@ -36,7 +36,7 @@ export class BusinessStartupService extends ChannelStartupService { public readonly repository: RepositoryBroker, public readonly cache: CacheService, public readonly chatwootCache: CacheService, - public readonly messagesLostCache: CacheService, + public readonly baileysCache: CacheService, private readonly providerFiles: ProviderFiles, ) { super(configService, eventEmitter, repository, chatwootCache); diff --git a/src/api/services/monitor.service.ts b/src/api/services/monitor.service.ts index c342e977..101b005e 100644 --- a/src/api/services/monitor.service.ts +++ b/src/api/services/monitor.service.ts @@ -44,7 +44,7 @@ export class WAMonitoringService { private readonly repository: RepositoryBroker, private readonly cache: CacheService, private readonly chatwootCache: CacheService, - private readonly messagesLostCache: CacheService, + private readonly baileysCache: CacheService, private readonly providerFiles: ProviderFiles, ) { this.logger.verbose('instance created'); @@ -368,7 +368,7 @@ export class WAMonitoringService { this.repository, this.cache, this.chatwootCache, - this.messagesLostCache, + this.baileysCache, this.providerFiles, ); @@ -380,7 +380,7 @@ export class WAMonitoringService { this.repository, this.cache, this.chatwootCache, - this.messagesLostCache, + this.baileysCache, this.providerFiles, ); From 9633412ef61dddb179a2e0a2fd784155ad727c2a Mon Sep 17 00:00:00 2001 From: Davidson Gomes Date: Wed, 5 Jun 2024 21:39:33 -0300 Subject: [PATCH 09/13] cacheGroupMetadata --- .../channels/whatsapp.baileys.service.ts | 26 +++++++------------ 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/src/api/services/channels/whatsapp.baileys.service.ts b/src/api/services/channels/whatsapp.baileys.service.ts index b61be675..df3a8816 100644 --- a/src/api/services/channels/whatsapp.baileys.service.ts +++ b/src/api/services/channels/whatsapp.baileys.service.ts @@ -55,9 +55,11 @@ import qrcode, { QRCodeToDataURLOptions } from 'qrcode'; import qrcodeTerminal from 'qrcode-terminal'; import sharp from 'sharp'; +import { CacheEngine } from '../../../cache/cacheengine'; import { CacheConf, ConfigService, + configService, ConfigSessionPhone, Database, Log, @@ -130,6 +132,8 @@ import { Events, MessageSubtype, TypeMediaMessage, wa } from '../../types/wa.typ import { CacheService } from './../cache.service'; import { ChannelStartupService } from './../channel.service'; +const groupMetadataCache = new CacheService(new CacheEngine(configService, 'groups').getEngine()); + export class BaileysStartupService extends ChannelStartupService { constructor( public readonly configService: ConfigService, @@ -190,7 +194,7 @@ export class BaileysStartupService extends ChannelStartupService { for (const group of groups) { await this.updateGroupMetadataCache(group.id); } - }, 60000); + }, 3600000); } public get connectionStatus() { @@ -660,12 +664,8 @@ export class BaileysStartupService extends ChannelStartupService { return; } - console.log('phoneNumber', phoneNumber); - const parsedPhoneNumber = parsePhoneNumber(phoneNumber); - console.log('parsedPhoneNumber', parsedPhoneNumber); - if (!parsedPhoneNumber?.isValid()) { this.logger.error('Phone number invalid'); return; @@ -687,7 +687,6 @@ export class BaileysStartupService extends ChannelStartupService { try { const response = await this.client.requestRegistrationCode(registration); - console.log('response', response); if (['ok', 'sent'].includes(response?.status)) { this.logger.verbose('Registration code sent successfully'); @@ -701,9 +700,8 @@ export class BaileysStartupService extends ChannelStartupService { public async receiveMobileCode(code: string) { await this.client .register(code.replace(/["']/g, '').trim().toLowerCase()) - .then(async (response) => { + .then(async () => { this.logger.verbose('Registration code received successfully'); - console.log(response); }) .catch((error) => { this.logger.error(error); @@ -3199,8 +3197,7 @@ export class BaileysStartupService extends ChannelStartupService { private async updateGroupMetadataCache(groupJid: string) { try { const meta = await this.client.groupMetadata(groupJid); - console.log('updateGroupMetadataCache', groupJid); - await this.baileysCache.set(`group-metadata-${groupJid}`, { + await groupMetadataCache.set(groupJid, { timestamp: Date.now(), data: meta, }); @@ -3214,16 +3211,13 @@ export class BaileysStartupService extends ChannelStartupService { private async getGroupMetadataCache(groupJid: string) { if (!isJidGroup(groupJid)) return null; - console.log('getGroupMetadataCache', groupJid); - if (this.baileysCache.has(`group-metadata-${groupJid}`)) { - console.log('has cache'); - const meta = await this.baileysCache.get(`group-metadata-${groupJid}`); + if (await groupMetadataCache.has(groupJid)) { + const meta = await groupMetadataCache.get(groupJid); - if (Date.now() - meta.timestamp > 60000) { + if (Date.now() - meta.timestamp > 3600000) { await this.updateGroupMetadataCache(groupJid); } - console.log('meta.data', meta.data); return meta.data; } From afcf6eab71c464468fd083da0f5e8b1d6bafbc41 Mon Sep 17 00:00:00 2001 From: Davidson Gomes Date: Wed, 5 Jun 2024 21:48:37 -0300 Subject: [PATCH 10/13] cacheGroupMetadata --- .../channels/whatsapp.baileys.service.ts | 39 ++++++++++++++++--- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/src/api/services/channels/whatsapp.baileys.service.ts b/src/api/services/channels/whatsapp.baileys.service.ts index df3a8816..902eefde 100644 --- a/src/api/services/channels/whatsapp.baileys.service.ts +++ b/src/api/services/channels/whatsapp.baileys.service.ts @@ -187,6 +187,12 @@ export class BaileysStartupService extends ChannelStartupService { } private async forceUpdateGroupMetadataCache() { + if ( + !this.configService.get('CACHE').REDIS.ENABLED && + !this.configService.get('CACHE').LOCAL.ENABLED + ) + return; + setInterval(async () => { this.logger.verbose('Forcing update group metadata cache'); const groups = await this.fetchAllGroups({ getParticipants: 'false' }); @@ -1874,8 +1880,11 @@ export class BaileysStartupService extends ChannelStartupService { let mentions: string[]; if (isJidGroup(sender)) { try { - // const group = await this.findGroup({ groupJid: sender }, 'inner'); - const group = await this.getGroupMetadataCache(sender); + let group; + + const cache = this.configService.get('CACHE'); + if (!cache.REDIS.ENABLED && !cache.LOCAL.ENABLED) group = await this.findGroup({ groupJid: sender }, 'inner'); + else group = await this.getGroupMetadataCache(sender); if (!group) { throw new NotFoundException('Group not found'); @@ -1930,7 +1939,11 @@ export class BaileysStartupService extends ChannelStartupService { } as unknown as AnyMessageContent, { ...option, - cachedGroupMetadata: this.getGroupMetadataCache, + cachedGroupMetadata: + !this.configService.get('CACHE').REDIS.ENABLED && + !this.configService.get('CACHE').LOCAL.ENABLED + ? null + : this.getGroupMetadataCache, } as unknown as MiscMessageGenerationOptions, ); } @@ -1946,7 +1959,11 @@ export class BaileysStartupService extends ChannelStartupService { } as unknown as AnyMessageContent, { ...option, - cachedGroupMetadata: this.getGroupMetadataCache, + cachedGroupMetadata: + !this.configService.get('CACHE').REDIS.ENABLED && + !this.configService.get('CACHE').LOCAL.ENABLED + ? null + : this.getGroupMetadataCache, } as unknown as MiscMessageGenerationOptions, ); } @@ -1964,7 +1981,11 @@ export class BaileysStartupService extends ChannelStartupService { }, { ...option, - cachedGroupMetadata: this.getGroupMetadataCache, + cachedGroupMetadata: + !this.configService.get('CACHE').REDIS.ENABLED && + !this.configService.get('CACHE').LOCAL.ENABLED + ? null + : this.getGroupMetadataCache, } as unknown as MiscMessageGenerationOptions, ); } @@ -1988,7 +2009,11 @@ export class BaileysStartupService extends ChannelStartupService { message as unknown as AnyMessageContent, { ...option, - cachedGroupMetadata: this.getGroupMetadataCache, + cachedGroupMetadata: + !this.configService.get('CACHE').REDIS.ENABLED && + !this.configService.get('CACHE').LOCAL.ENABLED + ? null + : this.getGroupMetadataCache, } as unknown as MiscMessageGenerationOptions, ); })(); @@ -3205,6 +3230,7 @@ export class BaileysStartupService extends ChannelStartupService { return meta; } catch (error) { this.logger.error(error); + return null; } } @@ -3212,6 +3238,7 @@ export class BaileysStartupService extends ChannelStartupService { if (!isJidGroup(groupJid)) return null; if (await groupMetadataCache.has(groupJid)) { + console.log('Has cache for group: ' + groupJid); const meta = await groupMetadataCache.get(groupJid); if (Date.now() - meta.timestamp > 3600000) { From 1f817df5f6e4f6ab6a4233d740243650786f864e Mon Sep 17 00:00:00 2001 From: Davidson Gomes Date: Sat, 8 Jun 2024 21:27:58 -0300 Subject: [PATCH 11/13] fix: Correction of variables breaking lines in typebot --- CHANGELOG.md | 1 + src/api/integrations/typebot/services/typebot.service.ts | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b96f10d..9a3ca50d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### Feature * New method of saving sessions to a file using worker, made in partnership with [codechat](https://github.com/code-chat-br/whatsapp-api) +* Correction of variables breaking lines in typebot # 1.8.0 (2024-05-27 16:10) diff --git a/src/api/integrations/typebot/services/typebot.service.ts b/src/api/integrations/typebot/services/typebot.service.ts index ebdaa920..ce56108d 100644 --- a/src/api/integrations/typebot/services/typebot.service.ts +++ b/src/api/integrations/typebot/services/typebot.service.ts @@ -525,10 +525,14 @@ export class TypebotService { } } - if (element.type === 'p') { + if (element.type === 'p' && element.type !== 'inline-variable') { text = text.trim() + '\n'; } + if (element.type === 'inline-variable') { + text = text.trim(); + } + if (element.type === 'ol') { text = '\n' + @@ -582,6 +586,8 @@ export class TypebotService { formattedText = formattedText.replace(/\*\*/g, '').replace(/__/, '').replace(/~~/, '').replace(/\n$/, ''); + formattedText = formattedText.replace(/\n$/, ''); + await instance.textMessage({ number: remoteJid.split('@')[0], options: { From 2002e1c38d7a332749258e1c214f687dd0ce3ad4 Mon Sep 17 00:00:00 2001 From: Davidson Gomes Date: Sat, 8 Jun 2024 21:31:02 -0300 Subject: [PATCH 12/13] changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a3ca50d..92a02aa0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,11 @@ # 1.8.1 (develop) ### Feature + * New method of saving sessions to a file using worker, made in partnership with [codechat](https://github.com/code-chat-br/whatsapp-api) + +### Fixed + * Correction of variables breaking lines in typebot # 1.8.0 (2024-05-27 16:10) @@ -13,12 +17,14 @@ * Build in docker for linux/amd64, linux/arm64 platforms ### Fixed + * Correction in message formatting when generated by AI as markdown in typebot * Security fix in fetch instance with client key when not connected to mongodb # 1.7.5 (2024-05-21 08:50) ### Fixed + * Add merge_brazil_contacts function to solve nine digit in brazilian numbers * Optimize ChatwootService method for updating contact * Fix swagger auth @@ -30,6 +36,7 @@ # 1.7.4 (2024-04-28 09:46) ### Fixed + * Adjusts in proxy on fetchAgent * Recovering messages lost with redis cache * Log when init redis cache service @@ -40,6 +47,7 @@ # 1.7.3 (2024-04-18 12:07) ### Fixed + * Revert fix audio encoding * Recovering messages lost with redis cache * Adjusts in redis for save instances From c898f1e62abc023807c8ed06c0bf8c90a25ba9af Mon Sep 17 00:00:00 2001 From: Davidson Gomes Date: Sat, 8 Jun 2024 21:32:50 -0300 Subject: [PATCH 13/13] v1.8.1 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 92a02aa0..e5adcb7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# 1.8.1 (develop) +# 1.8.1 (2024-06-08 21:32) ### Feature