Merge pull request #839 from judsonjuniorr/v2-is-on-whatsapp-cache

Save is on whatsapp on the database

.env.example

@@ -36,6 +36,8 @@ DATABASE_SAVE_DATA_CONTACTS=true
 DATABASE_SAVE_DATA_CHATS=true
 DATABASE_SAVE_DATA_LABELS=true
 DATABASE_SAVE_DATA_HISTORIC=true
+DATABASE_SAVE_IS_ON_WHATSAPP=true
+DATABASE_SAVE_IS_ON_WHATSAPP_DAYS=7
 
 # RabbitMQ - Environment variables
 RABBITMQ_ENABLED=false
@@ -171,6 +173,7 @@ DIFY_ENABLED=false
 # Redis Cache enabled
 CACHE_REDIS_ENABLED=true
 CACHE_REDIS_URI=redis://localhost:6379/6
+CACHE_REDIS_TTL=604800
 # Prefix serves to differentiate data from one installation to another that are using the same redis
 CACHE_REDIS_PREFIX_KEY=evolution
 # Enabling this variable will save the connection information in Redis and not in the database.

prisma/postgresql-migrations/20240828140837_add_is_on_whatsapp_table/migration.sql

@@ -0,0 +1,14 @@
+-- CreateTable
+CREATE TABLE "is_on_whatsapp" (
+    "id" TEXT NOT NULL,
+    "remote_jid" VARCHAR(100) NOT NULL,
+    "name" TEXT,
+    "jid_options" TEXT NOT NULL,
+    "created_at" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP NOT NULL,
+
+    CONSTRAINT "is_on_whatsapp_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "is_on_whatsapp_remote_jid_key" ON "is_on_whatsapp"("remote_jid");

prisma/postgresql-migrations/20240828141556_remove_name_column_from_on_whatsapp_table/migration.sql

@@ -0,0 +1,8 @@
+/*
+  Warnings:
+
+  - You are about to drop the column `name` on the `is_on_whatsapp` table. All the data in the column will be lost.
+
+*/
+-- AlterTable
+ALTER TABLE "is_on_whatsapp" DROP COLUMN "name";

prisma/postgresql-migrations/20240830193533_changed_table_case/migration.sql

@@ -0,0 +1,22 @@
+/*
+  Warnings:
+
+  - You are about to drop the `is_on_whatsapp` table. If the table is not empty, all the data it contains will be lost.
+
+*/
+-- DropTable
+DROP TABLE "is_on_whatsapp";
+
+-- CreateTable
+CREATE TABLE "IsOnWhatsapp" (
+    "id" TEXT NOT NULL,
+    "remoteJid" VARCHAR(100) NOT NULL,
+    "jidOptions" TEXT NOT NULL,
+    "createdAt" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP NOT NULL,
+
+    CONSTRAINT "IsOnWhatsapp_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "IsOnWhatsapp_remoteJid_key" ON "IsOnWhatsapp"("remoteJid");

prisma/postgresql-schema.prisma

@@ -574,3 +574,11 @@ model FlowiseSetting {
   Instance          Instance  @relation(fields: [instanceId], references: [id], onDelete: Cascade)
   instanceId        String    @unique
 }
+
+model IsOnWhatsapp {
+  id         String   @id @default(cuid())
+  remoteJid  String   @unique @db.VarChar(100)
+  jidOptions String
+  createdAt  DateTime @default(now()) @db.Timestamp
+  updatedAt  DateTime @updatedAt @db.Timestamp
+}

src/api/integrations/channel/whatsapp/whatsapp.baileys.service.ts

@@ -71,6 +71,7 @@ import ffmpegPath from '@ffmpeg-installer/ffmpeg';
 import { Boom } from '@hapi/boom';
 import { Instance } from '@prisma/client';
 import { makeProxyAgent } from '@utils/makeProxyAgent';
+import { getOnWhatsappCache, saveOnWhatsappCache } from '@utils/onWhatsappCache';
 import useMultiFileAuthStatePrisma from '@utils/use-multi-file-auth-state-prisma';
 import { AuthStateProvider } from '@utils/use-multi-file-auth-state-provider-files';
 import { useMultiFileAuthStateRedisDb } from '@utils/use-multi-file-auth-state-redis-db';
@@ -682,14 +683,19 @@ export class BaileysStartupService extends ChannelStartupService {
           instanceId: this.instanceId,
         }));
 
-        if (contactsRaw.length > 0) this.sendDataWebhook(Events.CONTACTS_UPSERT, contactsRaw);
-
         if (contactsRaw.length > 0) {
+          this.sendDataWebhook(Events.CONTACTS_UPSERT, contactsRaw);
+
           if (this.configService.get<Database>('DATABASE').SAVE_DATA.CONTACTS)
             await this.prismaRepository.contact.createMany({
               data: contactsRaw,
               skipDuplicates: true,
             });
+
+          const usersContacts = contactsRaw.filter((c) => c.remoteJid.includes('@s.whatsapp'));
+          if (usersContacts) {
+            await saveOnWhatsappCache(usersContacts.map((c) => ({ remoteJid: c.remoteJid })));
+          }
         }
 
         if (
@@ -717,9 +723,13 @@ export class BaileysStartupService extends ChannelStartupService {
           })),
         );
 
-        if (updatedContacts.length > 0) this.sendDataWebhook(Events.CONTACTS_UPDATE, updatedContacts);
-
         if (updatedContacts.length > 0) {
+          const usersContacts = updatedContacts.filter((c) => c.remoteJid.includes('@s.whatsapp'));
+          if (usersContacts) {
+            await saveOnWhatsappCache(usersContacts.map((c) => ({ remoteJid: c.remoteJid })));
+          }
+
+          this.sendDataWebhook(Events.CONTACTS_UPDATE, updatedContacts);
           await Promise.all(
             updatedContacts.map(async (contact) => {
               const update = this.prismaRepository.contact.updateMany({
@@ -778,6 +788,11 @@ export class BaileysStartupService extends ChannelStartupService {
         }),
       );
       await this.prismaRepository.$transaction(updateTransactions);
+
+      const usersContacts = contactsRaw.filter((c) => c.remoteJid.includes('@s.whatsapp'));
+      if (usersContacts) {
+        await saveOnWhatsappCache(usersContacts.map((c) => ({ remoteJid: c.remoteJid })));
+      }
     },
   };
 
@@ -1225,6 +1240,10 @@ export class BaileysStartupService extends ChannelStartupService {
               update: contactRaw,
               create: contactRaw,
             });
+
+          if (contactRaw.remoteJid.includes('@s.whatsapp')) {
+            await saveOnWhatsappCache([{ remoteJid: contactRaw.remoteJid }]);
+          }
         }
       } catch (error) {
         this.logger.error('line 1318');
@@ -2686,11 +2705,27 @@ export class BaileysStartupService extends ChannelStartupService {
     });
 
     const numbersToVerify = jids.users.map(({ jid }) => jid.replace('+', ''));
-    const verify = await this.client.onWhatsApp(...numbersToVerify);
+
+    const cachedNumbers = await getOnWhatsappCache(numbersToVerify);
+    const filteredNumbers = numbersToVerify.filter(
+      (jid) => !cachedNumbers.some((cached) => cached.jidOptions.includes(jid)),
+    );
+
+    const verify = await this.client.onWhatsApp(...filteredNumbers);
     const users: OnWhatsAppDto[] = await Promise.all(
       jids.users.map(async (user) => {
         let numberVerified: (typeof verify)[0] | null = null;
 
+        const cached = cachedNumbers.find((cached) => cached.jidOptions.includes(user.jid.replace('+', '')));
+        if (cached) {
+          return {
+            exists: true,
+            jid: cached.remoteJid,
+            name: contacts.find((c) => c.remoteJid === cached.remoteJid)?.pushName,
+            number: cached.number,
+          };
+        }
+
         // Brazilian numbers
         if (user.number.startsWith('55')) {
           const numberWithDigit =
@@ -2733,6 +2768,7 @@ export class BaileysStartupService extends ChannelStartupService {
         }
 
         const numberJid = numberVerified?.jid || user.jid;
+
         return {
           exists: !!numberVerified?.exists,
           jid: numberJid,
@@ -2742,6 +2778,8 @@ export class BaileysStartupService extends ChannelStartupService {
       }),
     );
 
+    await saveOnWhatsappCache(users.filter((user) => user.exists).map((user) => ({ remoteJid: user.jid })));
+
     onWhatsapp.push(...users);
 
     return onWhatsapp;
@@ -3525,8 +3563,15 @@ export class BaileysStartupService extends ChannelStartupService {
           imgUrl: participant.imgUrl ?? contact?.profilePicUrl,
         };
       });
+
+      const usersContacts = parsedParticipants.filter((c) => c.id.includes('@s.whatsapp'));
+      if (usersContacts) {
+        await saveOnWhatsappCache(usersContacts.map((c) => ({ remoteJid: c.id })));
+      }
+
       return { participants: parsedParticipants };
     } catch (error) {
+      console.error(error);
       this.logger.error('line 3583');
       throw new NotFoundException('No participants', error.toString());
     }

src/api/services/cache.service.ts

@@ -38,11 +38,11 @@ export class CacheService {
     }
   }
 
-  async set(key: string, value: any) {
+  async set(key: string, value: any, ttl?: number) {
     if (!this.cache) {
       return;
     }
-    this.cache.set(key, value);
+    this.cache.set(key, value, ttl);
   }
 
   public async hSet(key: string, field: string, value: any) {

src/config/env.config.ts

@@ -43,6 +43,8 @@ export type SaveData = {
   CONTACTS: boolean;
   CHATS: boolean;
   LABELS: boolean;
+  IS_ON_WHATSAPP: boolean;
+  IS_ON_WHATSAPP_DAYS: number;
 };
 
 export type DBConnection = {
@@ -295,6 +297,8 @@ export class ConfigService {
           CHATS: process.env?.DATABASE_SAVE_DATA_CHATS === 'true',
           HISTORIC: process.env?.DATABASE_SAVE_DATA_HISTORIC === 'true',
           LABELS: process.env?.DATABASE_SAVE_DATA_LABELS === 'true',
+          IS_ON_WHATSAPP: process.env?.DATABASE_SAVE_IS_ON_WHATSAPP === 'true',
+          IS_ON_WHATSAPP_DAYS: Number.parseInt(process.env?.DATABASE_SAVE_IS_ON_WHATSAPP_DAYS ?? '7'),
         },
       },
       RABBITMQ: {

src/utils/onWhatsappCache.ts

@@ -0,0 +1,100 @@
+import { prismaRepository } from '@api/server.module';
+import { configService, Database } from '@config/env.config';
+import dayjs from 'dayjs';
+
+function getAvailableNumbers(remoteJid: string) {
+  const numbersAvailable: string[] = [];
+
+  if (remoteJid.startsWith('+')) {
+    remoteJid = remoteJid.slice(1);
+  }
+
+  const [number, domain] = remoteJid.split('@');
+
+  // Brazilian numbers
+  if (remoteJid.startsWith('55')) {
+    const numberWithDigit =
+      number.slice(4, 5) === '9' && number.length === 13 ? number : `${number.slice(0, 4)}9${number.slice(4)}`;
+    const numberWithoutDigit = number.length === 12 ? number : number.slice(0, 4) + number.slice(5);
+
+    numbersAvailable.push(numberWithDigit);
+    numbersAvailable.push(numberWithoutDigit);
+  }
+
+  // Mexican/Argentina numbers
+  // Ref: https://faq.whatsapp.com/1294841057948784
+  else if (number.startsWith('52') || number.startsWith('54')) {
+    let prefix = '';
+    if (number.startsWith('52')) {
+      prefix = '1';
+    }
+    if (number.startsWith('54')) {
+      prefix = '9';
+    }
+
+    const numberWithDigit =
+      number.slice(2, 3) === prefix && number.length === 13
+        ? number
+        : `${number.slice(0, 2)}${prefix}${number.slice(2)}`;
+    const numberWithoutDigit = number.length === 12 ? number : number.slice(0, 2) + number.slice(3);
+
+    numbersAvailable.push(numberWithDigit);
+    numbersAvailable.push(numberWithoutDigit);
+  }
+
+  // Other countries
+  else {
+    numbersAvailable.push(remoteJid);
+  }
+
+  return numbersAvailable.map((number) => `${number}@${domain}`);
+}
+
+interface ISaveOnWhatsappCacheParams {
+  remoteJid: string;
+}
+export async function saveOnWhatsappCache(data: ISaveOnWhatsappCacheParams[]) {
+  if (configService.get<Database>('DATABASE').SAVE_DATA.IS_ON_WHATSAPP) {
+    const upsertsQuery = data.map((item) => {
+      const remoteJid = item.remoteJid.startsWith('+') ? item.remoteJid.slice(1) : item.remoteJid;
+      const numbersAvailable = getAvailableNumbers(remoteJid);
+
+      return prismaRepository.isOnWhatsapp.upsert({
+        create: { remoteJid: remoteJid, jidOptions: numbersAvailable.join(',') },
+        update: { jidOptions: numbersAvailable.join(',') },
+        where: { remoteJid: remoteJid },
+      });
+    });
+
+    await prismaRepository.$transaction(upsertsQuery);
+  }
+}
+
+export async function getOnWhatsappCache(remoteJids: string[]) {
+  let results: {
+    remoteJid: string;
+    number: string;
+    jidOptions: string[];
+  }[] = [];
+
+  if (configService.get<Database>('DATABASE').SAVE_DATA.IS_ON_WHATSAPP) {
+    const remoteJidsWithoutPlus = remoteJids.map((remoteJid) => getAvailableNumbers(remoteJid)).flat();
+
+    const onWhatsappCache = await prismaRepository.isOnWhatsapp.findMany({
+      where: {
+        OR: remoteJidsWithoutPlus.map((remoteJid) => ({ jidOptions: { contains: remoteJid } })),
+        updatedAt: {
+          gte: dayjs().subtract(configService.get<Database>('DATABASE').SAVE_DATA.IS_ON_WHATSAPP_DAYS, 'days').toDate(),
+        },
+      },
+    });
+
+    results = onWhatsappCache.map((item) => ({
+      remoteJid: item.remoteJid,
+      number: item.remoteJid.split('@')[0],
+      jidOptions: item.jidOptions.split(','),
+    }));
+  }
+
+  return results;
+}