refactor(plugin-sdk): unify channel dedupe primitives

extensions/feishu/src/bot.ts

@@ -9,7 +9,7 @@ import {
 } from "openclaw/plugin-sdk";
 import { resolveFeishuAccount } from "./accounts.js";
 import { createFeishuClient } from "./client.js";
-import { tryRecordMessage } from "./dedup.js";
+import { tryRecordMessagePersistent } from "./dedup.js";
 import { maybeCreateDynamicAgent } from "./dynamic-agent.js";
 import { normalizeFeishuExternalKey } from "./external-keys.js";
 import { downloadMessageResourceFeishu } from "./media.js";
@@ -510,9 +510,9 @@ export async function handleFeishuMessage(params: {
   const log = runtime?.log ?? console.log;
   const error = runtime?.error ?? console.error;
 
-  // Dedup check: skip if this message was already processed
+  // Dedup check: skip if this message was already processed (memory + disk).
   const messageId = event.message.message_id;
-  if (!tryRecordMessage(messageId)) {
+  if (!(await tryRecordMessagePersistent(messageId, account.accountId, log))) {
     log(`feishu: skipping duplicate message ${messageId}`);
     return;
   }

extensions/feishu/src/dedup.ts

@@ -1,33 +1,54 @@
-// Prevent duplicate processing when WebSocket reconnects or Feishu redelivers messages.
-const DEDUP_TTL_MS = 30 * 60 * 1000; // 30 minutes
-const DEDUP_MAX_SIZE = 1_000;
-const DEDUP_CLEANUP_INTERVAL_MS = 5 * 60 * 1000; // cleanup every 5 minutes
-const processedMessageIds = new Map<string, number>(); // messageId -> timestamp
-let lastCleanupTime = Date.now();
+import os from "node:os";
+import path from "node:path";
+import { createDedupeCache, createPersistentDedupe } from "openclaw/plugin-sdk";
 
-export function tryRecordMessage(messageId: string): boolean {
-  const now = Date.now();
+// Persistent TTL: 24 hours — survives restarts & WebSocket reconnects.
+const DEDUP_TTL_MS = 24 * 60 * 60 * 1000;
+const MEMORY_MAX_SIZE = 1_000;
+const FILE_MAX_ENTRIES = 10_000;
 
-  // Throttled cleanup: evict expired entries at most once per interval.
-  if (now - lastCleanupTime > DEDUP_CLEANUP_INTERVAL_MS) {
-    for (const [id, ts] of processedMessageIds) {
-      if (now - ts > DEDUP_TTL_MS) {
-        processedMessageIds.delete(id);
-      }
-    }
-    lastCleanupTime = now;
+const memoryDedupe = createDedupeCache({ ttlMs: DEDUP_TTL_MS, maxSize: MEMORY_MAX_SIZE });
+
+function resolveStateDirFromEnv(env: NodeJS.ProcessEnv = process.env): string {
+  const stateOverride = env.OPENCLAW_STATE_DIR?.trim() || env.CLAWDBOT_STATE_DIR?.trim();
+  if (stateOverride) {
+    return stateOverride;
   }
-
-  if (processedMessageIds.has(messageId)) {
-    return false;
+  if (env.VITEST || env.NODE_ENV === "test") {
+    return path.join(os.tmpdir(), `openclaw-vitest-${process.pid}`);
   }
-
-  // Evict oldest entries if cache is full.
-  if (processedMessageIds.size >= DEDUP_MAX_SIZE) {
-    const first = processedMessageIds.keys().next().value!;
-    processedMessageIds.delete(first);
-  }
-
-  processedMessageIds.set(messageId, now);
-  return true;
+  return path.join(os.homedir(), ".openclaw");
+}
+
+function resolveNamespaceFilePath(namespace: string): string {
+  const safe = namespace.replace(/[^a-zA-Z0-9_-]/g, "_");
+  return path.join(resolveStateDirFromEnv(), "feishu", "dedup", `${safe}.json`);
+}
+
+const persistentDedupe = createPersistentDedupe({
+  ttlMs: DEDUP_TTL_MS,
+  memoryMaxSize: MEMORY_MAX_SIZE,
+  fileMaxEntries: FILE_MAX_ENTRIES,
+  resolveFilePath: resolveNamespaceFilePath,
+});
+
+/**
+ * Synchronous dedup — memory only.
+ * Kept for backward compatibility; prefer {@link tryRecordMessagePersistent}.
+ */
+export function tryRecordMessage(messageId: string): boolean {
+  return !memoryDedupe.check(messageId);
+}
+
+export async function tryRecordMessagePersistent(
+  messageId: string,
+  namespace = "global",
+  log?: (...args: unknown[]) => void,
+): Promise<boolean> {
+  return persistentDedupe.checkAndRecord(messageId, {
+    namespace,
+    onDiskError: (error) => {
+      log?.(`feishu-dedup: disk error, falling back to memory: ${String(error)}`);
+    },
+  });
 }

extensions/tlon/src/monitor/processed-messages.ts

@@ -1,3 +1,5 @@
+import { createDedupeCache } from "openclaw/plugin-sdk";
+
 export type ProcessedMessageTracker = {
   mark: (id?: string | null) => boolean;
   has: (id?: string | null) => boolean;
@@ -5,29 +7,14 @@ export type ProcessedMessageTracker = {
 };
 
 export function createProcessedMessageTracker(limit = 2000): ProcessedMessageTracker {
-  const seen = new Set<string>();
-  const order: string[] = [];
+  const dedupe = createDedupeCache({ ttlMs: 0, maxSize: limit });
 
   const mark = (id?: string | null) => {
     const trimmed = id?.trim();
     if (!trimmed) {
       return true;
     }
-    if (seen.has(trimmed)) {
-      return false;
-    }
-    seen.add(trimmed);
-    order.push(trimmed);
-    if (order.length > limit) {
-      const overflow = order.length - limit;
-      for (let i = 0; i < overflow; i += 1) {
-        const oldest = order.shift();
-        if (oldest) {
-          seen.delete(oldest);
-        }
-      }
-    }
-    return true;
+    return !dedupe.check(trimmed);
   };
 
   const has = (id?: string | null) => {
@@ -35,12 +22,12 @@ export function createProcessedMessageTracker(limit = 2000): ProcessedMessageTra
     if (!trimmed) {
       return false;
     }
-    return seen.has(trimmed);
+    return dedupe.peek(trimmed);
   };
 
   return {
     mark,
     has,
-    size: () => seen.size,
+    size: () => dedupe.size(),
   };
 }

extensions/zalo/src/monitor.ts

@@ -2,6 +2,7 @@ import { timingSafeEqual } from "node:crypto";
 import type { IncomingMessage, ServerResponse } from "node:http";
 import type { OpenClawConfig, MarkdownTableMode } from "openclaw/plugin-sdk";
 import {
+  createDedupeCache,
   createReplyPrefixOptions,
   readJsonBodyWithLimit,
   registerWebhookTarget,
@@ -92,7 +93,10 @@ type WebhookTarget = {
 
 const webhookTargets = new Map<string, WebhookTarget[]>();
 const webhookRateLimits = new Map<string, WebhookRateLimitState>();
-const recentWebhookEvents = new Map<string, number>();
+const recentWebhookEvents = createDedupeCache({
+  ttlMs: ZALO_WEBHOOK_REPLAY_WINDOW_MS,
+  maxSize: 5000,
+});
 const webhookStatusCounters = new Map<string, number>();
 
 function isJsonContentType(value: string | string[] | undefined): boolean {
@@ -141,22 +145,7 @@ function isReplayEvent(update: ZaloUpdate, nowMs: number): boolean {
     return false;
   }
   const key = `${update.event_name}:${messageId}`;
-  const seenAt = recentWebhookEvents.get(key);
-  recentWebhookEvents.set(key, nowMs);
-
-  if (seenAt && nowMs - seenAt < ZALO_WEBHOOK_REPLAY_WINDOW_MS) {
-    return true;
-  }
-
-  if (recentWebhookEvents.size > 5000) {
-    for (const [eventKey, timestamp] of recentWebhookEvents) {
-      if (nowMs - timestamp >= ZALO_WEBHOOK_REPLAY_WINDOW_MS) {
-        recentWebhookEvents.delete(eventKey);
-      }
-    }
-  }
-
-  return false;
+  return recentWebhookEvents.check(key, nowMs);
 }
 
 function recordWebhookStatus(