Update botsignal.py

botsignal.py

@@ -3,9 +3,13 @@ import os
 import re
 import io
 import hashlib
-from collections import deque
+from collections import deque, defaultdict
+from datetime import datetime, timedelta, timezone
 from mimetypes import guess_extension
-from typing import List
+from typing import List, Tuple, Optional, Dict
+
+import nest_asyncio
+nest_asyncio.apply()
 
 from rapidfuzz import fuzz
 from telethon import TelegramClient, events
@@ -17,17 +21,25 @@ API_ID = int(os.environ.get("API_ID", "0"))
 API_HASH = os.environ.get("API_HASH", "")
 STRING_SESSION = os.environ.get("STRING_SESSION", "")
 
-# Sumber & target (boleh pakai @username, id, atau link t.me)
-SOURCE_CHATS = [
+# --- Definisikan sumber sebagai CORE vs SUPPORT ---
+# Kamu bisa pakai: "@username", id (int), atau "https://t.me/xxxxx"
+CORE_CHATS = [
     "https://t.me/PEPE_Calls28",
     "https://t.me/Tanjirocall",
+]
+SUPPORT_CHATS = [
     "https://t.me/ChinaPumpCommunity",
     "https://t.me/Milagrosdegencalls",
     "https://t.me/GM_Degencalls",
 ]
-TARGET_CHAT = os.environ.get("TARGET_CHAT", "https://t.me/MidasTouchsignalll")
 
-# Kata kunci topik; simple relevansi supaya tidak semua pesan dikirim
+# Gabungan digunakan untuk handler event
+SOURCE_CHATS = [*CORE_CHATS, *SUPPORT_CHATS]
+
+# Target (boleh @username / id / link)
+TARGET_CHAT = os.environ.get("TARGET_CHAT", "https://t.me/MidasTouchsignall")
+
+# Kata kunci topik untuk relevansi (tetap dari versi sebelumnya)
 THEME_KEYWORDS = [
     "call", "signal", "entry", "buy", "sell", "tp", "sl",
     "pump", "spot", "futures", "setup",
@@ -37,35 +49,29 @@ KEYWORD_WEIGHT = 1.0
 FUZZ_WEIGHT = 0.6
 RELEVANCE_THRESHOLD = float(os.environ.get("RELEVANCE_THRESHOLD", "1.0"))
 
-# Filter pengecualian: jika ada frasa ini (case-insensitive) -> skip seluruh pesan
+# Filter pengecualian
 EXCLUDE_PHRASES = [
     "achievement unlocked",
 ]
 
-# Frasa/tautan yang kalau ada di suatu baris -> baris itu dibuang
-# - "dm" akan cocok sebagai kata utuh (pakai regex \bdm\b), jadi "random" tidak ikut
-BLOCK_PATTERNS = [
-    r"\bdm\b",              # DM ajakan
-    r"\bcontact\b",
-    r"\bvip\b",
-    r"(?:https?://)?t\.me/\S+",   # link t.me/...
-    r"(?:https?://)?telegram\.me/\S+",
-    r"(?:https?://)?wa\.me/\S+",
-    r"@[\w\d_]{2,}"         # mention @username
-]
-
 # Media handling
 INCLUDE_MEDIA = os.environ.get("INCLUDE_MEDIA", "1") == "1"
 MAX_MEDIA_MB = float(os.environ.get("MAX_MEDIA_MB", "12"))
 SKIP_STICKERS = os.environ.get("SKIP_STICKERS", "1") == "1"
 ALLOW_GIFS_VIDEOS = os.environ.get("ALLOW_GIFS_VIDEOS", "0") == "1"
 
-# History backfill saat startup
+# Backfill
 INITIAL_BACKFILL = int(os.environ.get("INITIAL_BACKFILL", "20"))
 
 # Dedup buffer
 DEDUP_BUFFER_SIZE = int(os.environ.get("DEDUP_BUFFER_SIZE", "800"))
 
+# Jendela waktu penghitungan kelas (menit)
+CLASS_WINDOW_MINUTES = int(os.environ.get("CLASS_WINDOW_MINUTES", "10"))
+
+# Support gating minimal unik grup untuk boleh kirim
+SUPPORT_MIN_UNIQUE = int(os.environ.get("SUPPORT_MIN_UNIQUE", "2"))
+
 
 # ========= Client bootstrap =========
 def build_client() -> TelegramClient:
@@ -78,6 +84,9 @@ def build_client() -> TelegramClient:
 client = build_client()
 recent_hashes: deque[str] = deque(maxlen=DEDUP_BUFFER_SIZE)
 
+# Peta id_chat -> "core" / "support"
+chat_roles: Dict[int, str] = {}  # diisi saat startup setelah resolve entity
+
 
 # ========= Utilities =========
 def debug_log(reason: str, content: str = "") -> None:
@@ -87,32 +96,26 @@ def debug_log(reason: str, content: str = "") -> None:
 def normalize_for_filter(text: str) -> str:
     if not text:
         return ""
-    # Hilangkan quote >..., rapikan whitespace
-    s = re.sub(r"(?m)^>.*$", "", text)
+    s = re.sub(r"(?m)^>.*$", "", text)  # hilangin quote
     s = re.sub(r"\s+", " ", s).strip()
     return s
 
 def score_relevance(text: str, keywords: List[str]) -> float:
-    """Skor sederhana: exact keyword + fuzzy ratio rata-rata top 3."""
     if not text:
         return 0.0
     t = text.lower()
 
-    # exact hits
     exact_hits = 0
     for kw in keywords:
         if kw in t or re.search(rf"\b{re.escape(kw)}\b", t):
             exact_hits += 1
     exact_score = exact_hits * KEYWORD_WEIGHT
 
-    # fuzzy hits (ambil 3 nilai tertinggi)
     fuzz_scores = sorted((fuzz.partial_ratio(kw, t) / 100.0 for kw in keywords), reverse=True)[:3]
     fuzzy_score = (sum(fuzz_scores) / max(1, len(fuzz_scores))) * FUZZ_WEIGHT if fuzz_scores else 0.0
-
     return exact_score + fuzzy_score
 
 def hash_for_dedup(text: str, msg) -> str:
-    """Gabungkan teks dan sid media agar tidak double post."""
     parts = [text or ""]
     if getattr(msg, "id", None) is not None:
         parts.append(str(msg.id))
@@ -128,7 +131,6 @@ def hash_for_dedup(text: str, msg) -> str:
     return hashlib.sha1(raw).hexdigest()
 
 def is_image_message(msg) -> bool:
-    """True jika pesan mengandung foto atau dokumen gambar yang boleh."""
     if getattr(msg, "photo", None) is not None:
         return True
     doc = getattr(msg, "document", None)
@@ -150,43 +152,57 @@ def media_too_big(msg) -> bool:
         return (doc.size or 0) > MAX_MEDIA_MB * 1024 * 1024
     return False
 
-def filter_text_block(text: str) -> str:
+
+# ========= Class aggregator (windowed unique groups) =========
+# keyword -> { group_key(str): last_seen(datetime, UTC) }
+keyword_group_last_seen: defaultdict[str, dict[str, datetime]] = defaultdict(dict)
+
+def _prune_expired(now: datetime) -> None:
+    window = timedelta(minutes=CLASS_WINDOW_MINUTES)
+    cutoff = now - window
+    for kw, m in list(keyword_group_last_seen.items()):
+        for gk, ts in list(m.items()):
+            if ts < cutoff:
+                del m[gk]
+        if not m:
+            del keyword_group_last_seen[kw]
+
+def update_and_classify(keyword: str, group_key: str, now: Optional[datetime] = None) -> Tuple[str, int]:
     """
-    Hapus seluruh baris/kalimat yang mengandung pola terlarang (BLOCK_PATTERNS).
-    Kembalikan string yang sudah dibersihkan, plus merapikan blank lines berturut.
+    Update hit keyword oleh grup (dalam window waktu) & kembalikan (label, unique_count).
+    Label:
+      1 -> 'rendah'
+      2-3 -> 'sedang'
+      >=4 -> 'kuat'
     """
-    if not text:
-        return text
-
-    lines = text.splitlines()
-    keep: list[str] = []
-    compiled = [re.compile(pat, flags=re.IGNORECASE) for pat in BLOCK_PATTERNS]
+    if not now:
+        now = datetime.now(timezone.utc)
+    _prune_expired(now)
 
-    for line in lines:
-        l = line.strip()
-        if not l:
-            # Pertahankan baris kosong; nanti kita rapikan
-            keep.append(line)
-            continue
-        if any(rx.search(l) for rx in compiled):
-            # buang seluruh baris yang mengandung kata/tautan terlarang
-            continue
-        keep.append(line)
+    bucket = keyword_group_last_seen[keyword]
+    bucket[group_key] = now  # insert/update
+    unique_groups = len(bucket)
 
-    # rapikan blank lines berturut-turut
-    cleaned = "\n".join(keep)
-    cleaned = re.sub(r"\n{3,}", "\n\n", cleaned).strip()
-    return cleaned
+    if unique_groups >= 4:
+        return "kuat", unique_groups
+    elif unique_groups >= 2:
+        return "sedang", unique_groups
+    else:
+        return "rendah", unique_groups
 
 
 # ========= Core actions =========
-async def send_as_is(msg, override_text: str | None = None) -> None:
-    """Forward konten ke TARGET_CHAT, dengan media bila sesuai. Bisa override teks."""
-    orig_text = override_text if override_text is not None else (msg.message or getattr(msg, "raw_text", "") or "")
+async def send_as_is(msg, text_override: Optional[str] = None) -> None:
+    """
+    Forward konten 'apa adanya' (termasuk media yang diizinkan).
+    Kompatibel dengan versi awal kamu:contentReference[oaicite:2]{index=2}.
+    """
+    orig_text = text_override if text_override is not None else (
+        msg.message or (getattr(msg, "raw_text", None) or "")
+    )
 
     if INCLUDE_MEDIA and is_image_message(msg) and not media_too_big(msg):
         try:
-            # Jika pesan berupa photo asli
             if getattr(msg, "photo", None):
                 await client.send_file(
                     TARGET_CHAT,
@@ -197,13 +213,11 @@ async def send_as_is(msg, override_text: str | None = None) -> None:
                 )
                 return
 
-            # Jika berupa document image/gif/video
             doc = getattr(msg, "document", None)
             if doc:
                 data = await client.download_media(msg, file=bytes)
                 if data:
                     bio = io.BytesIO(data)
-                    # Tentukan ekstensi file dari mime
                     ext = ".jpg"
                     mt = (getattr(doc, "mime_type", "") or "").lower()
                     if mt:
@@ -223,7 +237,6 @@ async def send_as_is(msg, override_text: str | None = None) -> None:
         except Exception as e:
             debug_log("Gagal kirim sebagai media, fallback ke text", str(e))
 
-    # Fallback: text saja
     await client.send_message(
         TARGET_CHAT,
         orig_text,
@@ -231,19 +244,33 @@ async def send_as_is(msg, override_text: str | None = None) -> None:
         link_preview=True,
     )
 
+def _extract_main_keyword(text_norm: str) -> Optional[str]:
+    t = text_norm
+    t = re.sub(r"\$([a-z0-9]+)", r"\1", t, flags=re.I)  # $BTC -> btc
+    for kw in THEME_KEYWORDS:
+        if re.search(rf"(^|\W){re.escape(kw)}(\W|$)", t, flags=re.I):
+            return kw.lower()
+    return None
+
+def _role_of(chat_id: int) -> str:
+    # default ke 'core' kalau tak dikenal (lebih permisif)
+    return chat_roles.get(chat_id, "core")
+
 
-async def process_message(msg, source_name: str) -> None:
-    """Filter, dedup, relevansi, lalu kirim."""
+async def process_message(msg, source_chat_id: int) -> None:
+    """
+    Filter, dedup, relevansi, klasifikasi, 'gating' support vs core, lalu kirim.
+    """
     orig_text = msg.message or (getattr(msg, "raw_text", None) or "")
     text_norm = normalize_for_filter(orig_text).lower()
 
-    # Pengecualian eksplisit (skip seluruh pesan)
+    # Exclude phrases (case-insensitive)
     for phrase in EXCLUDE_PHRASES:
         if phrase.lower() in text_norm:
             debug_log("Dilewati karena EXCLUDE_PHRASES", orig_text)
             return
 
-    # Dedup (berdasarkan teks + sid media)
+    # Dedup
     h = hash_for_dedup(text_norm, msg)
     if h in recent_hashes:
         debug_log("Duplikat, dilewati", orig_text)
@@ -256,24 +283,39 @@ async def process_message(msg, source_name: str) -> None:
     if score < RELEVANCE_THRESHOLD:
         return
 
-    # 🔥 filter baris/kalimat promosi/DM/link
-    clean_text = filter_text_block(orig_text)
-    if not clean_text:
-        debug_log("Semua baris terblokir, dilewati", orig_text)
+    role = _role_of(source_chat_id)  # 'core' atau 'support'
+
+    # Tentukan keyword & kelas
+    main_kw = _extract_main_keyword(text_norm)
+    class_label = None
+    unique_groups = 0
+    if main_kw:
+        group_key = str(source_chat_id)
+        now = datetime.now(timezone.utc)
+        class_label, unique_groups = update_and_classify(main_kw, group_key, now)
+
+    # Gating: support hanya kirim jika unique_groups >= SUPPORT_MIN_UNIQUE
+    if role == "support" and unique_groups < SUPPORT_MIN_UNIQUE:
+        debug_log(f"Support ditahan (unique_groups={unique_groups} < {SUPPORT_MIN_UNIQUE})", orig_text)
         return
 
-    await send_as_is(msg, override_text=clean_text)
-    debug_log("Dikirim ke target", clean_text)
+    # Susun prefix kelas
+    text_to_send = orig_text
+    if class_label:
+        text_to_send = f"[{class_label.upper()}] {orig_text}"
+
+    await send_as_is(msg, text_override=text_to_send)
+    debug_log(f"Dikirim ke target (role={role}, unique_groups={unique_groups})", orig_text)
 
 
 async def backfill_history(entity, limit: int) -> None:
-    """Tarik pesan lama dari suatu source untuk diproses (opsional)."""
     if limit <= 0:
         return
     print(f"[Backfill] Tarik {limit} pesan terakhir dari {entity} ...")
     async for m in client.iter_messages(entity, limit=limit):
         try:
-            await process_message(m, source_name=str(entity))
+            # entity.id aman untuk identitas chat
+            await process_message(m, source_chat_id=entity.id)
         except Exception as e:
             debug_log("Error saat memproses backfill", str(e))
 
@@ -282,28 +324,37 @@ async def backfill_history(entity, limit: int) -> None:
 @client.on(events.NewMessage(chats=SOURCE_CHATS))
 async def on_new_message(event):
     try:
-        await process_message(event.message, source_name=str(event.chat_id))
+        await process_message(event.message, source_chat_id=event.chat_id)
     except Exception as e:
         print("Process error:", e)
 
 
 # ========= Entry points =========
+async def _resolve_and_tag_chats(raw_list, role_label: str) -> list:
+    """Resolve identifier menjadi entity + tandai perannya di chat_roles."""
+    resolved = []
+    for src in raw_list:
+        try:
+            ent = await client.get_entity(src)
+            resolved.append(ent)
+            chat_roles[int(ent.id)] = role_label
+        except Exception as e:
+            print(f"Gagal resolve sumber {src}: {e}")
+    return resolved
+
 async def start_bot_background() -> None:
     """
-    Dipanggil dari server FastAPI (server.py).
-    Menjalankan client + backfill tapi tidak memblokir event loop web-server.
+    Dipanggil dari server FastAPI (server.py) saat startup:contentReference[oaicite:3]{index=3}.
+    Menjalankan client + backfill tanpa memblokir web server.
     """
     await client.start()
 
-    # resolve semua sources
-    resolved_sources = []
-    for src in SOURCE_CHATS:
-        try:
-            ent = await client.get_entity(src)
-            resolved_sources.append(ent)
-        except Exception as e:
-            print(f"Gagal resolve sumber {src}: {e}")
+    # Resolve CORE & SUPPORT dan isi chat_roles
+    resolved_core = await _resolve_and_tag_chats(CORE_CHATS, "core")
+    resolved_support = await _resolve_and_tag_chats(SUPPORT_CHATS, "support")
+    resolved_sources = [*resolved_core, *resolved_support]
 
+    # Backfill opsional
     for ent in resolved_sources:
         try:
             await backfill_history(ent, INITIAL_BACKFILL)
@@ -311,24 +362,17 @@ async def start_bot_background() -> None:
             print(f"Backfill gagal untuk {ent}: {e}")
 
     print("Kurator berjalan (background task). Menunggu pesan baru...")
-    # Jangan blokir: jalankan client sampai disconnect sebagai task terpisah
     asyncio.create_task(client.run_until_disconnected())
 
-
 async def app_main() -> None:
     """
-    Jalur eksekusi legacy: biar bisa `python botsignal.py`
-    (blok hingga disconnect).
+    Mode legacy: `python botsignal.py` (blok hingga disconnect):contentReference[oaicite:4]{index=4}.
     """
     await client.start()
 
-    resolved_sources = []
-    for src in SOURCE_CHATS:
-        try:
-            ent = await client.get_entity(src)
-            resolved_sources.append(ent)
-        except Exception as e:
-            print(f"Gagal resolve sumber {src}: {e}")
+    resolved_core = await _resolve_and_tag_chats(CORE_CHATS, "core")
+    resolved_support = await _resolve_and_tag_chats(SUPPORT_CHATS, "support")
+    resolved_sources = [*resolved_core, *resolved_support]
 
     for ent in resolved_sources:
         await backfill_history(ent, INITIAL_BACKFILL)
@@ -338,7 +382,5 @@ async def app_main() -> None:
 
 
 if __name__ == "__main__":
-    # Hanya untuk run lokal; jangan di-import saat Uvicorn
-    import nest_asyncio
     nest_asyncio.apply()
     asyncio.run(app_main())