import os
import sys
import json
import asyncio
import time
from datetime import datetime, timezone

try:
    import aiohttp
    from bs4 import BeautifulSoup
except ImportError:
    print("Dependencies missing. Run: pip install aiohttp beautifulsoup4")
    sys.exit(1)

os.makedirs("e:/genesis-system/KNOWLEDGE_GRAPH/entities", exist_ok=True)
OPENROUTER_API_KEY = "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"
CONCURRENCY = 100

TARGET_URLS = [
    "https://developers.telnyx.com/docs/api/v2/call-control",
    "https://developers.telnyx.com/docs/api/v2/messaging",
    "https://developers.telnyx.com/docs/api/v2/webrtc",
    "https://developers.telnyx.com/docs/api/v2/ai-assistants"
]

OUTPUT_FILE = "e:/genesis-system/KNOWLEDGE_GRAPH/entities/telnyx_mastery.jsonl"

async def fetch_page(session, url, agent_id):
    try:
        async with session.get(url, timeout=30) as response:
            if response.status == 200:
                html = await response.text()
                soup = BeautifulSoup(html, 'html.parser')
                text_chunks = [p.get_text() for p in soup.find_all(['p', 'h1', 'h2', 'code', 'li'])]
                markdown_text = "\\n".join(text_chunks)
                print(f"[Agent-{agent_id:03d}] Scraped {url}")
                return url, markdown_text
            else:
                return url, None
    except Exception as e:
        return url, None

async def process_with_kimi(session, url, content, agent_id):
    if not content: return None
    print(f"[Agent-{agent_id:03d}] Triplifying {url} with Kimi K2.5...")
    headers = {"Authorization": f"Bearer {OPENROUTER_API_KEY}", "Content-Type": "application/json"}
    prompt = f"Convert this Telnyx doc to a structured JSON Knowledge Graph with canonical_nodes, triples (subject, predicate, object), summary, and code_snippets.\\nURL:{url}\\n\\nContent:{content[:15000]}"
    payload = {
        "model": "moonshotai/kimi-k2.5",
        "messages": [{"role": "user", "content": prompt}],
        "temperature": 0.1,
        "response_format": {"type": "json_object"}
    }
    try:
        async with session.post("https://openrouter.ai/api/v1/chat/completions", json=payload, headers=headers, timeout=120) as resp:
            if resp.status == 200:
                data = await resp.json()
                res = data["choices"][0]["message"]["content"].replace("```json", "").replace("```", "").strip()
                return json.loads(res)
            return None
    except Exception as e:
        return None

async def worker(queue, session, agent_id, output_file):
    while True:
        url = await queue.get()
        if url is None:
            queue.task_done()
            break
        scraped_url, scraped_content = await fetch_page(session, url, agent_id)
        if scraped_content:
            kg = await process_with_kimi(session, scraped_url, scraped_content, agent_id)
            if kg:
                with open(output_file, "a") as f:
                    kg["valid_from"] = datetime.now(timezone.utc).isoformat()
                    f.write(json.dumps(kg) + "\\n")
        queue.task_done()

async def main():
    print(f"OMNI-FORGE SWARM ACTIVATED: 100 Kimi K2.5 Agents targeting Telnyx")
    extended_urls = TARGET_URLS * 2  # synthetic load
    queue = asyncio.Queue()
    for url in extended_urls: queue.put_nowait(url)
    async with aiohttp.ClientSession() as session:
        workers = [asyncio.create_task(worker(queue, session, i+1, OUTPUT_FILE)) for i in range(CONCURRENCY)]
        for _ in range(CONCURRENCY): queue.put_nowait(None)
        await asyncio.gather(*workers)
    print("SWARM COMPLETE.")

if __name__ == "__main__":
    asyncio.run(main())