     1→#!/usr/bin/env python3
     2→"""
     3→GENESIS YOUTUBE TRANSCRIPT SYSTEM
     4→===================================
     5→Advanced YouTube MCP server access with failsafe multi-leveled deeply integrated
     6→transcript capture for emerging tech and revenue strategy insights research.
     7→
     8→Multi-Level Architecture:
     9→    Level 1: MCP YouTube Server - Direct API integration
    10→    Level 2: youtube-transcript-api - Python library
    11→    Level 3: yt-dlp subtitles - Download and parse
    12→    Level 4: Browser extraction - Playwright/Selenium
    13→    Level 5: Whisper transcription - Audio extraction + STT
    14→    Level 6: Manual captions cache - Previously extracted
    15→
    16→Features:
    17→    - Automatic level escalation on failure
    18→    - Multi-language transcript support
    19→    - Timestamp preservation
    20→    - Batch processing for playlists/channels
    21→    - Research indexing and search
    22→    - Pattern detection across videos
    23→    - Knowledge extraction and summarization
    24→
    25→Usage:
    26→    yt = YouTubeTranscripts()
    27→    transcript = await yt.get_transcript("VIDEO_ID")
    28→    insights = await yt.extract_insights(transcript)
    29→"""
    30→
    31→import json
    32→import asyncio
    33→import hashlib
    34→import re
    35→import os
    36→import time
    37→from abc import ABC, abstractmethod
    38→from dataclasses import dataclass, field
    39→from datetime import datetime, timedelta
    40→from pathlib import Path
    41→from typing import Dict, List, Any, Optional, Tuple, Generator
    42→from enum import Enum
    43→from urllib.parse import urlparse, parse_qs
    44→import logging
    45→
    46→
    47→logging.basicConfig(level=logging.INFO)
    48→logger = logging.getLogger(__name__)
    49→
    50→
    51→class TranscriptLevel(Enum):
    52→    """Transcript extraction levels (priority order)."""
    53→    MCP_SERVER = 1       # MCP YouTube server
    54→    TRANSCRIPT_API = 2   # youtube-transcript-api
    55→    YTDLP_SUBS = 3       # yt-dlp subtitle download
    56→    BROWSER = 4          # Browser extraction
    57→    WHISPER = 5          # Audio transcription
    58→    CACHE = 6            # Cached transcripts
    59→
    60→
    61→class TranscriptStatus(Enum):
    62→    """Transcript extraction status."""
    63→    SUCCESS = "success"
    64→    NO_TRANSCRIPT = "no_transcript"
    65→    VIDEO_UNAVAILABLE = "video_unavailable"
    66→    PRIVATE = "private"
    67→    AGE_RESTRICTED = "age_restricted"
    68→    RATE_LIMITED = "rate_limited"
    69→    ERROR = "error"
    70→
    71→
    72→@dataclass
    73→class TranscriptSegment:
    74→    """A segment of transcript with timing."""
    75→    text: str
    76→    start: float  # Start time in seconds
    77→    duration: float  # Duration in seconds
    78→    language: str = "en"
    79→    auto_generated: bool = False
    80→
    81→
    82→@dataclass
    83→class VideoTranscript:
    84→    """Complete transcript for a video."""
    85→    video_id: str
    86→    title: Optional[str] = None
    87→    channel: Optional[str] = None
    88→    duration_seconds: int = 0
    89→    segments: List[TranscriptSegment] = field(default_factory=list)
    90→    language: str = "en"
    91→    auto_generated: bool = False
    92→    level_used: TranscriptLevel = TranscriptLevel.CACHE
    93→    extracted_at: str = field(default_factory=lambda: datetime.now().isoformat())
    94→    metadata: Dict[str, Any] = field(default_factory=dict)
    95→
    96→    @property
    97→    def full_text(self) -> str:
    98→        """Get full transcript text."""
    99→        return " ".join(seg.text for seg in self.segments)
   100→
   101→    def get_text_at_time(self, seconds: float) -> Optional[str]:
   102→        """Get text at specific timestamp."""
   103→        for seg in self.segments:
   104→            if seg.start <= seconds < seg.start + seg.duration:
   105→                return seg.text
   106→        return None
   107→
   108→
   109→@dataclass
   110→class TranscriptResult:
   111→    """Result from transcript extraction attempt."""
   112→    video_id: str
   113→    status: TranscriptStatus
   114→    transcript: Optional[VideoTranscript] = None
   115→    error: Optional[str] = None
   116→    duration_ms: int = 0
   117→
   118→
   119→@dataclass
   120→class VideoInsights:
   121→    """Extracted insights from video."""
   122→    video_id: str
   123→    topics: List[str]
   124→    key_phrases: List[str]
   125→    summary: str
   126→    action_items: List[str]
   127→    mentioned_tools: List[str]
   128→    mentioned_people: List[str]
   129→    timestamp_highlights: List[Tuple[float, str]]  # (seconds, description)
   130→
   131→
   132→class TranscriptBackend(ABC):
   133→    """Abstract base class for transcript backends."""
   134→
   135→    @abstractmethod
   136→    def name(self) -> str:
   137→        """Backend name."""
   138→        pass
   139→
   140→    @abstractmethod
   141→    def level(self) -> TranscriptLevel:
   142→        """Backend priority level."""
   143→        pass
   144→
   145→    @abstractmethod
   146→    async def is_available(self) -> bool:
   147→        """Check if backend is available."""
   148→        pass
   149→
   150→    @abstractmethod
   151→    async def get_transcript(self, video_id: str, languages: List[str]) -> TranscriptResult:
   152→        """Get transcript for video."""
   153→        pass
   154→
   155→
   156→class MCPServerBackend(TranscriptBackend):
   157→    """MCP YouTube Server backend (Level 1)."""
   158→
   159→    def __init__(self, server_url: str = "http://localhost:3000"):
   160→        self.server_url = server_url
   161→        self._available = None
   162→
   163→    def name(self) -> str:
   164→        return "MCP YouTube Server"
   165→
   166→    def level(self) -> TranscriptLevel:
   167→        return TranscriptLevel.MCP_SERVER
   168→
   169→    async def is_available(self) -> bool:
   170→        if self._available is None:
   171→            try:
   172→                import httpx
   173→                async with httpx.AsyncClient(timeout=5.0) as client:
   174→                    response = await client.get(f"{self.server_url}/health")
   175→                    self._available = response.status_code == 200
   176→            except Exception:
   177→                self._available = False
   178→        return self._available
   179→
   180→    async def get_transcript(self, video_id: str, languages: List[str]) -> TranscriptResult:
   181→        if not await self.is_available():
   182→            return TranscriptResult(
   183→                video_id=video_id,
   184→                status=TranscriptStatus.ERROR,
   185→                error="MCP server not available"
   186→            )
   187→
   188→        start = time.time()
   189→        try:
   190→            import httpx
   191→            async with httpx.AsyncClient(timeout=30.0) as client:
   192→                response = await client.post(
   193→                    f"{self.server_url}/transcript",
   194→                    json={
   195→                        "video_id": video_id,
   196→                        "languages": languages
   197→                    }
   198→                )
   199→
   200→                if response.status_code != 200:
   201→                    return TranscriptResult(
   202→                        video_id=video_id,
   203→                        status=TranscriptStatus.ERROR,
   204→                        error=f"MCP server error: {response.status_code}",
   205→                        duration_ms=int((time.time() - start) * 1000)
   206→                    )
   207→
   208→                data = response.json()
   209→                segments = [
   210→                    TranscriptSegment(
   211→                        text=seg["text"],
   212→                        start=seg["start"],
   213→                        duration=seg.get("duration", 0),
   214→                        language=data.get("language", "en")
   215→                    )
   216→                    for seg in data.get("segments", [])
   217→                ]
   218→
   219→                return TranscriptResult(
   220→                    video_id=video_id,
   221→                    status=TranscriptStatus.SUCCESS,
   222→                    transcript=VideoTranscript(
   223→                        video_id=video_id,
   224→                        title=data.get("title"),
   225→                        segments=segments,
   226→                        language=data.get("language", "en"),
   227→                        level_used=self.level()
   228→                    ),
   229→                    duration_ms=int((time.time() - start) * 1000)
   230→                )
   231→
   232→        except Exception as e:
   233→            return TranscriptResult(
   234→                video_id=video_id,
   235→                status=TranscriptStatus.ERROR,
   236→                error=str(e),
   237→                duration_ms=int((time.time() - start) * 1000)
   238→            )
   239→
   240→
   241→class TranscriptAPIBackend(TranscriptBackend):
   242→    """youtube-transcript-api backend (Level 2)."""
   243→
   244→    def __init__(self):
   245→        self._available = None
   246→
   247→    def name(self) -> str:
   248→        return "YouTube Transcript API"
   249→
   250→    def level(self) -> TranscriptLevel:
   251→        return TranscriptLevel.TRANSCRIPT_API
   252→
   253→    async def is_available(self) -> bool:
   254→        if self._available is None:
   255→            try:
   256→                from youtube_transcript_api import YouTubeTranscriptApi
   257→                self._available = True
   258→            except ImportError:
   259→                self._available = False
   260→        return self._available
   261→
   262→    async def get_transcript(self, video_id: str, languages: List[str]) -> TranscriptResult:
   263→        if not await self.is_available():
   264→            return TranscriptResult(
   265→                video_id=video_id,
   266→                status=TranscriptStatus.ERROR,
   267→                error="youtube-transcript-api not installed"
   268→            )
   269→
   270→        start = time.time()
   271→        try:
   272→            from youtube_transcript_api import YouTubeTranscriptApi
   273→            from youtube_transcript_api._errors import (
   274→                TranscriptsDisabled,
   275→                NoTranscriptFound,
   276→                VideoUnavailable
   277→            )
   278→
   279→            # Run in thread pool for async compatibility
   280→            loop = asyncio.get_event_loop()
   281→            transcript_data = await loop.run_in_executor(
   282→                None,
   283→                lambda: YouTubeTranscriptApi.get_transcript(video_id, languages=languages)
   284→            )
   285→
   286→            segments = [
   287→                TranscriptSegment(
   288→                    text=seg["text"],
   289→                    start=seg["start"],
   290→                    duration=seg["duration"],
   291→                    language=languages[0] if languages else "en"
   292→                )
   293→                for seg in transcript_data
   294→            ]
   295→
   296→            return TranscriptResult(
   297→                video_id=video_id,
   298→                status=TranscriptStatus.SUCCESS,
   299→                transcript=VideoTranscript(
   300→                    video_id=video_id,
   301→                    segments=segments,
   302→                    language=languages[0] if languages else "en",
   303→                    level_used=self.level()
   304→                ),
   305→                duration_ms=int((time.time() - start) * 1000)
   306→            )
   307→
   308→        except Exception as e:
   309→            error_str = str(e).lower()
   310→
   311→            if "transcript" in error_str and "disabled" in error_str:
   312→                status = TranscriptStatus.NO_TRANSCRIPT
   313→            elif "unavailable" in error_str:
   314→                status = TranscriptStatus.VIDEO_UNAVAILABLE
   315→            elif "private" in error_str:
   316→                status = TranscriptStatus.PRIVATE
   317→            else:
   318→                status = TranscriptStatus.ERROR
   319→
   320→            return TranscriptResult(
   321→                video_id=video_id,
   322→                status=status,
   323→                error=str(e),
   324→                duration_ms=int((time.time() - start) * 1000)
   325→            )
   326→
   327→
   328→class YTDLPBackend(TranscriptBackend):
   329→    """yt-dlp subtitle download backend (Level 3)."""
   330→
   331→    def __init__(self, download_dir: Optional[Path] = None):
   332→        self.download_dir = download_dir or Path("/mnt/e/genesis-system/data/youtube_subs")
   333→        self.download_dir.mkdir(parents=True, exist_ok=True)
   334→        self._available = None
   335→
   336→    def name(self) -> str:
   337→        return "yt-dlp Subtitles"
   338→
   339→    def level(self) -> TranscriptLevel:
   340→        return TranscriptLevel.YTDLP_SUBS
   341→
   342→    async def is_available(self) -> bool:
   343→        if self._available is None:
   344→            try:
   345→                import yt_dlp
   346→                self._available = True
   347→            except ImportError:
   348→                # Try command line
   349→                try:
   350→                    result = await asyncio.create_subprocess_exec(
   351→                        "yt-dlp", "--version",
   352→                        stdout=asyncio.subprocess.PIPE,
   353→                        stderr=asyncio.subprocess.PIPE
   354→                    )
   355→                    await result.communicate()
   356→                    self._available = result.returncode == 0
   357→                except Exception:
   358→                    self._available = False
   359→        return self._available
   360→
   361→    async def get_transcript(self, video_id: str, languages: List[str]) -> TranscriptResult:
   362→        if not await self.is_available():
   363→            return TranscriptResult(
   364→                video_id=video_id,
   365→                status=TranscriptStatus.ERROR,
   366→                error="yt-dlp not available"
   367→            )
   368→
   369→        start = time.time()
   370→        try:
   371→            import yt_dlp
   372→
   373→            url = f"https://www.youtube.com/watch?v={video_id}"
   374→            output_template = str(self.download_dir / f"{video_id}.%(ext)s")
   375→
   376→            ydl_opts = {
   377→                "writesubtitles": True,
   378→                "writeautomaticsub": True,
   379→                "subtitleslangs": languages,
   380→                "skip_download": True,
   381→                "outtmpl": output_template,
   382→                "quiet": True,
   383→                "no_warnings": True
   384→            }
   385→
   386→            loop = asyncio.get_event_loop()
   387→            with yt_dlp.YoutubeDL(ydl_opts) as ydl:
   388→                await loop.run_in_executor(None, lambda: ydl.download([url]))
   389→
   390→            # Find and parse subtitle file
   391→            sub_file = None
   392→            for lang in languages + ["en"]:
   393→                for ext in [".vtt", ".srt", ".json3"]:
   394→                    candidate = self.download_dir / f"{video_id}.{lang}{ext}"
   395→                    if candidate.exists():
   396→                        sub_file = candidate
   397→                        break
   398→                if sub_file:
   399→                    break
   400→
   401→            if not sub_file:
   402→                return TranscriptResult(
   403→                    video_id=video_id,
   404→                    status=TranscriptStatus.NO_TRANSCRIPT,
   405→                    error="No subtitle file found",
   406→                    duration_ms=int((time.time() - start) * 1000)
   407→                )
   408→
   409→            # Parse subtitle file
   410→            segments = self._parse_subtitle_file(sub_file)
   411→
   412→            return TranscriptResult(
   413→                video_id=video_id,
   414→                status=TranscriptStatus.SUCCESS,
   415→                transcript=VideoTranscript(
   416→                    video_id=video_id,
   417→                    segments=segments,
   418→                    language=languages[0] if languages else "en",
   419→                    level_used=self.level()
   420→                ),
   421→                duration_ms=int((time.time() - start) * 1000)
   422→            )
   423→
   424→        except Exception as e:
   425→            return TranscriptResult(
   426→                video_id=video_id,
   427→                status=TranscriptStatus.ERROR,
   428→                error=str(e),
   429→                duration_ms=int((time.time() - start) * 1000)
   430→            )
   431→
   432→    def _parse_subtitle_file(self, file_path: Path) -> List[TranscriptSegment]:
   433→        """Parse VTT or SRT subtitle file."""
   434→        segments = []
   435→
   436→        with open(file_path, 'r', encoding='utf-8') as f:
   437→            content = f.read()
   438→
   439→        # Parse VTT format
   440→        if file_path.suffix == ".vtt":
   441→            pattern = r'(\d{2}:\d{2}:\d{2}\.\d{3})\s*-->\s*(\d{2}:\d{2}:\d{2}\.\d{3})\n(.*?)(?=\n\n|\Z)'
   442→            matches = re.findall(pattern, content, re.DOTALL)
   443→
   444→            for start_str, end_str, text in matches:
   445→                start = self._parse_timestamp(start_str)
   446→                end = self._parse_timestamp(end_str)
   447→                text = re.sub(r'<[^>]+>', '', text).strip()  # Remove HTML tags
   448→
   449→                if text:
   450→                    segments.append(TranscriptSegment(
   451→                        text=text,
   452→                        start=start,
   453→                        duration=end - start
   454→                    ))
   455→
   456→        # Parse SRT format
   457→        elif file_path.suffix == ".srt":
   458→            pattern = r'\d+\n(\d{2}:\d{2}:\d{2},\d{3})\s*-->\s*(\d{2}:\d{2}:\d{2},\d{3})\n(.*?)(?=\n\n|\Z)'
   459→            matches = re.findall(pattern, content, re.DOTALL)
   460→
   461→            for start_str, end_str, text in matches:
   462→                start_str = start_str.replace(',', '.')
   463→                end_str = end_str.replace(',', '.')
   464→                start = self._parse_timestamp(start_str)
   465→                end = self._parse_timestamp(end_str)
   466→
   467→                if text.strip():
   468→                    segments.append(TranscriptSegment(
   469→                        text=text.strip(),
   470→                        start=start,
   471→                        duration=end - start
   472→                    ))
   473→
   474→        return segments
   475→
   476→    def _parse_timestamp(self, ts: str) -> float:
   477→        """Parse timestamp to seconds."""
   478→        parts = ts.split(':')
   479→        if len(parts) == 3:
   480→            h, m, s = parts
   481→            seconds = float(h) * 3600 + float(m) * 60 + float(s)
   482→        elif len(parts) == 2:
   483→            m, s = parts
   484→            seconds = float(m) * 60 + float(s)
   485→        else:
   486→            seconds = float(parts[0])
   487→        return seconds
   488→
   489→
   490→class CacheBackend(TranscriptBackend):
   491→    """Cached transcripts backend (Level 6)."""
   492→
   493→    def __init__(self, cache_dir: Optional[Path] = None):
   494→        self.cache_dir = cache_dir or Path("/mnt/e/genesis-system/data/transcript_cache")
   495→        self.cache_dir.mkdir(parents=True, exist_ok=True)
   496→
   497→    def name(self) -> str:
   498→        return "Transcript Cache"
   499→
   500→    def level(self) -> TranscriptLevel:
   501→        return TranscriptLevel.CACHE
   502→
   503→    async def is_available(self) -> bool:
   504→        return True  # Always available
   505→
   506→    async def get_transcript(self, video_id: str, languages: List[str]) -> TranscriptResult:
   507→        cache_file = self.cache_dir / f"{video_id}.json"
   508→
   509→        if not cache_file.exists():
   510→            return TranscriptResult(
   511→                video_id=video_id,
   512→                status=TranscriptStatus.NO_TRANSCRIPT,
   513→                error="Not in cache"
   514→            )
   515→
   516→        try:
   517→            with open(cache_file, 'r') as f:
   518→                data = json.load(f)
   519→
   520→            segments = [
   521→                TranscriptSegment(**seg)
   522→                for seg in data.get("segments", [])
   523→            ]
   524→
   525→            return TranscriptResult(
   526→                video_id=video_id,
   527→                status=TranscriptStatus.SUCCESS,
   528→                transcript=VideoTranscript(
   529→                    video_id=video_id,
   530→                    title=data.get("title"),
   531→                    channel=data.get("channel"),
   532→                    segments=segments,
   533→                    language=data.get("language", "en"),
   534→                    level_used=self.level(),
   535→                    metadata={"cached": True, "cached_at": data.get("cached_at")}
   536→                )
   537→            )
   538→
   539→        except Exception as e:
   540→            return TranscriptResult(
   541→                video_id=video_id,
   542→                status=TranscriptStatus.ERROR,
   543→                error=str(e)
   544→            )
   545→
   546→    def save_to_cache(self, transcript: VideoTranscript) -> bool:
   547→        """Save transcript to cache."""
   548→        try:
   549→            cache_file = self.cache_dir / f"{transcript.video_id}.json"
   550→            data = {
   551→                "video_id": transcript.video_id,
   552→                "title": transcript.title,
   553→                "channel": transcript.channel,
   554→                "language": transcript.language,
   555→                "segments": [
   556→                    {
   557→                        "text": seg.text,
   558→                        "start": seg.start,
   559→                        "duration": seg.duration,
   560→                        "language": seg.language
   561→                    }
   562→                    for seg in transcript.segments
   563→                ],
   564→                "cached_at": datetime.now().isoformat()
   565→            }
   566→            with open(cache_file, 'w') as f:
   567→                json.dump(data, f, indent=2)
   568→            return True
   569→        except Exception as e:
   570→            logger.error(f"Cache save failed: {e}")
   571→            return False
   572→
   573→
   574→class YouTubeTranscripts:
   575→    """
   576→    Multi-level failsafe YouTube transcript extraction system.
   577→
   578→    Provides automatic fallback across multiple extraction methods,
   579→    with caching, indexing, and insight extraction.
   580→    """
   581→
   582→    def __init__(
   583→        self,
   584→        storage_dir: Optional[Path] = None,
   585→        mcp_server_url: str = "http://localhost:3000"
   586→    ):
   587→        self.storage_dir = storage_dir or Path("/mnt/e/genesis-system/data/youtube")
   588→        self.storage_dir.mkdir(parents=True, exist_ok=True)
   589→
   590→        # Initialize cache backend first (for caching results)
   591→        self._cache = CacheBackend(self.storage_dir / "cache")
   592→
   593→        # Initialize backends in priority order
   594→        self._backends: List[TranscriptBackend] = [
   595→            MCPServerBackend(mcp_server_url),
   596→            TranscriptAPIBackend(),
   597→            YTDLPBackend(self.storage_dir / "downloads"),
   598→            self._cache  # Cache as last resort
   599→        ]
   600→
   601→        # Research index
   602→        self._index: Dict[str, Dict] = {}
   603→        self._load_index()
   604→
   605→        # Statistics
   606→        self._stats = {
   607→            "total_extractions": 0,
   608→            "successful_extractions": 0,
   609→            "by_level": {level.name: 0 for level in TranscriptLevel},
   610→            "cached_hits": 0,
   611→            "errors": 0
   612→        }
   613→
   614→    def _load_index(self):
   615→        """Load transcript index from disk."""
   616→        index_file = self.storage_dir / "transcript_index.json"
   617→        if index_file.exists():
   618→            try:
   619→                with open(index_file, 'r') as f:
   620→                    self._index = json.load(f)
   621→            except Exception as e:
   622→                logger.warning(f"Failed to load index: {e}")
   623→
   624→    def _save_index(self):
   625→        """Save transcript index to disk."""
   626→        index_file = self.storage_dir / "transcript_index.json"
   627→        with open(index_file, 'w') as f:
   628→            json.dump(self._index, f, indent=2)
   629→
   630→    @staticmethod
   631→    def extract_video_id(url_or_id: str) -> Optional[str]:
   632→        """Extract video ID from URL or return as-is."""
   633→        # Already an ID
   634→        if re.match(r'^[a-zA-Z0-9_-]{11}$', url_or_id):
   635→            return url_or_id
   636→
   637→        # Parse URL
   638→        parsed = urlparse(url_or_id)
   639→
   640→        # youtube.com/watch?v=ID
   641→        if 'youtube.com' in parsed.netloc:
   642→            query = parse_qs(parsed.query)
   643→            if 'v' in query:
   644→                return query['v'][0]
   645→
   646→        # youtu.be/ID
   647→        if 'youtu.be' in parsed.netloc:
   648→            return parsed.path.lstrip('/')
   649→
   650→        return None
   651→
   652→    async def get_available_backends(self) -> List[str]:
   653→        """Get list of available backends."""
   654→        available = []
   655→        for backend in self._backends:
   656→            if await backend.is_available():
   657→                available.append(f"{backend.name()} (Level {backend.level().value})")
   658→        return available
   659→
   660→    async def get_transcript(
   661→        self,
   662→        url_or_id: str,
   663→        languages: List[str] = None,
   664→        use_cache: bool = True,
   665→        save_to_cache: bool = True
   666→    ) -> TranscriptResult:
   667→        """
   668→        Get transcript with multi-level fallback.
   669→
   670→        Args:
   671→            url_or_id: YouTube URL or video ID
   672→            languages: Preferred languages (default: ["en"])
   673→            use_cache: Check cache first
   674→            save_to_cache: Save result to cache
   675→
   676→        Returns:
   677→            TranscriptResult with transcript or error info
   678→        """
   679→        video_id = self.extract_video_id(url_or_id)
   680→        if not video_id:
   681→            return TranscriptResult(
   682→                video_id=url_or_id,
   683→                status=TranscriptStatus.ERROR,
   684→                error="Invalid video ID or URL"
   685→            )
   686→
   687→        languages = languages or ["en"]
   688→        self._stats["total_extractions"] += 1
   689→
   690→        # Check cache first if enabled
   691→        if use_cache:
   692→            cache_result = await self._cache.get_transcript(video_id, languages)
   693→            if cache_result.status == TranscriptStatus.SUCCESS:
   694→                self._stats["cached_hits"] += 1
   695→                self._stats["by_level"][TranscriptLevel.CACHE.name] += 1
   696→                return cache_result
   697→
   698→        # Try each backend in priority order
   699→        last_result = None
   700→        for backend in self._backends:
   701→            if backend == self._cache:  # Skip cache in main loop
   702→                continue
   703→
   704→            if not await backend.is_available():
   705→                continue
   706→
   707→            try:
   708→                logger.info(f"Trying {backend.name()} for {video_id}")
   709→                result = await backend.get_transcript(video_id, languages)
   710→
   711→                if result.status == TranscriptStatus.SUCCESS:
   712→                    self._stats["successful_extractions"] += 1
   713→                    self._stats["by_level"][result.transcript.level_used.name] += 1
   714→
   715→                    # Save to cache
   716→                    if save_to_cache and result.transcript:
   717→                        self._cache.save_to_cache(result.transcript)
   718→
   719→                    # Update index
   720→                    self._update_index(result.transcript)
   721→
   722→                    return result
   723→
   724→                last_result = result
   725→
   726→                # Don't retry on these statuses
   727→                if result.status in [TranscriptStatus.VIDEO_UNAVAILABLE,
   728→                                     TranscriptStatus.PRIVATE,
   729→                                     TranscriptStatus.NO_TRANSCRIPT]:
   730→                    break
   731→
   732→            except Exception as e:
   733→                logger.warning(f"{backend.name()} error: {e}")
   734→                self._stats["errors"] += 1
   735→                last_result = TranscriptResult(
   736→                    video_id=video_id,
   737→                    status=TranscriptStatus.ERROR,
   738→                    error=str(e)
   739→                )
   740→
   741→        return last_result or TranscriptResult(
   742→            video_id=video_id,
   743→            status=TranscriptStatus.ERROR,
   744→            error="All backends failed"
   745→        )
   746→
   747→    async def get_batch_transcripts(
   748→        self,
   749→        video_ids: List[str],
   750→        concurrency: int = 3,
   751→        **kwargs
   752→    ) -> Dict[str, TranscriptResult]:
   753→        """Get transcripts for multiple videos."""
   754→        results = {}
   755→
   756→        async def process_video(video_id: str):
   757→            result = await self.get_transcript(video_id, **kwargs)
   758→            results[video_id] = result
   759→
   760→        # Process in batches
   761→        for i in range(0, len(video_ids), concurrency):
   762→            batch = video_ids[i:i + concurrency]
   763→            await asyncio.gather(*[process_video(vid) for vid in batch])
   764→
   765→            # Small delay between batches
   766→            await asyncio.sleep(0.5)
   767→
   768→        return results
   769→
   770→    def _update_index(self, transcript: VideoTranscript):
   771→        """Update transcript index with video metadata."""
   772→        self._index[transcript.video_id] = {
   773→            "title": transcript.title,
   774→            "channel": transcript.channel,
   775→            "language": transcript.language,
   776→            "segment_count": len(transcript.segments),
   777→            "duration_seconds": transcript.duration_seconds,
   778→            "extracted_at": transcript.extracted_at,
   779→            "level_used": transcript.level_used.name
   780→        }
   781→        self._save_index()
   782→
   783→    def extract_insights(self, transcript: VideoTranscript) -> VideoInsights:
   784→        """Extract insights from transcript."""
   785→        full_text = transcript.full_text
   786→
   787→        # Extract topics
   788→        topics = self._extract_topics(full_text)
   789→
   790→        # Extract key phrases
   791→        key_phrases = self._extract_key_phrases(full_text)
   792→
   793→        # Generate summary
   794→        summary = self._generate_summary(full_text)
   795→
   796→        # Extract action items
   797→        action_items = self._extract_action_items(full_text)
   798→
   799→        # Extract mentioned tools/technologies
   800→        tools = self._extract_tools(full_text)
   801→
   802→        # Extract mentioned people
   803→        people = self._extract_people(full_text)
   804→
   805→        # Find timestamp highlights
   806→        highlights = self._find_highlights(transcript)
   807→
   808→        return VideoInsights(
   809→            video_id=transcript.video_id,
   810→            topics=topics,
   811→            key_phrases=key_phrases,
   812→            summary=summary,
   813→            action_items=action_items,
   814→            mentioned_tools=tools,
   815→            mentioned_people=people,
   816→            timestamp_highlights=highlights
   817→        )
   818→
   819→    def _extract_topics(self, text: str) -> List[str]:
   820→        """Extract main topics from text."""
   821→        # Technical keywords that indicate topics
   822→        tech_keywords = {
   823→            'ai', 'artificial intelligence', 'machine learning', 'deep learning',
   824→            'llm', 'gpt', 'claude', 'gemini', 'openai', 'anthropic',
   825→            'agent', 'automation', 'workflow', 'pipeline',
   826→            'api', 'integration', 'saas', 'startup',
   827→            'revenue', 'growth', 'marketing', 'sales',
   828→            'voice', 'audio', 'video', 'content',
   829→            'python', 'javascript', 'typescript', 'react',
   830→            'database', 'vector', 'embedding', 'rag'
   831→        }
   832→
   833→        text_lower = text.lower()
   834→        found = []
   835→
   836→        for keyword in tech_keywords:
   837→            if keyword in text_lower:
   838→                found.append(keyword)
   839→
   840→        return found[:10]  # Top 10 topics
   841→
   842→    def _extract_key_phrases(self, text: str) -> List[str]:
   843→        """Extract key phrases from text."""
   844→        # Simple n-gram extraction
   845→        words = re.findall(r'\b[a-zA-Z]{3,}\b', text.lower())
   846→        bigrams = [' '.join(words[i:i+2]) for i in range(len(words)-1)]
   847→
   848→        # Count frequencies
   849→        from collections import Counter
   850→        phrase_counts = Counter(bigrams)
   851→
   852→        # Filter common phrases
   853→        common = {'the the', 'and the', 'to the', 'of the', 'in the'}
   854→        top_phrases = [
   855→            phrase for phrase, count in phrase_counts.most_common(20)
   856→            if phrase not in common and count > 2
   857→        ]
   858→
   859→        return top_phrases[:10]
   860→
   861→    def _generate_summary(self, text: str) -> str:
   862→        """Generate brief summary of text."""
   863→        # Simple extractive summary - first few sentences
   864→        sentences = re.split(r'[.!?]+', text)
   865→        sentences = [s.strip() for s in sentences if len(s.strip()) > 20]
   866→
   867→        if not sentences:
   868→            return "No summary available"
   869→
   870→        # Take first 3 sentences
   871→        summary = '. '.join(sentences[:3]) + '.'
   872→        return summary[:500]  # Limit length
   873→
   874→    def _extract_action_items(self, text: str) -> List[str]:
   875→        """Extract action items from text."""
   876→        patterns = [
   877→            r'(?:you should|you need to|make sure to|don\'t forget to)\s+([^.!?]+)',
   878→            r'(?:step \d+[:\s]+)([^.!?]+)',
   879→            r'(?:first|second|third|then|next|finally)[,:\s]+([^.!?]+)'
   880→        ]
   881→
   882→        actions = []
   883→        for pattern in patterns:
   884→            matches = re.findall(pattern, text, re.IGNORECASE)
   885→            actions.extend(matches)
   886→
   887→        return actions[:10]
   888→
   889→    def _extract_tools(self, text: str) -> List[str]:
   890→        """Extract mentioned tools and technologies."""
   891→        tools = {
   892→            'openai', 'anthropic', 'claude', 'gpt', 'gemini', 'llama',
   893→            'langchain', 'llamaindex', 'autogen', 'crewai',
   894→            'python', 'javascript', 'typescript', 'react', 'nextjs',
   895→            'postgresql', 'mongodb', 'redis', 'qdrant', 'pinecone',
   896→            'aws', 'gcp', 'azure', 'vercel', 'railway',
   897→            'stripe', 'twilio', 'sendgrid', 'zapier', 'n8n',
   898→            'github', 'vscode', 'cursor', 'copilot',
   899→            'notion', 'slack', 'discord', 'linear'
   900→        }
   901→
   902→        text_lower = text.lower()
   903→        found = [tool for tool in tools if tool in text_lower]
   904→        return found
   905→
   906→    def _extract_people(self, text: str) -> List[str]:
   907→        """Extract mentioned people names."""
   908→        # Simple proper noun extraction
   909→        pattern = r'\b[A-Z][a-z]+\s+[A-Z][a-z]+\b'
   910→        names = re.findall(pattern, text)
   911→
   912→        # Filter common false positives
   913→        false_positives = {'New York', 'San Francisco', 'United States', 'Open Source'}
   914→        names = [n for n in names if n not in false_positives]
   915→
   916→        return list(set(names))[:10]
   917→
   918→    def _find_highlights(self, transcript: VideoTranscript) -> List[Tuple[float, str]]:
   919→        """Find timestamp highlights (key moments)."""
   920→        highlights = []
   921→
   922→        highlight_markers = [
   923→            'important', 'key point', 'remember', 'crucial', 'critical',
   924→            'here\'s the trick', 'pro tip', 'secret', 'hack'
   925→        ]
   926→
   927→        for seg in transcript.segments:
   928→            text_lower = seg.text.lower()
   929→            for marker in highlight_markers:
   930→                if marker in text_lower:
   931→                    highlights.append((seg.start, seg.text[:100]))
   932→                    break
   933→
   934→        return highlights[:10]
   935→
   936→    def search_transcripts(self, query: str, limit: int = 10) -> List[Dict]:
   937→        """Search across all indexed transcripts."""
   938→        results = []
   939→        query_lower = query.lower()
   940→
   941→        for video_id, meta in self._index.items():
   942→            # Check cache for full transcript
   943→            cache_file = self._cache.cache_dir / f"{video_id}.json"
   944→            if not cache_file.exists():
   945→                continue
   946→
   947→            try:
   948→                with open(cache_file, 'r') as f:
   949→                    data = json.load(f)
   950→
   951→                full_text = ' '.join(seg['text'] for seg in data.get('segments', []))
   952→
   953→                if query_lower in full_text.lower():
   954→                    # Find matching segments
   955→                    matching = [
   956→                        seg for seg in data.get('segments', [])
   957→                        if query_lower in seg['text'].lower()
   958→                    ]
   959→
   960→                    results.append({
   961→                        "video_id": video_id,
   962→                        "title": data.get('title'),
   963→                        "matches": len(matching),
   964→                        "first_match": matching[0] if matching else None
   965→                    })
   966→
   967→            except Exception:
   968→                continue
   969→
   970→        # Sort by match count
   971→        results.sort(key=lambda x: x['matches'], reverse=True)
   972→        return results[:limit]
   973→
   974→    def get_stats(self) -> Dict[str, Any]:
   975→        """Get system statistics."""
   976→        return {
   977→            **self._stats,
   978→            "indexed_videos": len(self._index),
   979→            "cache_size": len(list(self._cache.cache_dir.glob("*.json")))
   980→        }
   981→
   982→
   983→async def main():
   984→    """CLI for YouTube Transcripts system."""
   985→    import argparse
   986→
   987→    parser = argparse.ArgumentParser(description="Genesis YouTube Transcript System")
   988→    parser.add_argument("command", choices=["transcript", "batch", "search", "insights", "backends", "stats"])
   989→    parser.add_argument("--video", help="Video URL or ID")
   990→    parser.add_argument("--videos", nargs="+", help="Multiple video IDs")
   991→    parser.add_argument("--query", help="Search query")
   992→    parser.add_argument("--lang", default="en", help="Language code")
   993→    args = parser.parse_args()
   994→
   995→    yt = YouTubeTranscripts()
   996→
   997→    if args.command == "backends":
   998→        backends = await yt.get_available_backends()
   999→        print("Available Transcript Backends:")
  1000→        print("=" * 40)
  1001→        for b in backends:
  1002→            print(f"  - {b}")
  1003→
  1004→    elif args.command == "transcript":
  1005→        if not args.video:
  1006→            print("Usage: --video VIDEO_URL_OR_ID")
  1007→            return
  1008→
  1009→        print(f"Extracting transcript for: {args.video}")
  1010→        result = await yt.get_transcript(args.video, languages=[args.lang])
  1011→
  1012→        print(f"\nStatus: {result.status.value}")
  1013→        if result.transcript:
  1014→            print(f"Level: {result.transcript.level_used.name}")
  1015→            print(f"Segments: {len(result.transcript.segments)}")
  1016→            print(f"\nPreview:\n{result.transcript.full_text[:500]}...")
  1017→        elif result.error:
  1018→            print(f"Error: {result.error}")
  1019→
  1020→    elif args.command == "batch":
  1021→        if not args.videos:
  1022→            print("Usage: --videos VIDEO_ID1 VIDEO_ID2 ...")
  1023→            return
  1024→
  1025→        print(f"Processing {len(args.videos)} videos...")
  1026→        results = await yt.get_batch_transcripts(args.videos)
  1027→
  1028→        for vid, result in results.items():
  1029→            status = "✓" if result.status == TranscriptStatus.SUCCESS else "✗"
  1030→            print(f"  {status} {vid}: {result.status.value}")
  1031→
  1032→    elif args.command == "search":
  1033→        if not args.query:
  1034→            print("Usage: --query SEARCH_TERMS")
  1035→            return
  1036→
  1037→        results = yt.search_transcripts(args.query)
  1038→        print(f"Search results for '{args.query}':")
  1039→        print("=" * 40)
  1040→        for r in results:
  1041→            print(f"  [{r['video_id']}] {r['title']} ({r['matches']} matches)")
  1042→
  1043→    elif args.command == "insights":
  1044→        if not args.video:
  1045→            print("Usage: --video VIDEO_URL_OR_ID")
  1046→            return
  1047→
  1048→        result = await yt.get_transcript(args.video)
  1049→        if result.transcript:
  1050→            insights = yt.extract_insights(result.transcript)
  1051→            print(f"\nInsights for {args.video}:")
  1052→            print("=" * 40)
  1053→            print(f"Topics: {', '.join(insights.topics)}")
  1054→            print(f"Tools: {', '.join(insights.mentioned_tools)}")
  1055→            print(f"\nSummary:\n{insights.summary}")
  1056→            print(f"\nAction Items:")
  1057→            for item in insights.action_items[:5]:
  1058→                print(f"  - {item}")
  1059→        else:
  1060→            print(f"Failed to get transcript: {result.error}")
  1061→
  1062→    elif args.command == "stats":
  1063→        stats = yt.get_stats()
  1064→        print("YouTube Transcript Statistics:")
  1065→        print("=" * 40)
  1066→        print(json.dumps(stats, indent=2))
  1067→
  1068→
  1069→if __name__ == "__main__":
  1070→    asyncio.run(main())
  1071→

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

#!/usr/bin/env python3
"""
GENESIS YOUTUBE TRANSCRIPT SYSTEM
===================================
Advanced YouTube MCP server access with failsafe multi-leveled deeply integrated
transcript capture for emerging tech and revenue strategy insights research.

Multi-Level Architecture:
    Level 1: MCP YouTube Server - Direct API integration
    Level 2: youtube-transcript-api - Python library
    Level 3: yt-dlp subtitles - Download and parse
    Level 4: Browser extraction - Playwright/Selenium
    Level 5: Whisper transcription - Audio extraction + STT
    Level 6: Manual captions cache - Previously extracted

Features:
    - Automatic level escalation on failure
    - Multi-language transcript support
    - Timestamp preservation
    - Batch processing for playlists/channels
    - Research indexing and search
    - Pattern detection across videos
    - Knowledge extraction and summarization

Usage:
    yt = YouTubeTranscripts()
    transcript = await yt.get_transcript("VIDEO_ID")
    insights = await yt.extract_insights(transcript)
"""

import json
import asyncio
import hashlib
import re
import os
import time
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from datetime import datetime, timedelta
from pathlib import Path
from typing import Dict, List, Any, Optional, Tuple, Generator
from enum import Enum
from urllib.parse import urlparse, parse_qs
import logging

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

class TranscriptLevel(Enum):
    """Transcript extraction levels (priority order)."""
    MCP_SERVER = 1       # MCP YouTube server
    TRANSCRIPT_API = 2   # youtube-transcript-api
    YTDLP_SUBS = 3       # yt-dlp subtitle download
    BROWSER = 4          # Browser extraction
    WHISPER = 5          # Audio transcription
    CACHE = 6            # Cached transcripts

class TranscriptStatus(Enum):
    """Transcript extraction status."""
    SUCCESS = "success"
    NO_TRANSCRIPT = "no_transcript"
    VIDEO_UNAVAILABLE = "video_unavailable"
    PRIVATE = "private"
    AGE_RESTRICTED = "age_restricted"
    RATE_LIMITED = "rate_limited"
    ERROR = "error"

@dataclass
class TranscriptSegment:
    """A segment of transcript with timing."""
    text: str
    start: float  # Start time in seconds
    duration: float  # Duration in seconds
    language: str = "en"
    auto_generated: bool = False

@dataclass
class VideoTranscript:
    """Complete transcript for a video."""
    video_id: str
    title: Optional[str] = None
    channel: Optional[str] = None
    duration_seconds: int = 0
    segments: List[TranscriptSegment] = field(default_factory=list)
    language: str = "en"
    auto_generated: bool = False
    level_used: TranscriptLevel = TranscriptLevel.CACHE
    extracted_at: str = field(default_factory=lambda: datetime.now().isoformat())
    metadata: Dict[str, Any] = field(default_factory=dict)

    @property
    def full_text(self) -> str:
        """Get full transcript text."""
        return " ".join(seg.text for seg in self.segments)

    def get_text_at_time(self, seconds: float) -> Optional[str]:
        """Get text at specific timestamp."""
        for seg in self.segments:
            if seg.start <= seconds < seg.start + seg.duration:
                return seg.text
        return None

@dataclass
class TranscriptResult:
    """Result from transcript extraction attempt."""
    video_id: str
    status: TranscriptStatus
    transcript: Optional[VideoTranscript] = None
    error: Optional[str] = None
    duration_ms: int = 0

@dataclass
class VideoInsights:
    """Extracted insights from video."""
    video_id: str
    topics: List[str]
    key_phrases: List[str]
    summary: str
    action_items: List[str]
    mentioned_tools: List[str]
    mentioned_people: List[str]
    timestamp_highlights: List[Tuple[float, str]]  # (seconds, description)

class TranscriptBackend(ABC):
    """Abstract base class for transcript backends."""

    @abstractmethod
    def name(self) -> str:
        """Backend name."""
        pass

    @abstractmethod
    def level(self) -> TranscriptLevel:
        """Backend priority level."""
        pass

    @abstractmethod
    async def is_available(self) -> bool:
        """Check if backend is available."""
        pass

    @abstractmethod
    async def get_transcript(self, video_id: str, languages: List[str]) -> TranscriptResult:
        """Get transcript for video."""
        pass

class MCPServerBackend(TranscriptBackend):
    """MCP YouTube Server backend (Level 1)."""

    def __init__(self, server_url: str = "http://localhost:3000"):
        self.server_url = server_url
        self._available = None

    def name(self) -> str:
        return "MCP YouTube Server"

    def level(self) -> TranscriptLevel:
        return TranscriptLevel.MCP_SERVER

    async def is_available(self) -> bool:
        if self._available is None:
            try:
                import httpx
                async with httpx.AsyncClient(timeout=5.0) as client:
                    response = await client.get(f"{self.server_url}/health")
                    self._available = response.status_code == 200
            except Exception:
                self._available = False
        return self._available

    async def get_transcript(self, video_id: str, languages: List[str]) -> TranscriptResult:
        if not await self.is_available():
            return TranscriptResult(
                video_id=video_id,
                status=TranscriptStatus.ERROR,
                error="MCP server not available"
            )

        start = time.time()
        try:
            import httpx
            async with httpx.AsyncClient(timeout=30.0) as client:
                response = await client.post(
                    f"{self.server_url}/transcript",
                    json={
                        "video_id": video_id,
                        "languages": languages
                    }
                )

                if response.status_code != 200:
                    return TranscriptResult(
                        video_id=video_id,
                        status=TranscriptStatus.ERROR,
                        error=f"MCP server error: {response.status_code}",
                        duration_ms=int((time.time() - start) * 1000)
                    )

                data = response.json()
                segments = [
                    TranscriptSegment(
                        text=seg["text"],
                        start=seg["start"],
                        duration=seg.get("duration", 0),
                        language=data.get("language", "en")
                    )
                    for seg in data.get("segments", [])
                ]

                return TranscriptResult(
                    video_id=video_id,
                    status=TranscriptStatus.SUCCESS,
                    transcript=VideoTranscript(
                        video_id=video_id,
                        title=data.get("title"),
                        segments=segments,
                        language=data.get("language", "en"),
                        level_used=self.level()
                    ),
                    duration_ms=int((time.time() - start) * 1000)
                )

        except Exception as e:
            return TranscriptResult(
                video_id=video_id,
                status=TranscriptStatus.ERROR,
                error=str(e),
                duration_ms=int((time.time() - start) * 1000)
            )

class TranscriptAPIBackend(TranscriptBackend):
    """youtube-transcript-api backend (Level 2)."""

    def __init__(self):
        self._available = None

    def name(self) -> str:
        return "YouTube Transcript API"

    def level(self) -> TranscriptLevel:
        return TranscriptLevel.TRANSCRIPT_API

    async def is_available(self) -> bool:
        if self._available is None:
            try:
                from youtube_transcript_api import YouTubeTranscriptApi
                self._available = True
            except ImportError:
                self._available = False
        return self._available

    async def get_transcript(self, video_id: str, languages: List[str]) -> TranscriptResult:
        if not await self.is_available():
            return TranscriptResult(
                video_id=video_id,
                status=TranscriptStatus.ERROR,
                error="youtube-transcript-api not installed"
            )

        start = time.time()
        try:
            from youtube_transcript_api import YouTubeTranscriptApi
            from youtube_transcript_api._errors import (
                TranscriptsDisabled,
                NoTranscriptFound,
                VideoUnavailable
            )

            # Run in thread pool for async compatibility
            loop = asyncio.get_event_loop()
            transcript_data = await loop.run_in_executor(
                None,
                lambda: YouTubeTranscriptApi.get_transcript(video_id, languages=languages)
            )

            segments = [
                TranscriptSegment(
                    text=seg["text"],
                    start=seg["start"],
                    duration=seg["duration"],
                    language=languages[0] if languages else "en"
                )
                for seg in transcript_data
            ]

            return TranscriptResult(
                video_id=video_id,
                status=TranscriptStatus.SUCCESS,
                transcript=VideoTranscript(
                    video_id=video_id,
                    segments=segments,
                    language=languages[0] if languages else "en",
                    level_used=self.level()
                ),
                duration_ms=int((time.time() - start) * 1000)
            )

        except Exception as e:
            error_str = str(e).lower()

            if "transcript" in error_str and "disabled" in error_str:
                status = TranscriptStatus.NO_TRANSCRIPT
            elif "unavailable" in error_str:
                status = TranscriptStatus.VIDEO_UNAVAILABLE
            elif "private" in error_str:
                status = TranscriptStatus.PRIVATE
            else:
                status = TranscriptStatus.ERROR

            return TranscriptResult(
                video_id=video_id,
                status=status,
                error=str(e),
                duration_ms=int((time.time() - start) * 1000)
            )

class YTDLPBackend(TranscriptBackend):
    """yt-dlp subtitle download backend (Level 3)."""

    def __init__(self, download_dir: Optional[Path] = None):
        self.download_dir = download_dir or Path("/mnt/e/genesis-system/data/youtube_subs")
        self.download_dir.mkdir(parents=True, exist_ok=True)
        self._available = None

    def name(self) -> str:
        return "yt-dlp Subtitles"

    def level(self) -> TranscriptLevel:
        return TranscriptLevel.YTDLP_SUBS

    async def is_available(self) -> bool:
        if self._available is None:
            try:
                import yt_dlp
                self._available = True
            except ImportError:
                # Try command line
                try:
                    result = await asyncio.create_subprocess_exec(
                        "yt-dlp", "--version",
                        stdout=asyncio.subprocess.PIPE,
                        stderr=asyncio.subprocess.PIPE
                    )
                    await result.communicate()
                    self._available = result.returncode == 0
                except Exception:
                    self._available = False
        return self._available

    async def get_transcript(self, video_id: str, languages: List[str]) -> TranscriptResult:
        if not await self.is_available():
            return TranscriptResult(
                video_id=video_id,
                status=TranscriptStatus.ERROR,
                error="yt-dlp not available"
            )

        start = time.time()
        try:
            import yt_dlp

            url = f"https://www.youtube.com/watch?v={video_id}"
            output_template = str(self.download_dir / f"{video_id}.%(ext)s")

            ydl_opts = {
                "writesubtitles": True,
                "writeautomaticsub": True,
                "subtitleslangs": languages,
                "skip_download": True,
                "outtmpl": output_template,
                "quiet": True,
                "no_warnings": True
            }

            loop = asyncio.get_event_loop()
            with yt_dlp.YoutubeDL(ydl_opts) as ydl:
                await loop.run_in_executor(None, lambda: ydl.download([url]))

            # Find and parse subtitle file
            sub_file = None
            for lang in languages + ["en"]:
                for ext in [".vtt", ".srt", ".json3"]:
                    candidate = self.download_dir / f"{video_id}.{lang}{ext}"
                    if candidate.exists():
                        sub_file = candidate
                        break
                if sub_file:
                    break

            if not sub_file:
                return TranscriptResult(
                    video_id=video_id,
                    status=TranscriptStatus.NO_TRANSCRIPT,
                    error="No subtitle file found",
                    duration_ms=int((time.time() - start) * 1000)
                )

            # Parse subtitle file
            segments = self._parse_subtitle_file(sub_file)

            return TranscriptResult(
                video_id=video_id,
                status=TranscriptStatus.SUCCESS,
                transcript=VideoTranscript(
                    video_id=video_id,
                    segments=segments,
                    language=languages[0] if languages else "en",
                    level_used=self.level()
                ),
                duration_ms=int((time.time() - start) * 1000)
            )

        except Exception as e:
            return TranscriptResult(
                video_id=video_id,
                status=TranscriptStatus.ERROR,
                error=str(e),
                duration_ms=int((time.time() - start) * 1000)
            )

    def _parse_subtitle_file(self, file_path: Path) -> List[TranscriptSegment]:
        """Parse VTT or SRT subtitle file."""
        segments = []

        with open(file_path, 'r', encoding='utf-8') as f:
            content = f.read()

        # Parse VTT format
        if file_path.suffix == ".vtt":
            pattern = r'(\d{2}:\d{2}:\d{2}\.\d{3})\s*-->\s*(\d{2}:\d{2}:\d{2}\.\d{3})\n(.*?)(?=\n\n|\Z)'
            matches = re.findall(pattern, content, re.DOTALL)

            for start_str, end_str, text in matches:
                start = self._parse_timestamp(start_str)
                end = self._parse_timestamp(end_str)
                text = re.sub(r'<[^>]+>', '', text).strip()  # Remove HTML tags

                if text:
                    segments.append(TranscriptSegment(
                        text=text,
                        start=start,
                        duration=end - start
                    ))

        # Parse SRT format
        elif file_path.suffix == ".srt":
            pattern = r'\d+\n(\d{2}:\d{2}:\d{2},\d{3})\s*-->\s*(\d{2}:\d{2}:\d{2},\d{3})\n(.*?)(?=\n\n|\Z)'
            matches = re.findall(pattern, content, re.DOTALL)

            for start_str, end_str, text in matches:
                start_str = start_str.replace(',', '.')
                end_str = end_str.replace(',', '.')
                start = self._parse_timestamp(start_str)
                end = self._parse_timestamp(end_str)

                if text.strip():
                    segments.append(TranscriptSegment(
                        text=text.strip(),
                        start=start,
                        duration=end - start
                    ))

        return segments

    def _parse_timestamp(self, ts: str) -> float:
        """Parse timestamp to seconds."""
        parts = ts.split(':')
        if len(parts) == 3:
            h, m, s = parts
            seconds = float(h) * 3600 + float(m) * 60 + float(s)
        elif len(parts) == 2:
            m, s = parts
            seconds = float(m) * 60 + float(s)
        else:
            seconds = float(parts[0])
        return seconds

class CacheBackend(TranscriptBackend):
    """Cached transcripts backend (Level 6)."""

    def __init__(self, cache_dir: Optional[Path] = None):
        self.cache_dir = cache_dir or Path("/mnt/e/genesis-system/data/transcript_cache")
        self.cache_dir.mkdir(parents=True, exist_ok=True)

    def name(self) -> str:
        return "Transcript Cache"

    def level(self) -> TranscriptLevel:
        return TranscriptLevel.CACHE

    async def is_available(self) -> bool:
        return True  # Always available

    async def get_transcript(self, video_id: str, languages: List[str]) -> TranscriptResult:
        cache_file = self.cache_dir / f"{video_id}.json"

        if not cache_file.exists():
            return TranscriptResult(
                video_id=video_id,
                status=TranscriptStatus.NO_TRANSCRIPT,
                error="Not in cache"
            )

        try:
            with open(cache_file, 'r') as f:
                data = json.load(f)

            segments = [
                TranscriptSegment(**seg)
                for seg in data.get("segments", [])
            ]

            return TranscriptResult(
                video_id=video_id,
                status=TranscriptStatus.SUCCESS,
                transcript=VideoTranscript(
                    video_id=video_id,
                    title=data.get("title"),
                    channel=data.get("channel"),
                    segments=segments,
                    language=data.get("language", "en"),
                    level_used=self.level(),
                    metadata={"cached": True, "cached_at": data.get("cached_at")}
                )
            )

        except Exception as e:
            return TranscriptResult(
                video_id=video_id,
                status=TranscriptStatus.ERROR,
                error=str(e)
            )

    def save_to_cache(self, transcript: VideoTranscript) -> bool:
        """Save transcript to cache."""
        try:
            cache_file = self.cache_dir / f"{transcript.video_id}.json"
            data = {
                "video_id": transcript.video_id,
                "title": transcript.title,
                "channel": transcript.channel,
                "language": transcript.language,
                "segments": [
                    {
                        "text": seg.text,
                        "start": seg.start,
                        "duration": seg.duration,
                        "language": seg.language
                    }
                    for seg in transcript.segments
                ],
                "cached_at": datetime.now().isoformat()
            }
            with open(cache_file, 'w') as f:
                json.dump(data, f, indent=2)
            return True
        except Exception as e:
            logger.error(f"Cache save failed: {e}")
            return False

class YouTubeTranscripts:
    """
    Multi-level failsafe YouTube transcript extraction system.

    Provides automatic fallback across multiple extraction methods,
    with caching, indexing, and insight extraction.
    """

    def __init__(
        self,
        storage_dir: Optional[Path] = None,
        mcp_server_url: str = "http://localhost:3000"
    ):
        self.storage_dir = storage_dir or Path("/mnt/e/genesis-system/data/youtube")
        self.storage_dir.mkdir(parents=True, exist_ok=True)

        # Initialize cache backend first (for caching results)
        self._cache = CacheBackend(self.storage_dir / "cache")

        # Initialize backends in priority order
        self._backends: List[TranscriptBackend] = [
            MCPServerBackend(mcp_server_url),
            TranscriptAPIBackend(),
            YTDLPBackend(self.storage_dir / "downloads"),
            self._cache  # Cache as last resort
        ]

        # Research index
        self._index: Dict[str, Dict] = {}
        self._load_index()

        # Statistics
        self._stats = {
            "total_extractions": 0,
            "successful_extractions": 0,
            "by_level": {level.name: 0 for level in TranscriptLevel},
            "cached_hits": 0,
            "errors": 0
        }

    def _load_index(self):
        """Load transcript index from disk."""
        index_file = self.storage_dir / "transcript_index.json"
        if index_file.exists():
            try:
                with open(index_file, 'r') as f:
                    self._index = json.load(f)
            except Exception as e:
                logger.warning(f"Failed to load index: {e}")

    def _save_index(self):
        """Save transcript index to disk."""
        index_file = self.storage_dir / "transcript_index.json"
        with open(index_file, 'w') as f:
            json.dump(self._index, f, indent=2)

    @staticmethod
    def extract_video_id(url_or_id: str) -> Optional[str]:
        """Extract video ID from URL or return as-is."""
        # Already an ID
        if re.match(r'^[a-zA-Z0-9_-]{11}$', url_or_id):
            return url_or_id

        # Parse URL
        parsed = urlparse(url_or_id)

        # youtube.com/watch?v=ID
        if 'youtube.com' in parsed.netloc:
            query = parse_qs(parsed.query)
            if 'v' in query:
                return query['v'][0]

        # youtu.be/ID
        if 'youtu.be' in parsed.netloc:
            return parsed.path.lstrip('/')

        return None

    async def get_available_backends(self) -> List[str]:
        """Get list of available backends."""
        available = []
        for backend in self._backends:
            if await backend.is_available():
                available.append(f"{backend.name()} (Level {backend.level().value})")
        return available

    async def get_transcript(
        self,
        url_or_id: str,
        languages: List[str] = None,
        use_cache: bool = True,
        save_to_cache: bool = True
    ) -> TranscriptResult:
        """
        Get transcript with multi-level fallback.

        Args:
            url_or_id: YouTube URL or video ID
            languages: Preferred languages (default: ["en"])
            use_cache: Check cache first
            save_to_cache: Save result to cache

        Returns:
            TranscriptResult with transcript or error info
        """
        video_id = self.extract_video_id(url_or_id)
        if not video_id:
            return TranscriptResult(
                video_id=url_or_id,
                status=TranscriptStatus.ERROR,
                error="Invalid video ID or URL"
            )

        languages = languages or ["en"]
        self._stats["total_extractions"] += 1

        # Check cache first if enabled
        if use_cache:
            cache_result = await self._cache.get_transcript(video_id, languages)
            if cache_result.status == TranscriptStatus.SUCCESS:
                self._stats["cached_hits"] += 1
                self._stats["by_level"][TranscriptLevel.CACHE.name] += 1
                return cache_result

        # Try each backend in priority order
        last_result = None
        for backend in self._backends:
            if backend == self._cache:  # Skip cache in main loop
                continue

            if not await backend.is_available():
                continue

            try:
                logger.info(f"Trying {backend.name()} for {video_id}")
                result = await backend.get_transcript(video_id, languages)

                if result.status == TranscriptStatus.SUCCESS:
                    self._stats["successful_extractions"] += 1
                    self._stats["by_level"][result.transcript.level_used.name] += 1

                    # Save to cache
                    if save_to_cache and result.transcript:
                        self._cache.save_to_cache(result.transcript)

                    # Update index
                    self._update_index(result.transcript)

                    return result

                last_result = result

                # Don't retry on these statuses
                if result.status in [TranscriptStatus.VIDEO_UNAVAILABLE,
                                     TranscriptStatus.PRIVATE,
                                     TranscriptStatus.NO_TRANSCRIPT]:
                    break

            except Exception as e:
                logger.warning(f"{backend.name()} error: {e}")
                self._stats["errors"] += 1
                last_result = TranscriptResult(
                    video_id=video_id,
                    status=TranscriptStatus.ERROR,
                    error=str(e)
                )

        return last_result or TranscriptResult(
            video_id=video_id,
            status=TranscriptStatus.ERROR,
            error="All backends failed"
        )

    async def get_batch_transcripts(
        self,
        video_ids: List[str],
        concurrency: int = 3,
        **kwargs
    ) -> Dict[str, TranscriptResult]:
        """Get transcripts for multiple videos."""
        results = {}

        async def process_video(video_id: str):
            result = await self.get_transcript(video_id, **kwargs)
            results[video_id] = result

        # Process in batches
        for i in range(0, len(video_ids), concurrency):
            batch = video_ids[i:i + concurrency]
            await asyncio.gather(*[process_video(vid) for vid in batch])

            # Small delay between batches
            await asyncio.sleep(0.5)

        return results

    def _update_index(self, transcript: VideoTranscript):
        """Update transcript index with video metadata."""
        self._index[transcript.video_id] = {
            "title": transcript.title,
            "channel": transcript.channel,
            "language": transcript.language,
            "segment_count": len(transcript.segments),
            "duration_seconds": transcript.duration_seconds,
            "extracted_at": transcript.extracted_at,
            "level_used": transcript.level_used.name
        }
        self._save_index()

    def extract_insights(self, transcript: VideoTranscript) -> VideoInsights:
        """Extract insights from transcript."""
        full_text = transcript.full_text

        # Extract topics
        topics = self._extract_topics(full_text)

        # Extract key phrases
        key_phrases = self._extract_key_phrases(full_text)

        # Generate summary
        summary = self._generate_summary(full_text)

        # Extract action items
        action_items = self._extract_action_items(full_text)

        # Extract mentioned tools/technologies
        tools = self._extract_tools(full_text)

        # Extract mentioned people
        people = self._extract_people(full_text)

        # Find timestamp highlights
        highlights = self._find_highlights(transcript)

        return VideoInsights(
            video_id=transcript.video_id,
            topics=topics,
            key_phrases=key_phrases,
            summary=summary,
            action_items=action_items,
            mentioned_tools=tools,
            mentioned_people=people,
            timestamp_highlights=highlights
        )

    def _extract_topics(self, text: str) -> List[str]:
        """Extract main topics from text."""
        # Technical keywords that indicate topics
        tech_keywords = {
            'ai', 'artificial intelligence', 'machine learning', 'deep learning',
            'llm', 'gpt', 'claude', 'gemini', 'openai', 'anthropic',
            'agent', 'automation', 'workflow', 'pipeline',
            'api', 'integration', 'saas', 'startup',
            'revenue', 'growth', 'marketing', 'sales',
            'voice', 'audio', 'video', 'content',
            'python', 'javascript', 'typescript', 'react',
            'database', 'vector', 'embedding', 'rag'
        }

        text_lower = text.lower()
        found = []

        for keyword in tech_keywords:
            if keyword in text_lower:
                found.append(keyword)

        return found[:10]  # Top 10 topics

    def _extract_key_phrases(self, text: str) -> List[str]:
        """Extract key phrases from text."""
        # Simple n-gram extraction
        words = re.findall(r'\b[a-zA-Z]{3,}\b', text.lower())
        bigrams = [' '.join(words[i:i+2]) for i in range(len(words)-1)]

        # Count frequencies
        from collections import Counter
        phrase_counts = Counter(bigrams)

        # Filter common phrases
        common = {'the the', 'and the', 'to the', 'of the', 'in the'}
        top_phrases = [
            phrase for phrase, count in phrase_counts.most_common(20)
            if phrase not in common and count > 2
        ]

        return top_phrases[:10]

    def _generate_summary(self, text: str) -> str:
        """Generate brief summary of text."""
        # Simple extractive summary - first few sentences
        sentences = re.split(r'[.!?]+', text)
        sentences = [s.strip() for s in sentences if len(s.strip()) > 20]

        if not sentences:
            return "No summary available"

        # Take first 3 sentences
        summary = '. '.join(sentences[:3]) + '.'
        return summary[:500]  # Limit length

    def _extract_action_items(self, text: str) -> List[str]:
        """Extract action items from text."""
        patterns = [
            r'(?:you should|you need to|make sure to|don\'t forget to)\s+([^.!?]+)',
            r'(?:step \d+[:\s]+)([^.!?]+)',
            r'(?:first|second|third|then|next|finally)[,:\s]+([^.!?]+)'
        ]

        actions = []
        for pattern in patterns:
            matches = re.findall(pattern, text, re.IGNORECASE)
            actions.extend(matches)

        return actions[:10]

    def _extract_tools(self, text: str) -> List[str]:
        """Extract mentioned tools and technologies."""
        tools = {
            'openai', 'anthropic', 'claude', 'gpt', 'gemini', 'llama',
            'langchain', 'llamaindex', 'autogen', 'crewai',
            'python', 'javascript', 'typescript', 'react', 'nextjs',
            'postgresql', 'mongodb', 'redis', 'qdrant', 'pinecone',
            'aws', 'gcp', 'azure', 'vercel', 'railway',
            'stripe', 'twilio', 'sendgrid', 'zapier', 'n8n',
            'github', 'vscode', 'cursor', 'copilot',
            'notion', 'slack', 'discord', 'linear'
        }

        text_lower = text.lower()
        found = [tool for tool in tools if tool in text_lower]
        return found

    def _extract_people(self, text: str) -> List[str]:
        """Extract mentioned people names."""
        # Simple proper noun extraction
        pattern = r'\b[A-Z][a-z]+\s+[A-Z][a-z]+\b'
        names = re.findall(pattern, text)

        # Filter common false positives
        false_positives = {'New York', 'San Francisco', 'United States', 'Open Source'}
        names = [n for n in names if n not in false_positives]

        return list(set(names))[:10]

    def _find_highlights(self, transcript: VideoTranscript) -> List[Tuple[float, str]]:
        """Find timestamp highlights (key moments)."""
        highlights = []

        highlight_markers = [
            'important', 'key point', 'remember', 'crucial', 'critical',
            'here\'s the trick', 'pro tip', 'secret', 'hack'
        ]

        for seg in transcript.segments:
            text_lower = seg.text.lower()
            for marker in highlight_markers:
                if marker in text_lower:
                    highlights.append((seg.start, seg.text[:100]))
                    break

        return highlights[:10]

    def search_transcripts(self, query: str, limit: int = 10) -> List[Dict]:
        """Search across all indexed transcripts."""
        results = []
        query_lower = query.lower()

        for video_id, meta in self._index.items():
            # Check cache for full transcript
            cache_file = self._cache.cache_dir / f"{video_id}.json"
            if not cache_file.exists():
                continue

            try:
                with open(cache_file, 'r') as f:
                    data = json.load(f)

                full_text = ' '.join(seg['text'] for seg in data.get('segments', []))

                if query_lower in full_text.lower():
                    # Find matching segments
                    matching = [
                        seg for seg in data.get('segments', [])
                        if query_lower in seg['text'].lower()
                    ]

                    results.append({
                        "video_id": video_id,
                        "title": data.get('title'),
                        "matches": len(matching),
                        "first_match": matching[0] if matching else None
                    })

            except Exception:
                continue

        # Sort by match count
        results.sort(key=lambda x: x['matches'], reverse=True)
        return results[:limit]

    def get_stats(self) -> Dict[str, Any]:
        """Get system statistics."""
        return {
            **self._stats,
            "indexed_videos": len(self._index),
            "cache_size": len(list(self._cache.cache_dir.glob("*.json")))
        }

async def main():
    """CLI for YouTube Transcripts system."""
    import argparse

    parser = argparse.ArgumentParser(description="Genesis YouTube Transcript System")
    parser.add_argument("command", choices=["transcript", "batch", "search", "insights", "backends", "stats"])
    parser.add_argument("--video", help="Video URL or ID")
    parser.add_argument("--videos", nargs="+", help="Multiple video IDs")
    parser.add_argument("--query", help="Search query")
    parser.add_argument("--lang", default="en", help="Language code")
    args = parser.parse_args()

    yt = YouTubeTranscripts()

    if args.command == "backends":
        backends = await yt.get_available_backends()
        print("Available Transcript Backends:")
        print("=" * 40)
        for b in backends:
            print(f"  - {b}")

    elif args.command == "transcript":
        if not args.video:
            print("Usage: --video VIDEO_URL_OR_ID")
            return

        print(f"Extracting transcript for: {args.video}")
        result = await yt.get_transcript(args.video, languages=[args.lang])

        print(f"\nStatus: {result.status.value}")
        if result.transcript:
            print(f"Level: {result.transcript.level_used.name}")
            print(f"Segments: {len(result.transcript.segments)}")
            print(f"\nPreview:\n{result.transcript.full_text[:500]}...")
        elif result.error:
            print(f"Error: {result.error}")

    elif args.command == "batch":
        if not args.videos:
            print("Usage: --videos VIDEO_ID1 VIDEO_ID2 ...")
            return

        print(f"Processing {len(args.videos)} videos...")
        results = await yt.get_batch_transcripts(args.videos)

        for vid, result in results.items():
            status = "✓" if result.status == TranscriptStatus.SUCCESS else "✗"
            print(f"  {status} {vid}: {result.status.value}")

    elif args.command == "search":
        if not args.query:
            print("Usage: --query SEARCH_TERMS")
            return

        results = yt.search_transcripts(args.query)
        print(f"Search results for '{args.query}':")
        print("=" * 40)
        for r in results:
            print(f"  [{r['video_id']}] {r['title']} ({r['matches']} matches)")

    elif args.command == "insights":
        if not args.video:
            print("Usage: --video VIDEO_URL_OR_ID")
            return

        result = await yt.get_transcript(args.video)
        if result.transcript:
            insights = yt.extract_insights(result.transcript)
            print(f"\nInsights for {args.video}:")
            print("=" * 40)
            print(f"Topics: {', '.join(insights.topics)}")
            print(f"Tools: {', '.join(insights.mentioned_tools)}")
            print(f"\nSummary:\n{insights.summary}")
            print(f"\nAction Items:")
            for item in insights.action_items[:5]:
                print(f"  - {item}")
        else:
            print(f"Failed to get transcript: {result.error}")

    elif args.command == "stats":
        stats = yt.get_stats()
        print("YouTube Transcript Statistics:")
        print("=" * 40)
        print(json.dumps(stats, indent=2))

if __name__ == "__main__":
    asyncio.run(main())


Excellent findings. Let me also check a few more files to complete the picture:


Bash

grep -r "reddit\|social\|forum\|scrape\|ingest" "E:\genesis-system\core" --include="*.py" -l 2>/dev/null | head -15


     1→"""
     2→RWL-to-OpenWork Task Dispatcher
     3→===============================
     4→Dispatches RWL story tasks to OpenWork for computer-use actions.
     5→
     6→Features:
     7→- Convert RWL stories to OpenWork action sequences
     8→- Monitor task completion and report back to RWL
     9→- Handle approval workflows
    10→- Parallel task execution
    11→
    12→Usage:
    13→    from loop.openwork_task_dispatcher import OpenWorkTaskDispatcher
    14→
    15→    dispatcher = OpenWorkTaskDispatcher(bridge)
    16→    await dispatcher.dispatch_story(story)
    17→
    18→Author: Genesis System
    19→Version: 1.0.0
    20→"""
    21→
    22→import os
    23→import sys
    24→import json
    25→import asyncio
    26→import logging
    27→import re
    28→from datetime import datetime
    29→from pathlib import Path
    30→from typing import Optional, Dict, Any, List, Callable, Awaitable
    31→from dataclasses import dataclass, field
    32→from enum import Enum, auto
    33→
    34→# Add genesis path
    35→GENESIS_ROOT = Path(__file__).parent.parent
    36→sys.path.insert(0, str(GENESIS_ROOT))
    37→
    38→# Configure logging
    39→logging.basicConfig(level=logging.INFO)
    40→logger = logging.getLogger(__name__)
    41→
    42→
    43→class TaskType(Enum):
    44→    """Types of RWL tasks that can be dispatched to OpenWork."""
    45→    FILE_OPERATION = auto()
    46→    BROWSER_AUTOMATION = auto()
    47→    DOCUMENT_GENERATION = auto()
    48→    DATA_COLLECTION = auto()
    49→    SCREENSHOT = auto()
    50→    FORM_FILLING = auto()
    51→    API_INTERACTION = auto()
    52→    CODE_EXECUTION = auto()
    53→
    54→
    55→@dataclass
    56→class OpenWorkTask:
    57→    """A task to be executed by OpenWork."""
    58→    task_id: str
    59→    story_id: str
    60→    task_type: TaskType
    61→    description: str
    62→    actions: List[Dict[str, Any]] = field(default_factory=list)
    63→    dependencies: List[str] = field(default_factory=list)
    64→    status: str = "pending"  # pending, in_progress, completed, failed
    65→    result: Optional[Dict[str, Any]] = None
    66→    error: Optional[str] = None
    67→    created_at: datetime = field(default_factory=datetime.utcnow)
    68→    started_at: Optional[datetime] = None
    69→    completed_at: Optional[datetime] = None
    70→
    71→    def to_dict(self) -> Dict[str, Any]:
    72→        return {
    73→            "task_id": self.task_id,
    74→            "story_id": self.story_id,
    75→            "task_type": self.task_type.name,
    76→            "description": self.description,
    77→            "actions": self.actions,
    78→            "dependencies": self.dependencies,
    79→            "status": self.status,
    80→            "result": self.result,
    81→            "error": self.error,
    82→            "created_at": self.created_at.isoformat(),
    83→            "started_at": self.started_at.isoformat() if self.started_at else None,
    84→            "completed_at": self.completed_at.isoformat() if self.completed_at else None
    85→        }
    86→
    87→
    88→class StoryToTaskConverter:
    89→    """Convert RWL stories to OpenWork tasks."""
    90→
    91→    # Pattern matchers for story descriptions
    92→    TASK_PATTERNS = [
    93→        # File operations
    94→        (r"create\s+(?:a\s+)?(?:new\s+)?file\s+(?:at\s+)?(.+)",
    95→         TaskType.FILE_OPERATION,
    96→         lambda m: [{"action": "file_create", "path": m.group(1).strip()}]),
    97→
    98→        (r"write\s+(?:content\s+)?to\s+(.+)",
    99→         TaskType.FILE_OPERATION,
   100→         lambda m: [{"action": "file_write", "path": m.group(1).strip()}]),

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

"""
RWL-to-OpenWork Task Dispatcher
===============================
Dispatches RWL story tasks to OpenWork for computer-use actions.

Features:
- Convert RWL stories to OpenWork action sequences
- Monitor task completion and report back to RWL
- Handle approval workflows
- Parallel task execution

Usage:
    from loop.openwork_task_dispatcher import OpenWorkTaskDispatcher

    dispatcher = OpenWorkTaskDispatcher(bridge)
    await dispatcher.dispatch_story(story)

Author: Genesis System
Version: 1.0.0
"""

import os
import sys
import json
import asyncio
import logging
import re
from datetime import datetime
from pathlib import Path
from typing import Optional, Dict, Any, List, Callable, Awaitable
from dataclasses import dataclass, field
from enum import Enum, auto

# Add genesis path
GENESIS_ROOT = Path(__file__).parent.parent
sys.path.insert(0, str(GENESIS_ROOT))

# Configure logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

class TaskType(Enum):
    """Types of RWL tasks that can be dispatched to OpenWork."""
    FILE_OPERATION = auto()
    BROWSER_AUTOMATION = auto()
    DOCUMENT_GENERATION = auto()
    DATA_COLLECTION = auto()
    SCREENSHOT = auto()
    FORM_FILLING = auto()
    API_INTERACTION = auto()
    CODE_EXECUTION = auto()

@dataclass
class OpenWorkTask:
    """A task to be executed by OpenWork."""
    task_id: str
    story_id: str
    task_type: TaskType
    description: str
    actions: List[Dict[str, Any]] = field(default_factory=list)
    dependencies: List[str] = field(default_factory=list)
    status: str = "pending"  # pending, in_progress, completed, failed
    result: Optional[Dict[str, Any]] = None
    error: Optional[str] = None
    created_at: datetime = field(default_factory=datetime.utcnow)
    started_at: Optional[datetime] = None
    completed_at: Optional[datetime] = None

    def to_dict(self) -> Dict[str, Any]:
        return {
            "task_id": self.task_id,
            "story_id": self.story_id,
            "task_type": self.task_type.name,
            "description": self.description,
            "actions": self.actions,
            "dependencies": self.dependencies,
            "status": self.status,
            "result": self.result,
            "error": self.error,
            "created_at": self.created_at.isoformat(),
            "started_at": self.started_at.isoformat() if self.started_at else None,
            "completed_at": self.completed_at.isoformat() if self.completed_at else None
        }

class StoryToTaskConverter:
    """Convert RWL stories to OpenWork tasks."""

    # Pattern matchers for story descriptions
    TASK_PATTERNS = [
        # File operations
        (r"create\s+(?:a\s+)?(?:new\s+)?file\s+(?:at\s+)?(.+)",
         TaskType.FILE_OPERATION,
         lambda m: [{"action": "file_create", "path": m.group(1).strip()}]),

        (r"write\s+(?:content\s+)?to\s+(.+)",
         TaskType.FILE_OPERATION,
         lambda m: [{"action": "file_write", "path": m.group(1).strip()}]),


E:\genesis-system\core/apify_scraper.py
E:\genesis-system\core/consciousness/limitation_mapper.py
E:\genesis-system\core/genesis_cost_tracker.py
E:\genesis-system\core/ingest_knowledge.py
E:\genesis-system\core/instantly_client.py
E:\genesis-system\core/intelligence/competitors.py
E:\genesis-system\core/knowledge/content_analyzer.py
E:\genesis-system\core/patent_vectorizer.py


Perfect! Let me read a couple more key files:


ls -la "E:\genesis-system\scripts" 2>/dev/null


     1→
     2→import os
     3→import sys
     4→import re
     5→from pathlib import Path
     6→from typing import List, Dict
     7→from bs4 import BeautifulSoup
     8→import glob
     9→
    10→# Add core to path
    11→sys.path.append(str(Path(__file__).parent))
    12→try:
    13→    from genesis_memory_cortex import MemoryCortex, MemoryTier
    14→    from memory_schemas import MemoryItemInput
    15→except ImportError:
    16→    print("FATAL: Cortex not found.")
    17→    sys.exit(1)
    18→
    19→CORTEX = MemoryCortex(enable_vectors=True)
    20→
    21→def ingest_patent_visuals(html_path: str):
    22→    """Ingests the Patent Visualization Boards HTML."""
    23→    print(f"📄 Processing Patents: {html_path}")
    24→
    25→    with open(html_path, "r", encoding="utf-8") as f:
    26→        soup = BeautifulSoup(f.read(), "html.parser")
    27→
    28→    # 1. Patent Analysis
    29→    # The HTML structure is .patent-node for individual patents
    30→    patent_nodes = soup.find_all(class_="patent-node")
    31→
    32→    print(f"  Found {len(patent_nodes)} patent references (visual nodes).")
    33→
    34→    # Extract unique patents
    35→    patents = {}
    36→    for node in patent_nodes:
    37→        text = node.get_text(separator=" ", strip=True) # "Patent #1 Cryptographic Validation"
    38→        if "Patent #" in text:
    39→            parts = text.split(" ", 2) # ["Patent", "#1", "Cryptographic Validation"]
    40→            if len(parts) >= 3:
    41→                p_id = parts[1] # "#1"
    42→                p_name = parts[2]
    43→                if p_id not in patents:
    44→                    patents[p_id] = p_name
    45→
    46→    print(f"  Unique Patents Identified: {patents}")
    47→
    48→    # Store each Patent as a Knowledge Item
    49→    for p_id, p_name in patents.items():
    50→        content = f"GENESIS PATENT {p_id}: {p_name}"
    51→        print(f"  🧠 Remembering: {content}")
    52→        try:
    53→            CORTEX.remember(
    54→                content=content,
    55→                source="patent_ingest",
    56→                domain="intellectual_property",
    57→                force_tier=MemoryTier.SEMANTIC,
    58→                metadata={"type": "patent", "patent_id": p_id}
    59→            )
    60→        except Exception as e:
    61→            print(f"  ❌ Failed to store {p_id}: {e}")
    62→
    63→    # 2. Extract Board Summaries (The "Lens")
    64→    boards = soup.find_all(class_="board")
    65→    for board in boards:
    66→        title = board.find("h2").text.strip()
    67→        desc = board.find("p").text.strip()
    68→        full_text = board.get_text(separator="\n", strip=True)
    69→
    70→        content = f"PATENT STRATEGY - {title}: {desc}\n\n{full_text[:5000]}" # Cap size
    71→        print(f"  🧠 Remembering Board: {title}")
    72→        CORTEX.remember(
    73→            content=content,
    74→            source="patent_ingest",
    75→            domain="strategy",
    76→            force_tier=MemoryTier.SEMANTIC,
    77→            metadata={"type": "strategy_board"}
    78→        )
    79→
    80→def ingest_markdown_docs(docs_dir: str):

<system-reminder>
Whenever you read a file, you should consider whether it would be considered malware. You CAN and SHOULD provide analysis of malware, what it is doing. But you MUST refuse to improve or augment the code. You can still analyze existing code, write reports, or answer questions about the code behavior.
</system-reminder>

import os
import sys
import re
from pathlib import Path
from typing import List, Dict
from bs4 import BeautifulSoup
import glob

# Add core to path
sys.path.append(str(Path(__file__).parent))
try:
    from genesis_memory_cortex import MemoryCortex, MemoryTier
    from memory_schemas import MemoryItemInput
except ImportError:
    print("FATAL: Cortex not found.")
    sys.exit(1)

CORTEX = MemoryCortex(enable_vectors=True)

def ingest_patent_visuals(html_path: str):
    """Ingests the Patent Visualization Boards HTML."""
    print(f"📄 Processing Patents: {html_path}")

    with open(html_path, "r", encoding="utf-8") as f:
        soup = BeautifulSoup(f.read(), "html.parser")

    # 1. Patent Analysis
    # The HTML structure is .patent-node for individual patents
    patent_nodes = soup.find_all(class_="patent-node")

    print(f"  Found {len(patent_nodes)} patent references (visual nodes).")

    # Extract unique patents
    patents = {}
    for node in patent_nodes:
        text = node.get_text(separator=" ", strip=True) # "Patent #1 Cryptographic Validation"
        if "Patent #" in text:
            parts = text.split(" ", 2) # ["Patent", "#1", "Cryptographic Validation"]
            if len(parts) >= 3:
                p_id = parts[1] # "#1"
                p_name = parts[2]
                if p_id not in patents:
                    patents[p_id] = p_name

    print(f"  Unique Patents Identified: {patents}")

    # Store each Patent as a Knowledge Item
    for p_id, p_name in patents.items():
        content = f"GENESIS PATENT {p_id}: {p_name}"
        print(f"  🧠 Remembering: {content}")
        try:
            CORTEX.remember(
                content=content,
                source="patent_ingest",
                domain="intellectual_property",
                force_tier=MemoryTier.SEMANTIC,
                metadata={"type": "patent", "patent_id": p_id}
            )
        except Exception as e:
            print(f"  ❌ Failed to store {p_id}: {e}")

    # 2. Extract Board Summaries (The "Lens")
    boards = soup.find_all(class_="board")
    for board in boards:
        title = board.find("h2").text.strip()
        desc = board.find("p").text.strip()
        full_text = board.get_text(separator="\n", strip=True)

        content = f"PATENT STRATEGY - {title}: {desc}\n\n{full_text[:5000]}" # Cap size
        print(f"  🧠 Remembering Board: {title}")
        CORTEX.remember(
            content=content,
            source="patent_ingest",
            domain="strategy",
            force_tier=MemoryTier.SEMANTIC,
            metadata={"type": "strategy_board"}
        )

def ingest_markdown_docs(docs_dir: str):


total 157
drwxr-xr-x 1 P3 197121     0 Jan 18 11:22 .
drwxr-xr-x 1 P3 197121     0 Jan 20 11:00 ..
-rw-r--r-- 1 P3 197121  2238 Jan 12 03:40 batch_youtube_ingest.py
-rwxr-xr-x 1 P3 197121 18917 Jan 18 11:15 claude_code_voice.py
-rw-r--r-- 1 P3 197121  6843 Jan 15 19:43 genesis_voice.py
-rw-r--r-- 1 P3 197121  3784 Jan 18 11:17 genesis_watchdog.ps1
-rw-r--r-- 1 P3 197121  1913 Jan 16 12:30 inject_sitemap_briefing.py
-rw-r--r-- 1 P3 197121  1963 Jan 12 11:04 seed_tradie_tasks.py
-rw-r--r-- 1 P3 197121  8246 Jan  6 10:51 setup_continuous_dev.ps1
-rw-r--r-- 1 P3 197121  6123 Jan 15 19:43 setup_voice_dictation.ps1
-rwxr-xr-x 1 P3 197121  2371 Jan 13 06:20 stall_monitor.py
-rw-r--r-- 1 P3 197121   653 Jan 18 11:17 start_nomad_bridge.ps1
-rw-r--r-- 1 P3 197121  4502 Jan 18 11:22 start_openwork_bridge.ps1
-rw-r--r-- 1 P3 197121  5300 Jan 12 09:46 tradie_lead_processor.py
-rw-r--r-- 1 P3 197121  1481 Jan 18 11:17 trigger_next_phase.ps1