
    9_ioT                     F   S r SSKrSSKrSSKrSSKrSSKrSSKJs  Jr	  SSK
J
r
JrJr  SSKJr  SSKr\" S5      r\S-  S-  q["        S-  q["        S	-  rS
SSSSS.SSSSSS.SSSSSS./rSrSrSrSrSrSr\R6                  " S5      rS7S\SS4S jjr\4S\S \ S\4S! jjr!SSK"r"S"\S\4S# jr#S\$4S$ jr%S%\&S&\$S\4S' jr'S8S(\&S)\S\(4S* jjr)S\(4S+ jr*S,r+S\(4S- jr,S9S%\&S.\S/\ SS4S0 jjr-S%\&S\&4S1 jr.    S:S2\S3\S)\S\S\&4
S4 jjr/S5 r0SSK1r1\2S6:X  a  \0" 5         gg);u+  
Memory Competitor Intelligence Scout — Standalone Cron Fallback
===============================================================
Runs the full GitHub + Hacker News + arXiv intelligence sweep without requiring
n8n. Designed as a fallback when n8n is unavailable, or as a one-shot
manual trigger.

Usage:
    python scout_cron.py --dry-run          # Print discoveries, write nothing
    python scout_cron.py --ingest           # Write to KG + tasks.json
    python scout_cron.py --ingest --verbose # Verbose output

The script outputs a structured summary at the end showing:
  - Total items fetched per source
  - Items passing age filter (< 7 days)
  - Items that were new (not already known)
  - Items written to KG / tasks created

Dependencies: requests (pip install requests), everything else is stdlib.
    N)datetime	timedeltatimezone)PathzE:/genesis-systemn8nmemory_competitor_scoutdiscovery_log.jsonlzknown_memory_systems.jsonlz"ai memory knowledge-graph episodicstarsdesc   github_stars)qsortorderper_pagelabelztemporal memory LLM RAGupdatedgithub_updatedz&agent memory persistent session python   github_agent_memoryz*https://api.github.com/search/repositoriesz`https://hn.algolia.com/api/v1/search?query=AI+memory+knowledge+graph&tags=show_hn&hitsPerPage=10zhttp://export.arxiv.org/api/query?search_query=all:episodic+memory+LLM&sortBy=lastUpdatedDate&sortOrder=descending&max_results=10   g      ?
scout_cronverbosereturnc                     U (       a  [         R                  O[         R                  n[         R                  " US[         R                  " [
        R                  5      /S9  g )Nz'%(asctime)s [%(levelname)s] %(message)s)levelformathandlers)loggingDEBUGINFObasicConfigStreamHandlersysstdout)r   r   s     ;E:/genesis-system/n8n/memory_competitor_scout/scout_cron.pysetup_loggingr'   V   s<    $GMM',,E8''

34    date_strmax_daysc                 ^   U (       d  g U R                  S5      (       a  U SS S-   n [        R                  " U 5      nUR                  c  UR	                  [
        R                  S9n[        R                  " [
        R                  5      U-
  [        US9:*  $ ! [        [        4 a     gf = f)z2Return True if date_str is within max_days of now.FZNz+00:00)tzinfo)days)endswithr   fromisoformatr.   replacer   utcnowr   
ValueError	TypeError)r)   r*   dts      r&   	is_recentr8   c   s    S!!}x/H##H-998<<0BX\\*R/I84LLL	" s   BB B,+B,urlc                     [         R                  " U R                  5       R                  5       R	                  5       5      R                  5       S S $ )N   )hashlibmd5lowerstripencode	hexdigest)r9   s    r&   	url_to_idrB   y   s9    ;;syy{((*1134>>@"EEr(   c                  d   [        5       n [        [        4 H  nUR                  5       (       d  M  UR	                  SSS9 nU H  nUR                  5       nU(       d  M   [        R                  " U5      nUR                  SS5      nU(       a  U R                  [        U5      5        UR                  SS5      R                  5       R                  5       nU(       a  U R                  U5        M  M     SSS5        M     U $ ! [        R                   a     M  f = f! , (       d  f       GM  = f)z=Load known system identifiers from seed file + discovery log.rutf-8encodingr9    nameN)setKNOWN_SYSTEMS_FILEDISCOVERY_LOGexistsopenr?   jsonloadsgetaddrB   r>   JSONDecodeError)knownfpathflineentryr9   rI   s          r&   load_known_idsrY   }   s    EE$m4||~~ZZgZ.!zz|	 JJt,E))E2.C		)C.1 99VR0668>>@D		$   /. 5$ L ++  /.s1   D!BD0DD	DD	D
D/	item	known_idsc                     U R                  SS5      nU R                  SS5      R                  5       R                  5       n[        U5      U;   =(       d    X1;   $ )Nr9   rH   rI   )rQ   r>   r?   rB   )rZ   r[   r9   rI   s       r&   is_knownr]      sK    
((5"
C88FB%%'--/DS>Y&;$*;;r(   query_configgithub_tokenc                 x   U S   U S   U S   U S   S.nSS0nU(       a  SU 3US	'    [         R                  " [        UU[        S
9nUR	                  5         UR                  5       nUR                  S5      =(       d    / n/ nU GHF  n	U	R                  S5      =(       d    U	R                  S5      =(       d    Sn
[        U
5      (       d  MH  SnU	R                  S5      (       aP  [        U	S   [        5      (       a8  U	S   R                  S5      =(       d    U	S   R                  S5      =(       d    SnUR                  U	R                  S5      =(       d    U	R                  SS5      U	R                  SS5      U	R                  S5      =(       d    SU	R                  SS5      U	R                  S5      =(       d    / UU
U S   U
S.	5        GMI     [        R                  SU S   [        U5      [        U5      5        U$ ! [         R                   a&  n[        R                  SU S   U5        / s SnA$ SnAff = f)zX
Fetch repositories from GitHub search API.
Returns list of normalised discovery dicts.
r   r   r   r   )r   r   r   r   Acceptzapplication/vnd.github+jsonzBearer Authorization)paramsheaderstimeoutz&GitHub fetch failed for query '%s': %sNitems	pushed_at
updated_atrH   licensespdx_idrI   	full_namehtml_urldescriptionstargazers_countr   topicsr   )	rI   r9   rm   r
   ro   ri   rg   sourceraw_datez(GitHub [%s]: fetched %d repos, %d recent)requestsrQ   GITHUB_API_BASEREQUEST_TIMEOUTraise_for_statusrO   RequestExceptionlogwarningr8   
isinstancedictappendinfolen)r^   r_   rc   rd   respdataexcrf   resultsreporg   license_names               r&   fetch_githubr      s    #V$g& ,	F 67G%,\N#; ||#	
 	yy{
 HHW#EGHH[)ITXXl-CIr	##88I:d9ot#D#D	?..y9^T)_=P=PQW=X^\^LHHV$Ab(A88J+88M28bXX0!4hhx(.B#""7+!

 
	 * HH2Ws5z3w< N? $$ <l3>OQTU	s   ?G? ?H9H4.H94H9c                  *    [         R                  " [        [        S9n U R	                  5         U R                  5       nUR                  S5      =(       d    / n/ nU GHO  nUR                  S5      =(       d    UR                  S5      n[        U[        [        45      (       a1  [        R                  " U[        R                  S9R!                  5       n[#        U=(       d    S5      (       d  M  UR                  S	5      =(       d    S
UR                  SS5       3nUR                  S5      =(       d    SnUR                  S5      =(       d    Sn	UR%                  UUU	(       a  U	SS OUUR                  SS5      / SU=(       d    SSU=(       d    SUR                  SS5      S.
5        GMR     [        R'                  S[)        U5      [)        U5      5        U$ ! [         R                   a"  n[        R                  SU5        / s SnA$ SnAff = f)zo
Fetch Show HN posts about AI memory from Hacker News Algolia API.
Returns list of normalised discovery dicts.
re   zHacker News fetch failed: %sNhits
created_atcreated_at_i)tzrH   r9   z%https://news.ycombinator.com/item?id=objectIDtitle
story_texti  pointsr   hackernews_show_hn)
rI   r9   rm   r
   ro   ri   rg   rp   rq   hn_idz'Hacker News: fetched %d hits, %d recent)rr   rQ   
HN_API_URLrt   ru   rO   rv   rw   rx   ry   intfloatr   fromtimestampr   r3   	isoformatr8   r{   r|   r}   )
r~   r   r   r   r   hitr   r9   r   r   s
             r&   fetch_hacker_newsr      s   
||J@yy{
 88F!rDGWW\*Ecggn.E
j3,//!//
x||LVVXJ)r**ggena"GPZ\^H_G` a &BWW\*0b
/9:ds+uWWXq)#)r*"(bWWZ,
 	 2 HH6D	3w<PNA $$ 2C8	s   =G H0HHHzhttp://www.w3.org/2005/Atomc                  ~    [         R                  " [        [        S9n U R	                  5         U R
                  n [        R                  " U5      nUR                  S[         S35      n/ nU GH.  nUR                  S[         S35      nUb  UR
                  OSn[        U=(       d    S5      (       d  MI  UR                  S[         S	35      n	U	b3  U	R
                  =(       d    SR!                  5       R#                  S
S5      OSn
UR                  S[         S35      nUb6  UR
                  =(       d    SR!                  5       R#                  S
S5      SS OSnUR                  S[         S35      nUb#  UR
                  =(       d    SR!                  5       OSn/ nUR                  S[         S35       H]  nUR                  S[         S35      nUc  M!  UR
                  (       d  M4  UR%                  UR
                  R!                  5       5        M_     SnSUR'                  5       ;   a0  UR)                  5        H  nSU;   d  M  UR!                  S5      n  O   UR%                  U
U=(       d    UUS/ SQSU=(       d    SSU=(       d    SUSS US.5        GM1     [        R+                  S[-        U5      [-        U5      5        U$ ! [         R                   a"  n[        R                  SU5        / s SnA$ SnAff = f! [        R                   a"  n[        R                  SU5        / s SnA$ SnAff = f)z
Fetch recent arXiv papers on episodic memory + LLMs.
Parses Atom XML returned by the arXiv API.
Returns list of normalised discovery dicts.
r   zarXiv fetch failed: %sNzarXiv XML parse error: %s{z}entryz}updatedrH   z}title
 z}summaryi  z}idz}authorz}namez
github.comz.,)([]r   )arxivzresearch-paperllmmemoryr      )rI   r9   rm   r
   ro   ri   rg   rp   rq   authors	arxiv_urlz$arXiv: fetched %d entries, %d recent)rr   rQ   ARXIV_API_URLrt   ru   textrv   rw   rx   ET
fromstring
ParseErrorfindallARXIV_NSfindr8   r?   r2   r{   r>   splitr|   r}   )r~   xml_contentr   rootentriesr   rX   
updated_elr   title_elr   
summary_elsummaryid_elr   r   	author_elname_el
github_urltokens                       r&   fetch_arxivr     s   ||M?Cii
}}[)
 llRz12GGZZ"XJi 89
%/%;*//B''::8*G45DLDX$"++-55dC@^`ZZ"XJi 89
NXNd:??(b//199$DTcJjl

Rz./272CUZZ%2,,.	 H:X'>?Innr(6%:;G"w|||w||1134 @ 
7==?* 5(!&X!6J )
 	*"B B2r{"
 	? Z HH3S\3w<PNu $$ ,c2	 == /5	s:   9K L L!K>8L>LL<L71L<7L<actionscorec           
         [         R                  SSS9  [        R                  " [        R
                  5      R                  5       U R                  SS5      U R                  SS5      UU R                  SS5      U R                  S5      =(       d    SSS	 U R                  S
S5      US.n[        R                  SSS9 nUR                  [        R                  " USS9S-   5        SSS5        g! , (       d  f       g= f)zAppend a discovery log entry.T)parentsexist_okrI   rH   r9   rp   rm   N   r
   r   )	timestamprI   r9   r   rp   rm   r
   r   arE   rF   F)ensure_asciir   )	SCOUT_DIRmkdirr   r4   r   r3   r   rQ   rL   rN   writerO   dumps)rZ   r   r   rX   rV   s        r&   write_log_entryr   [  s    OOD4O0\\(,,/99;$xxr"((8R(/52t<'1%	E 
		C'		2a	

5u5<= 
3	2	2s   ?(C00
C>c                     [         R                  R                  S[        [        5      5        SSKJn  U" U SS9$ ! [         a    [        R                  S5         Of = f [        R                  " SU SS9nUR                  5         UR                  5       $ ! [        R                   a.  n[        R                  S	U5        S
[        U5      S.s SnA$ SnAff = f)z~
Pass a discovery to the process_discovery.py webhook processor.
Falls back to inline import if webhook server isn't running.
r   )process_discoveryF)dry_runu;   Could not import process_discovery — falling back to HTTPz)http://localhost:8765/memory-scout-ingest
   )rO   re   zWebhook POST failed: %serror)action_takenr   N)r$   pathinsertstrr   r   ImportErrorrw   rx   rr   postru   rO   rv   r   )rZ   r   r~   r   s       r&   process_via_webhookr   l  s    S3y>*7 u55 SQRS
<}}7

 	yy{$$ <		+S1 '#c(;;<s-   9< AA"5B C,#CCCr   ingestc                    [        U5        [        R                  SU (       a  SO
U(       a  SOS5        [        5       n[        R                  S[	        U5      5        / n[
         H7  n[        XbS9nUR                  U5        [        R                  " [        5        M9     [        5       nUR                  U5        [        R                  " [        5        [        5       n	UR                  U	5        [        R                  S[	        U5      5        [        5       n
/ nU HR  nUR                  SS	5      n[        U5      nX;  d  M'  U(       d  M0  U
R!                  U5        UR#                  U5        MT     [        R                  S
[	        U5      5        U Vs/ s H  n[%        X5      (       a  M  UPM     nn[        R                  S[	        U5      5        [&        R(                  " [*        R,                  5      R/                  5       U (       a  SO
U(       a  SOS[	        U5      [	        U5      [	        U5      / 0 S.nU He  nUR                  SS5      nUS   R1                  USSS.5        US   U   S==   S-  ss'   [%        X5      (       a  MR  US   U   S==   S-  ss'   Mg     U GH  nUR                  SS	5      nUR                  SS	5      nUR                  SS	5      nUR                  SS5      nU (       a@  [        R                  SUUUU5        US   R#                  UUSUS.5        [3        US5        M  U(       a  [5        U5      nUR                  SS5      n[        R                  SUUUR                  SS5      5        US   R#                  UUUUR                  S5      UR                  S 5      UUR                  S!5      UR                  S"5      S#.5        GM;  [        R                  S$UUUU5        UR                  S%5      =(       d    S	S&S' nU(       a  [        R                  S(U5        [3        US5        US   R#                  UUS)US.5        GM     [7        S*5        [7        S+5        [7        S,5        [7        S-US.    35        [7        S/US0    35        [7        S1US2    35        [7        S3US4    35        [7        S5US6    35        [7        5         [7        S75        US   R9                  5        H!  u  nn[7        S8US9 S:US    S;US    35        M#     US   (       a  [7        5         [7        S<5        US    Hl  nUR                  S5      b  S=US    3OS	nUR                  S 5      (       a  S>US     3OS	n[7        S?US@    SAU U S8US    35        [7        SBUS    35        Mn     O[7        SC5        [7        SD5        U$ s  snf )EaC  
Execute the full intelligence sweep.

Args:
    dry_run: Print findings without writing anything
    ingest:  Write discoveries to KG + tasks.json
    github_token: GitHub personal access token (optional, increases rate limit)
    verbose: Extra logging

Returns:
    Summary dict with counts per source and actions taken
u9   Memory Competitor Intelligence Scout starting — mode=%sr   r   report_onlyz,Loaded %d known system IDs for deduplication)r_   z*Total items fetched across all sources: %dr9   rH   z*Unique items after batch deduplication: %dzNew items (not in KG yet): %d)run_atmodetotal_fetchedunique_fetched	new_itemsactions	by_sourcerp   unknownr   r   )totalnewr      r   rI   r
   u7   [DRY RUN] Would process: %s (%s) — source=%s stars=%dr   would_process)rI   r9   r   rp   dry_run_candidater   zIngested %s: action=%s score=%sr   ?	tool_typekg_entity_idtask_id)rI   r9   r   r   r   rp   r   r   u%   FOUND: %s (%s) — source=%s stars=%drm   Nx   z  Description: %sreportedz=
============================================================zMEMORY SCOUT SWEEP SUMMARYz<============================================================zRun at:         r   zMode:           r   zTotal fetched:  r   zUnique items:   r   zNew (unknown):  r   z
By source:z  z<30z total=z  new=zActions taken:z  score=z  type=z  [r   ]z    z
No new discoveries this sweep.z=============================================================
)r'   rw   r|   rY   r}   GITHUB_QUERIESr   extendtimesleepRATE_LIMIT_SLEEPr   r   rJ   rQ   rB   rR   r{   r]   r   r4   r   r3   r   
setdefaultr   r   printrf   )r   r   r_   r   r[   	all_itemsqueryrf   hn_itemsarxiv_items	seen_urlsunique_itemsrZ   r9   uidr   r   srcrI   rp   r
   resultr   r   countsaction_entry	score_strtype_strs                               r&   	run_scoutr     se   $ 'HHH!YFP  IHH;S^L I  U>

#$   !"HXJJ  -K[!HH93y>J ILhhub!nCCMM#%  HH93|;LM #/P,$ht6O,IPHH,c)n= ,,x||,668$	v8=Yl+^G hhx+''q-CDS!'*a/*((K %e,1,  xx#hhub!(B'!$HHNPTVY[achiI%%SOv'  D"56(.FZZ	:FHH6ffjjQXZ]F^_I%% G,#ZZ4  &

> :!::i0	' 	 HH<dCQVWHH]+1r4C8D,d3D-0I%%SJ&' C L 
/	
&'	(O	WX./
01	WV_-
./	W_56
78	W%567
89	W[12
34	G	,{+113V3s)76'?"36&-IJ 4 y#I.L>J>N>Nw>W>c(<#8"9:ikI@L@P@PQ\@]@]k!: ;<ceHCX./qH:RU[H\G]^_De,-./	 / 	01	/Nk Qs   W.Wc                     [         R                  " S[         R                  SS9n U R                  SS9nUR	                  SSSS	9  UR	                  S
SSS	9  UR	                  SSSS	9  U R	                  SS SS9  U R	                  SSSS	9  U R	                  SS SS9  U R                  5       nUR                  (       a  [        UR                  5      q[        S-  q	UR                  =(       d    [        R                  R                  S5      n[        UR                  UR                   UUR"                  S9nUR                   (       a,  UR                  SS5      S:  a  [$        R&                  " S5        [$        R&                  " S5        g )Nu?   Memory Competitor Intelligence Scout — standalone cron runneraR  
Examples:
  python scout_cron.py --dry-run            Print discoveries, write nothing
  python scout_cron.py --ingest             Write new items to KG + tasks.json
  python scout_cron.py --ingest --verbose   Verbose output during ingest
  python scout_cron.py --dry-run --token ghp_xxx  Use GitHub token for higher rate limits
        )rm   formatter_classepilogT)requiredz	--dry-run
store_truez+Print discoveries without writing any files)r   helpz--ingestz;Process discoveries: write KG entities and evaluation tasksz--reportz:Log findings to discovery_log.jsonl but do not write to KGz--tokenzFGitHub personal access token (increases rate limit from 60 to 5000/hr))defaultr  z	--verbosezEnable debug loggingz--output-dirzROverride output directory (default: E:\genesis-system\n8n\memory_competitor_scout)r	   GITHUB_TOKEN)r   r   r_   r   r   r   )argparseArgumentParserRawDescriptionHelpFormatteradd_mutually_exclusive_groupadd_argument
parse_args
output_dirr   r   rL   r   osenvironrQ   r   r   r   r   r$   exit)parser
mode_groupargsr_   r   s        r&   mainr  !  s   $$U <<
F 44d4CJ:  
 J  
 I   U  
 #  
 d   D )	!$99 ::?!?L{{!	G {{w{{;2Q6HHQKr(   __main__)F)N)r   )FFNF)3__doc__r  rO   r   r$   r   xml.etree.ElementTreeetreeElementTreer   r   r   r   pathlibr   rr   BASE_DIRr   rL   rK   r   rs   r   r   MAX_AGE_DAYSrt   r   	getLoggerrw   boolr'   r   r   r8   r<   rB   rJ   rY   rz   r]   listr   r   r   r   r   r   r   r  r  __name__ r(   r&   <module>r!     s1  *    
  " " 2 2   #$u88	11!== 
 2 '! 6&0 ?o
B    %4 D  .:  s d & F3 F3 F 0<4 <C <D <7t 73 7$ 7|)4 )` )DT DV>$ > >C > >"<d <t <@ 	OOO O 	O
 
OlCL 
zF r(   