
    JGjQ                     h   d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlmZ ddlmZ ddlmZ ddlZddlZej$                  j'                  dej$                  j)                  ej$                  j+                  e                   ddlmZ ddlmZmZmZ  ej:                  e      Zded	efd
Z ded	efdZ!d$de"d	e#fdZ$de"d	e#dz  fdZ%de&e"   de"de&e"   d	e&e"   fdZ'd%de#de#de(d	e)fdZ*d&de#de"de+d	e)fdZ,	 	 d'de#de#de"de"d	e)f
dZ- G d  d!      Z.d" Z/ed#k(  r e/        yy)(a  
Independent RAG collector for Kimi sessions.

Watches one or more Kimi sessions (by session id or work directory) and calls
qwen to decide whether the conversation is worth saving to the RAG knowledge
base. Designed to run quietly in the background and accumulate valuable
experience without interfering with the live Kimi wrapper.

Examples:
    # Watch a single session
    python3 rag_collector.py --session-id ec414c6e-dd35-461e-9bba-ead77e7d3ac1

    # Watch all sessions under a project directory
    python3 rag_collector.py --work-dir "/Users/admin/workspace/jyq/qsjnic"

    # Analyze current data once, without writing to RAG (good for testing)
    python3 rag_collector.py --session-id ec414c6e-... --once --dry-run
    N)datetime)Path)Any)	HintPanel)Turnlist_all_sessionsload_sessionvaluereturnc                 z    t        | t              s| S t        j                  d      }d }|j	                  ||       S )z5Replace ${VAR:default} placeholders in config values.z\$\{([^}:]+)(?::([^}]*))?\}c                     | j                  d      }| j                  d      xs d}t        j                  j                  ||      S )N       )grouposenvironget)matchvar_namedefaults      8/Users/admin/workspace/jyq/kimi-copilot/rag_collector.pyreplz_resolve_env.<locals>.repl5   s6    ;;q>++a.&Bzz~~h00    )
isinstancestrrecompilesub)r
   patternr   s      r   _resolve_envr!   /   s8    eS!jj78G1
 ;;tU##r   objc                     t        | t              r-| j                         D ci c]  \  }}|t        |       c}}S t        | t              r| D cg c]  }t        |       c}S t        |       S c c}}w c c}w )N)r   dictitems_deep_resolve_envlistr!   )r"   kvitems       r   r&   r&   =   sg    #t47IIK@DAq$Q''@@#t478D!$'88 A8s   A2A8pathc                     t        | dd      5 }t        j                  |      }d d d        t        |      S # 1 sw Y   t              S xY w)Nrutf-8encoding)openyaml	safe_loadr&   )r+   fraws      r   load_configr6   E   sD    	dC'	*  annQ S!! S!!s	   8Atextc                    | j                         } | j                  d      rC| j                  d      } | j                         j                  d      r| dd j                         } 	 t        j                  |       S # t        j
                  $ r Y nw xY wt        j                  d| t        j                        }|r>	 t        j                  |j                  d            S # t        j
                  $ r Y yw xY wy)z.Best-effort JSON extraction from model output.z````json   Nz\{.*\}r   )
strip
startswithlowerr:   loadsJSONDecodeErrorr   searchDOTALLr   )r7   r   s     r   extract_jsonrC   K   s    ::<Duzz#::<""6*8>>#Dzz$  IIiryy1E	::ekk!n--## 		s$   &A; ;BB<#C   C65C6tagsprojectdefault_scopesc                     t        | xs g       }|rd| }||vr|j                  |       |D ]  }||vs|j                  |        |S )z2Add project and default scope tags to a RAG entry.zproject:)r'   append)rD   rE   rF   enrichedproject_tagscopes         r   _enrich_tagsrL   `   s]    DJBH 	*h&OOK( # OOE"# Or   ab	thresholdc                    | j                  d      xs dj                         }|j                  d      xs dj                         }|r-|r+t        j                  d||      j	                         }||k\  ry| j                  d      xs dj                         }|j                  d      xs dj                         }|r-|r+t        j                  d||      j	                         }||k\  ryy)z4Return True if two RAG entries look like duplicates.titler   NTquestionF)r   r>   difflibSequenceMatcherratio)rM   rN   rO   a_titleb_titlerU   
a_question
b_questions           r   _is_similar_ragrZ   m   s    uuW~#**,GuuW~#**,G7''gw?EEGI%%
#)r002J%%
#)r002Jj''j*EKKMIr   	rag_entry
panel_pathlookbackc                 r   t        |      }|j                         sy	 t        |dd      5 }t        j                  |      }ddd       j                  dg       D cg c]  }|j                  d      dk(  s| }}|| d D ]  }t        | |      s y	 	 y# 1 sw Y   XxY wc c}w # t        $ r Y yw xY w)
zDCheck whether a similar RAG entry already exists in the local panel.Fr-   r.   r/   NrecordstyperagT)r   existsr1   r:   loadr   rZ   	Exception)	r[   r\   r]   r+   r4   datar-   r_   records	            r   _has_duplicate_in_panelrg      s    
D;;=$g. 	 !99Q<D	 "hhy"5Pv%9O1PPxij) 	Fy&1	
 	  	 P  sF   B* BB* B%8B%<B* B* B* B"B* *	B65B6configsourcec           
      $   |j                  di       }|j                  d      }|st        j                  d       y| j                  dd      }| j                  dd      }| j                  d	d      }| j                  d
d      }	t        | j                  dg       ||j                  dg             }
t	        |       rt        j                  d|       yd| d| d|	 ddj                  |
       d| 
}||d}t        j                  d|       	 t        j                  ||d      }|j                          t        j                  d|j                                y# t        j                  $ r }t        j                  d|       Y d}~yd}~ww xY w)z3Write a structured entry to the RAG knowledge base.ra   urlz'RAG URL not configured, skipping write.FrQ      未命名知识rR   r   wrong_approachcorrect_approachrD   rF   z Skipping duplicate RAG entry: %su	   问题：u   
错误方案：u   
正确方案：u
   
标签：z, u
   
来源：)rQ   contentzWriting to RAG: %s   )r:   timeoutzRAG write OK: %sTzRAG write failed: %sN)r   loggerwarningrL   rg   infojoinrequestspostraise_for_statusr:   RequestExceptionerror)r[   rh   rE   ri   rag_cfgrk   rQ   rR   wrongcorrectrD   ro   payloadrespes                  r   	write_ragr      s    jj#G
++e
C@AMM'#45E}}Z,HMM*B/Emm.3Gfb!$b)D y)6> H:  !! #IIdO$ %8		  '2G
KK$e,}}Sw;&		4$$ +Q/s   AE F/F

Fc                       e Zd ZdZ	 	 	 	 ddededz  dedz  dedef
dZd	 Zd
efdZ	d
e
eeeef      fdZded
efdZde
e   d
e
e   fdZde
e   d
edz  fdZdededefdZd Zd Zd Zd Zd Zy)RAGCollectorzIBackground service that watches Kimi sessions and summarizes them to RAG.Nrh   
session_idwork_dirdry_runoncec                 *   || _         |xs dj                         j                         xs d | _        |xs dj                         xs d | _        || _        || _        |j                  di       }|j                  dd      | _        |j                  dd      | _	        |j                  dd      | _
        |j                  dd	      | _        |j                  d
d      | _        |j                  dd      | _        |j                  dd      | _        t        j                          | _        t%        j&                         | _        i | _        i | _        i | _        t1               | _        d | _        d | _        t9               | _        | j=                          y )Nr   rag_collectorpoll_interval_seconds   analysis_cooldown_seconds<   silence_window_secondsmax_turns_per_analysis
   rq   
max_tokensi   judge_promptzprompts/copilot_rag_judge.txt)rh   r<   r>   target_session_idtarget_work_dirr   r   r   poll_intervalcooldownsilence_window	max_turnsrq   r   r   	threadingEvent_stop_eventqueueQueue_analysis_queue_processed_turns_last_analysis_time_last_turn_timeset_summarized_sessions_poll_thread_worker_threadr   panel_setup_logging)selfrh   r   r   r   r   cfgs          r   __init__zRAGCollector.__init__   sS    ","2!9!9!;!A!A!C!Kt (B557?4	jj"- WW%<a@ ;R@!gg&>C!92>wwy"-'',5GGN4ST$??,=B[[]5757 13.1e!597; [
r   c                    | j                   j                  di       j                  dd      }| j                   j                  di       j                  dd      }t        j                         }|j                  ry t        j
                  |d      }|j                  t        j                  d             |j                  t        t        |j                         t        j                               |j                  |       y )	NlogginglevelINFOfilezkimi-copilot.logr.   r/   z'%(asctime)s [%(levelname)s] %(message)s)rh   r   r   	getLoggerhandlersFileHandlersetFormatter	FormattersetLevelgetattrupperr   
addHandler)r   r   log_pathroothandlers        r   r   zRAGCollector._setup_logging   s    	2.227FC;;??9b155f>PQ  "==%%hAGH	
 	ggu{{}gllCD r   r   c                     t         j                  j                  | j                        r2t	        | j                  dd      5 }|j                         cd d d        S 	 y# 1 sw Y   	 yxY w)Nr-   r.   r/   zJudge whether the following coding conversation is worth saving to a RAG knowledge base. Output JSON with worth_rag and rag_entry fields.)r   r+   rb   r   r1   read)r   r4   s     r   _load_judge_promptzRAGCollector._load_judge_prompt   s`    77>>$++,d''w?  1vvx   S	
 S	
s   AA(c                 j   t               }g }|D ]  }|j                  j                         }| j                  r|| j                  k7  r| j                  |vrG| j                  r| j                  |j
                  xs dvrp|j                  |j                  |j                  |j                  f        |S )z:Return list of (session_id, session_dir, source) to watch.r   )	r   r   r>   r   r   r   rH   session_dirri   )r   metasresultsmetasids        r   _target_sessionszRAGCollector._target_sessions  s    !# 	MD//'')C%%#1G1G*G))4##(<(<T]]EXVX(YNNDOOT-=-=t{{KL	M r   turnc                    d|j                   xs |j                   g}|j                  s|j                  r.|j                  xs |j                  }|j	                  d|        |j
                  s|j                  r.|j
                  xs |j                  }|j	                  d|        |j                  r|j	                  d|j                          dj                  |      S )Nu   【用户】u   【Kimi 思考】u   【Kimi 回答】u   【工具调用】z

)	clean_user_input
user_inputclean_thinkingassistant_thinkingrH   clean_answerassistant_texttool_summaryru   )r   r   partsthinkinganswers        r   _format_turnzRAGCollector._format_turn  s     5 5 HIJK$"9"9**Ed.E.EHLL,XJ78 3 3&&=$*=*=FLL,VH56LL-d.?.?-@AB{{5!!r   turnsc                    | j                         }d|dg}g }|j                  r|j                  d|j                          |j                  r|j                  d|j                          |j                  d|j                          |j                  d       |j                  ddj                  |      d       |D ]%  }|j                  d| j                  |      d       ' |j                  dd	d       |S )
Nsystem)rolero   u   工作目录：u   数据来源：u   Session ID：u-   以下是对话内容（按时间顺序）：user
us   请根据以上对话，判断是否值得写入 RAG 知识库。只输出 JSON，不要有任何解释性文字。)r   r   rH   ri   r   ru   r   )r   sessionr   system_contentmessagescontext_partsr   s          r   _build_analysis_messagesz%RAGCollector._build_analysis_messages  s    002%.AB  ?73C3C2D!EF>>  ?7>>2B!CD}W-?-?,@ABLMDIIm4LMN 	RDOOV8I8I$8OPQ	R 	I	
 r   r   c           	         | j                   j                  di       }|j                  d      || j                   j                  di       j                  dd      | j                  d}	 t        j                  |j                  d      dd	|j                  d
d       d|| j
                        }|j                  s:t        j                  d|j                  |j                  |j                  d d        y |j                         }|d   d   d   d   }t#        |      }|st        j%                  d|d d        y |S # t        j                  t        t        t        j                   f$ r }t        j                  d|       Y d }~y d }~ww xY w)Nmodelnamecopilottemperatureg?)r   r   r   r   base_urlzapplication/jsonzBearer api_keyr   )zContent-TypeAuthorization)headersr:   rq   z$RAG judge request failed: %s %s - %si  choicesr   messagero   zRAG judge request failed: %sz'RAG judge did not return valid JSON: %r   )rh   r   r   rv   rw   rq   okrr   rz   status_codereasonr7   r:   ry   KeyError
IndexErrorr@   rC   rs   )	r   r   	model_cfgr~   r   re   ro   r   results	            r   
_call_qwenzRAGCollector._call_qwen:  s]   KKOOGR0	]]6* ;;??9b9==mSQ//	
	==j)$6'.y}}Y/K.L%M D 77:$$KKIIdsO	 99;D9oa(3I>G
 g&NNDgdsmT ))8ZAUAUV 	LL7;	s   (BD= 6D= =-F
*FF
r   ri   c                 &   | j                   r%|| j                  v rt        j                  d|       y t	        j                         }| j
                  j                  |d      }||z
  | j                  k  rt        j                  d|       y t        ||      }|j                  sd|_
        t        |j                        | j                  kD  r|j                  | j                   d  n|j                  }|sy | j                  ||      }t        d |D              }	|	dkD  rt        j!                  d|       y t        j                  d	|t        |             | j#                  |      }
|
sy t	        j                         | j
                  |<   | j$                  rTt        j                  d
|t'        j(                  |
d             t+        d| dt'        j(                  |
dd              y |
j                  d      st        j                  d|       y |
j                  dd      }t-        |t.        t0        f      r|dk  rt        j                  d||       y |
j                  d      xs i }|s|
j                  d      r]|
j                  dd      |
j                  dd      |
j                  dd      |
j                  dd      |
j                  dg       xs g d}nt        j!                  d|       y |
j                  d      xs d}|s+|j                  rt3        |j                        j4                  }|j                  dd      }|j                  dd      }|j                  dd      }|j                  dd      }|j                  dg       xs g }t7        || j8                  |d !      }|rc| j                  j;                  |       | j<                  j?                  |||||"       t+        d#| d$| d%|
j                  d&d       d'|        y y )(Nz3Session %s already summarized in this run, skippingr   z)Session %s in cooldown, skipping analysisri   r   c              3   R   K   | ]  }t        |j                  d d             ! yw)ro   r   N)lenr   ).0ms     r   	<genexpr>z0RAGCollector._analyze_session.<locals>.<genexpr>t  s      G3quuY34Gs   %'i z&Session %s request too large, skippingz"Analyzing session %s with %d turnszDry-run result for %s: %sF)ensure_asciiz%[RAG Collector] dry-run analysis for z:
r   )r   indent	worth_ragzSession %s deemed not worth RAG
confidencer   z2Session %s confidence too low (%s), skipping writer[   rQ   rR   rm   rn   rD   )rQ   rR   rm   rn   rD   z*Session %s worth_rag=true but no rag_entryrE   rl   r   )rE   ri   )rQ   rR   r|   r}   rD   u2   [RAG Collector] ✅ 已写入知识库
  Session: u   
  标题: u   
  分类: categoryu   
  项目: ) r   r   rr   rt   timer   r   r   debugr	   r   r   r   r   r   sumrs   r   r   r:   dumpsprintr   intfloatr   r   r   rh   addr   add_rag)r   r   r   ri   nowlastr   r   r   request_sizer   r   r[   rE   rQ   rR   r|   r}   rD   r   s                       r   _analyze_sessionzRAGCollector._analyze_session`  s   99t'@'@@KKMzZiik''++J::%LLDjQ{6:!G474F4Wt~~o/0]d]j]j00%@GhGG& NNCZP8*c%jQ*/3yy{  ,<<KK3ZFafAgh7
|3::f5CDF zz+&KK9:FZZa0
*sEl3zA~KKLjZdeJJ{+1r	zz'"#ZZ4 &

:r :&,jj1A2&F(.

3Er(J"JJvr28b	 KZX**Y'-27++7++,11Gg'89==R0.3-- 2B7}}VR(.By$++wW%%))*5JJ!   (\ *"G $#ZZ
B78 9$I	' r   c                 N   | j                   j                         s	 | j                         }|D ]  \  }}}t        ||      }| j                  j                  |t                     }d}|j                  D ]2  }t        |j                        }	|	|vs|j                  |	       |dz  }4 |s~t        j                  d||       t        j                         | j                  |<   | j                  j!                  |||f        | j"                  r| j                  j!                  d       yt        j                         }
t%        | j                  j'                               D ]W  \  }}|
|z
  | j(                  k\  s|D ]+  \  }}}||k(  s| j                  j!                  |||f        n |
| j                  |<   Y 	 | j                   j/                  | j0                        ry| j                   j                         syy# t*        $ r t        j-                  d       Y dw xY w)zBContinuously discover sessions and enqueue new turns for analysis.r   r   r   zSession %s has %d new turn(s)NzError in poll loop)r   is_setr   r	   r   
setdefaultr   r   r   turn_idr  rr   rt   r   r   r   putr   r'   r%   r   rd   	exceptionwaitr   )r   targetsr   r   ri   r   	processednew_turn_countr   tidr  	last_turnr   sdirsrcs                  r   
_poll_loopzRAGCollector._poll_loop  s   ""))+#7//17> T3JV*;vFG $ 5 5 @ @SU SI%&N ' 0!$,,/i/%MM#.*a/N	0 &;Z <@99;,,Z8,,00*k61RST" 99((,,T2 iik-1$2F2F2L2L2N-O ?)J	Y$*=*==.5 &NCs"j0 $ 4 4 8 8#tS9I J %&
 <?,,Z8? $$T%7%78O ""))++F  7  !567s2   A4H H -A?H -AH =H 2H H$#H$c                    	 	 | j                   j                  d      }|y|\  }}}	 | j                  |||       | j                   j                          U# t        j                  $ r | j                  j                         rY yY w xY w# t        $ r t        j                  d|       Y qw xY w# | j                   j                          w xY w)z%Consume analysis jobs from the queue.r   rq   NzError analyzing session %s)r   r   r   Emptyr   r  r
  rd   rr   r  	task_done)r   r*   r   r   ri   s        r   _worker_loopzRAGCollector._worker_loop  s    ++///: |.2+JV1%%j+vF $$..0%  ;; ##**,  K  !=zJK $$..0s4   A B -B	B	B.+B1 -B..B1 1Cc                 .   t         j                  d       t        j                  | j                  d      | _        t        j                  | j                  d      | _        | j
                  j                          | j                  j                          y )NzStarting RAG collectorT)targetdaemon)	rr   rt   r   Threadr  r   r  r   startr   s    r   r#  zRAGCollector.start  sh    ,-%,,DOODQ'..d6G6GPTU!!!#r   c                 :   t         j                  d       | j                  j                          | j                  r| j                  j                  d       | j                  r8| j                  j                  d        | j                  j                  d       y y )NzStopping RAG collectorr   r  r   )	rr   rt   r   r   r   ru   r   r   r  r$  s    r   stopzRAGCollector.stop  sz    ,-""1"-  $$T*$$R$0 r   c                    | j                          | j                  rM| j                  r| j                  j                          | j                  r| j                  j                          y	 | j
                  j                         s0t        j                  d       | j
                  j                         s0| j                          y# t        $ r Y w xY w# | j                          w xY w)z'Block until stopped or --once finishes.Nr   )r#  r   r   ru   r   r   r  r   sleepKeyboardInterruptr&  r$  s    r   runzRAGCollector.run  s    

99  !!&&(""##((*	&&--/

1 &&--/
 IIK ! 		 IIKs%   +A	C 	CC CC C')NNFF)__name__
__module____qualname____doc__r$   r   boolr   r   r   r'   tupler   r   r   r   r   r   r
  r  r  r#  r&  r*   r   r   r   r      s   S
 "&#(( $J( *	(
 ( (T!
C 
$uS$^'<"= 
" 
"# 
"tDz d4j 6$4: $$+ $LZ3 ZT Z3 Zx)V1,$1r   r   c                  >   t        j                  d      } | j                  dt        d       | j                  dt        d       | j                  dd	d
       | j                  dd	d       | j                  dt        dd       | j	                         }|j
                  s|j                  s| j                  d       t        |j                        }t        ||j
                  |j                  |j                  |j                        }|j                          y )Nz+Independent RAG collector for Kimi sessions)descriptionz--session-idzWatch a specific session by ID)r`   helpz
--work-dirz2Watch sessions whose work dir contains this stringz	--dry-run
store_truezAnalyze but do not write to RAG)actionr4  z--oncez#Analyze existing data once and exitz--configconfig.yamlzPath to config.yaml)r`   r   r4  z-One of --session-id or --work-dir is required)rh   r   r   r   r   )argparseArgumentParseradd_argumentr   
parse_argsr   r   rz   r6   rh   r   r   r   r*  )parserargsrh   	collectors       r   mainr?  !  s    $$1^_F
S7WX
35ij
L?`a
<ab

mJ_`D??4==DE%F??YYI MMOr   __main__)r7  )g      ?)z
panel.json2   )r   r   )0r.  r8  rS   r:   r   r   r   r   sysr   r   r   pathlibr   typingr   rv   r2   r+   insertdirnameabspath__file__copilot_panelr   session_capturer   r   r	   r   r+  rr   r!   r&   r   r$   r6   rC   r'   rL   r  r/  rZ   r  rg   r   r   r?  r1  r   r   <module>rK     s  &     	  	 
        277??277??8#<= > # A A 
		8	$$ $ $3 3 "c "d "s td{ *
tCy 
3 
S	 
dSVi 
t   $ &t  WZ dh ( !	--- - 	-
 
-`] ]@. zF r   