i?dZddlZddlZddlZddlZddlmZddlmZm Z m Z ddl Z ddl Z ddl ZddlmZmZmZmZddlmZmZddlmZej2eZGdd eZy) zH FileMemory: Memory system for vectorizing and retrieving file contents N)Path)ListDictAny) MemoryBaseMemoryContentSnapshot MemoryItemMemoryWritePayload)MemoryStoreConfigFileSourceConfig)FileMemoryConfigc @eZdZdZdeffd Zd!dZd!dZded e d e d e de e f d Zd eddfdZd!dZd!dZdefdZdede efdZdede edefdZd"dedede efdZdede efdZde ede e fdZdedefdZd"dededdfdZde eddfd ZxZ S)# FileMemoryz File-based memory system that indexes and retrieves content from files/directories. Supports multiple file types, chunking strategies, and incremental updates. storec&|jt}|s tdt|||j s td||_|j |_|j j|_d|_d|_ i|_ y)Nz5FileMemory requires a file memory store configurationz=FileMemory requires at least one file_source in configurationi2) as_configr ValueErrorsuper__init__ file_sources file_config index_path chunk_size chunk_overlap file_metadata)selfrconfig __class__s R/Users/bowang/.openclaw/workspace/ChatDev/runtime/node/agent/memory/file_memory.pyrzFileMemory.__init__!s!12TU U ""\] ]!4:4G4G**559;returnNc|jrtjj|jritj d|j|j |jr&tj d|jyytj d|j|jr|jyy)z Load existing index or build new one from file sources. Validates index integrity and performs incremental updates if needed. zLoading existing index from z!Index updated due to file changesz$Building new index from file sourcesN) rospathexistsloggerinfo_load_from_file_validate_and_update_indexsave_build_index_from_sources)rs r loadzFileMemory.load5s ??rww~~doo> KK6t6GH I  "..0 ?@ 1 KK> ?  * * , r!cf|jstjdytjtj j |jd|j|jDcgc]}|jc}|j|jdd}t|jdd 5}tj||d d dddtjd |jdt!|jdycc}w#1swYHxYw)z Persist the memory index to diskz&No index_path specified, skipping saveNT)exist_ok)rr)rcontentsrwutf-8encodingF)indent ensure_asciizIndex saved to z (z chunks))rr'warningr$makedirsr%dirnamerr0to_dictrropenjsondumpr(len)ritemdatafs r r+zFileMemory.saveHs NNC D  BGGOODOO4tD"//48MMBMDMB"oo!%!3!3 $//3 9Q IIdAae <:  odoo%6bT]]9K8LHUVC: 9s:D"D''D0 agent_rolequerytop_ksimilarity_thresholdc ,|jdk(rgS|jj|j}t |t r%t j|t j}|jdd}tj||jd}g}g}|jD]} | jt| j|k7r6tj!d| j"t| j|^|j%| j|j%| |sgSt j|t j}tj&|jd} | j)|| j+|t-|t|\} } g} t/t| dD]2}| d|}| d|}|dk7s||k\s| j%||4| S)aV Retrieve relevant file chunks based on query. Args: agent_role: Agent role (not used in file memory) inputs: Query text top_k: Number of results to return similarity_threshold: Minimum similarity score Returns: List of MemoryItem with file chunks rdtypez8Skipping memory item %s: embedding dim %d != expected %d)count_memories embedding get_embeddingtext isinstancelistnparrayfloat32reshapefaiss normalize_L2shaper0r?r'r8idappend IndexFlatIPaddsearchminrange)rrCrDrErFquery_embedding expected_dimmemory_embeddings valid_itemsr@index similaritiesindicesresultsiidx similaritys r retrievezFileMemory.retrieveas&    A %I..66uzzB ot , hhbjjIO)11!R8 ?+&,,Q/  MMD~~)t~~&,6NNRT^^!4l!((8""4("!IHH%6bjjI!!"3"9"9!"<= #$ % _c%[IY>Z [ gs71:'A!*Q-C%a+JbyZ+??{3/0 (r!payloadc.tjdy)z FileMemory is read-only, updates are not supported. This method is a no-op to maintain interface compatibility. z6FileMemory.update() called but FileMemory is read-onlyN)r'debug)rrls r updatezFileMemory.updates  MN r!c t|jdd5}tj|}dddj di|_|j dg}g}|D]'} |j tj|)||_ |j di}|j d|j|_ |j d |j|_ tjd t|jd y#1swYxYw#t$rYwxYw#t$r0}tj!d |i|_g|_ Yd}~yd}~wwxYw) zLoad index from JSON filerr2r3Nrr0rrrzLoaded z chunks from indexzError loading index: )r<rr=r-getrrZr from_dict Exceptionr0rrr'r(r?error)rrBrA raw_contentsr0rawres r r)zFileMemory._load_from_files7 doosW=yy|>"&/2!>D 88J3L)+H#OOJ$8$8$=>$ %DMXXh+F$jjtGDO!'OT=O=O!PD  KK'#dmm"4!55GH I%>=!  LL04 5!#D DMM sRD2D8D2)$D# B D2D D2# D/,D2.D//D22 E+;&E&&E+cg}|jD]}tjd|j|j |}tjdt |d|j|D]/}|j ||j}|j|1tjdt ||j||_ tjdt |jdy)z(Build index by scanning all file sourceszScanning source: zFound z files in zTotal chunks to index: zIndex built with z chunksN) rr'r( source_path _scan_filesr?_read_and_chunk_filer4extend_build_embeddingsr0)r all_chunkssourcefiles file_pathchunkss r r,z$FileMemory._build_index_from_sourcess ''F KK+F,>,>+?@ A$$V,E KK&U Jv7I7I6JK L" 229fooN!!&)# (  -c*o->?@..z:  'DMM(:';7CDr!cd}t}|jD]$}|j|}|j|&t|jj }||z }|r5t jdt|d|j|d}|jD]}|j|}|D]}|j|}||jvr7t jd||j||jd}Y|j|jd|k7s{t jd||j|g|j||jd}|S)z Validate index integrity and update if files changed. Returns: True if index was updated, False otherwise Fz Removing z deleted files from indexTzIndexing new file: hashzRe-indexing modified file: )setrr{rorkeysr'r(r?_remove_files_from_index_compute_file_hash _index_filer4rr) rupdated current_filesrr indexed_files deleted_filesr file_hashs r r*z%FileMemory._validate_and_update_indexst ''F$$V,E   '( D..3356 % 5  KK)C $6#77PQ R  ) )- 8G''F$$V,E"  33I> D$6$66KK"5i[ AB$$Y@"G'' 266v>)KKK"=i[ IJ119+>$$Y@"G#(&r!rct|jjj}|j r*|j ||j r t|gSgS|js$tjd|jgSg}|jr`|jdD]J}|j s|j ||j s1|jt|L|S|jdD]J}|j s|j ||j s1|jt|L|S)z Scan file path and return list of matching files. Args: source: FileSourceConfig with path and filters Returns: List of absolute file paths zPath does not exist: *)rrz expanduserresolveis_file_matches_file_types file_typesstris_dirr'r8 recursiverglobrZglob)rrr%rrs r r{zFileMemory._scan_files s'F&&'224<<> <<>''f.?.?@D {"I{{} NN263E3E2FG HI   !ZZ_ $$&4+C+CIvO`O`+aLLY0- "YYs^ $$&4+C+CIvO`O`+aLLY0, r!rrc$|y|j|vS)z*Check if file matches the file type filterT)suffix)rrrs r rzFileMemory._matches_file_types1s  :--r!r4c  t|d|d5}|j}dddj sgS|j |}tjj|}|j|}g} t|D]H\} } | j| d|tjj|||| t||dd J||t|tjd |j |<| S#1swYxYw#t$r'}tj d|d|gcYd}~Sd}~wwxYw) z Read file and split into chunks. Args: file_path: Path to file encoding: File encoding Returns: List of chunk dictionaries with content and metadata rqignore)r4errorsNzError reading file : file) source_typer file_namer file_size chunk_index total_chunksr4)contentmetadata)rsize chunks_count indexed_at)r<readrtr'rustriprr$r%getsize _chunk_text enumeraterZbasenamer?timer) rrr4rBrrxrrr chunk_dictsrh chunk_texts r r|zFileMemory._read_and_chunk_file7s9 ixIQ&&(J }}I++I6 GGOOI. !!'* &v.MAz   %#)!*!#!1!1)!E E rc g}|D]}|d}|d} |jj|}t|tr5t j |tj jdd}tj||jd}|d d |d } t| |||tj } |j!| |S#t$r#}tjd|Yd}~d}~wwxYw) z Generate embeddings for chunks and create MemoryItems. Args: chunks: List of chunk dictionaries Returns: List of MemoryItem objects rrrHrJrKrz&Error generating embedding for chunk: Nr_r)rYcontent_summaryrrM timestamp)rMrNrPrQrRrSrTrUrVrWtolistrtr'rur rrZ) rr memory_items chunk_dictrrrMembedding_listrxitem_id memory_items r r~zFileMemory._build_embeddingss   J +G!*-H  NN88A i. ""** E M MaQS TI""9-!*!1!1!3A!6 "+./q-1H0IJG$ '!())+ K    ,3!6!  EaSIJ sBC D (DD cDtj} t|d5tfddD]}|j | ddd|j ddS#1swYxYw#t $r%}tjd|d|Yd}~yd}~wwxYw) zCompute MD5 hash of filerbc&jdS)Ni)r)rBsr z/FileMemory._compute_file_hash..s !&&,r!r!NzError computing hash for rru) hashlibmd5r<iterro hexdigestrtr'ru)rrhash_md5rrxrBs @r rzFileMemory._compute_file_hashs;;= i&!!"6**9h? ..v6I MM  + r! file_pathsct|}|jDcgc]!}|jjd|vr|#c}|_|D]}|jj |d ycc}w)z Remove chunks from deleted filesrN)rr0rrrrpop)rrfile_paths_setr@rs r rz#FileMemory._remove_files_from_indexsmZ"]] *T}}  -^C ]  $I    " "9d 3$  s&A,)r"N)r2)!__name__ __module__ __qualname____doc__r rr-r+rrintfloatrr rkr ror)r,boolr*r r{rrrr|rr~rrr __classcell__)rs@r rrs] ;/;(&W2AA%A A $ A j  AF 0 T 6E(,D,\$"2$tCy$L.T.tCy.T. 5c5S5tTXz5n-(-(S -(^'T 'tJ7G'R C C ,S,C,d, 449 4 4r!r)rr=r$rloggingpathlibrtypingrrrrrVnumpyrR%runtime.node.agent.memory.memory_baserrr r entity.configsr r entity.configs.node.memoryr getLoggerrr'rr!r rs\ ""  ?7   8 $J4J4r!