From a43cb1676a939512a2140860a34aa4dd60980d68 Mon Sep 17 00:00:00 2001 From: "yuan.wang" Date: Mon, 27 Oct 2025 11:36:40 +0800 Subject: [PATCH 01/21] add hybrid search and fine extractor --- .../textual/prefer_text_memory/adder.py | 163 ++++++++++++------ .../textual/prefer_text_memory/retrievers.py | 4 +- src/memos/templates/prefer_complete_prompt.py | 33 +++- src/memos/vec_dbs/milvus.py | 108 ++++++++++-- 4 files changed, 237 insertions(+), 71 deletions(-) diff --git a/src/memos/memories/textual/prefer_text_memory/adder.py b/src/memos/memories/textual/prefer_text_memory/adder.py index 390f048ef..41cf5e596 100644 --- a/src/memos/memories/textual/prefer_text_memory/adder.py +++ b/src/memos/memories/textual/prefer_text_memory/adder.py @@ -9,6 +9,7 @@ from memos.templates.prefer_complete_prompt import ( NAIVE_JUDGE_UPDATE_OR_ADD_PROMPT, NAIVE_JUDGE_UPDATE_OR_ADD_PROMPT_OP_TRACE, + NAIVE_JUDGE_UPDATE_OR_ADD_PROMPT_FINE, ) from memos.vec_dbs.item import MilvusVecDBItem @@ -56,15 +57,34 @@ def _judge_update_or_add_fast(self, old_msg: str, new_msg: str) -> bool: response = response.strip().replace("```json", "").replace("```", "").strip() result = json.loads(response) response = result.get("is_same", False) - return response if isinstance(response, bool) else response == "true" + return response if isinstance(response, bool) else response.lower() == "true" except Exception as e: logger.error(f"Error in judge_update_or_add: {e}") # Fallback to simple string comparison return old_msg == new_msg + def _judge_update_or_add_fine( + self, new_mem: str, retrieved_mems: str + ) -> dict[str, Any] | None: + if not retrieved_mems: + return None + prompt = NAIVE_JUDGE_UPDATE_OR_ADD_PROMPT_FINE.replace("{new_memory}", new_mem).replace( + "{retrieved_memories}", retrieved_mems + ) + try: + response = self.llm_provider.generate([{"role": "user", "content": prompt}]) + response = response.strip().replace("```json", "").replace("```", "").strip() + result = json.loads(response) + return result + except Exception as e: + logger.error(f"Error in judge_update_or_add_fine: {e}") + return None + def _judge_update_or_add_trace_op( self, new_mem: str, retrieved_mems: str ) -> dict[str, Any] | None: + if not retrieved_mems: + return None prompt = NAIVE_JUDGE_UPDATE_OR_ADD_PROMPT_OP_TRACE.replace("{new_memory}", new_mem).replace( "{retrieved_memories}", retrieved_mems ) @@ -84,18 +104,15 @@ def _update_memory_op_trace( collection_name: str, preference_type: str, ) -> list[str] | str: - if not retrieved_memories: - payload = new_memory.to_dict()["metadata"] - fields_to_remove = {"dialog_id", "dialog_str", "embedding"} - payload = {k: v for k, v in payload.items() if k not in fields_to_remove} - vec_db_item = MilvusVecDBItem( - id=new_memory.id, - memory=new_memory.memory, - vector=new_memory.metadata.embedding, - payload=payload, - ) - self.vector_db.add(collection_name, [vec_db_item]) - return new_memory.id + payload = new_memory.to_dict()["metadata"] + fields_to_remove = {"dialog_id", "dialog_str", "embedding"} + payload = {k: v for k, v in payload.items() if k not in fields_to_remove} + new_vec_db_item = MilvusVecDBItem( + id=new_memory.id, + memory=new_memory.memory, + vector=new_memory.metadata.embedding, + payload=payload, + ) new_mem_input = { "context_summary": new_memory.memory, @@ -113,57 +130,36 @@ def _update_memory_op_trace( ] rsp = self._judge_update_or_add_trace_op( - new_mem=json.dumps(new_mem_input), retrieved_mems=json.dumps(retrieved_mem_inputs) + new_mem=json.dumps(new_mem_input), + retrieved_mems=json.dumps(retrieved_mem_inputs) if retrieved_mem_inputs else "", ) if not rsp: - payload = new_memory.to_dict()["metadata"] - fields_to_remove = {"dialog_id", "dialog_str", "embedding"} - payload = {k: v for k, v in payload.items() if k not in fields_to_remove} - vec_db_item = MilvusVecDBItem( - id=new_memory.id, - memory=new_memory.memory, - vector=new_memory.metadata.embedding, - payload=payload, - ) - self.vector_db.add(collection_name, [vec_db_item]) + self.vector_db.add(collection_name, [new_vec_db_item]) return new_memory.id - def execute_op(op): + def execute_op(op, new_mem_vec_db_item: MilvusVecDBItem, + retrieved_memories: list[MilvusVecDBItem]) -> str | None: op_type = op["type"].lower() if op_type == "add": - payload = new_memory.to_dict()["metadata"] - payload = { - k: v - for k, v in payload.items() - if k not in {"dialog_id", "dialog_str", "embedding"} - } - vec_db_item = MilvusVecDBItem( - id=new_memory.id, - memory=new_memory.memory, - vector=new_memory.metadata.embedding, - payload=payload, - ) - self.vector_db.add(collection_name, [vec_db_item]) + self.vector_db.add(collection_name, [new_mem_vec_db_item]) return new_memory.id elif op_type == "update": - payload = { - "preference_type": preference_type, - preference_type: op["new_preference"], - } - vec_db_item = MilvusVecDBItem( - id=op["target_id"], - memory=op["new_context_summary"], - vector=self.embedder.embed([op["new_context_summary"]])[0], - payload=payload, - ) - self.vector_db.update(collection_name, op["target_id"], vec_db_item) + update_item = [mem for mem in retrieved_memories if mem.id == op["target_id"]] + if not update_item: + self.vector_db.add(collection_name, [new_mem_vec_db_item]) + return new_memory.id + update_vec_db_item = update_item[0] + update_vec_db_item.payload[preference_type] = op["new_preference"] + update_vec_db_item.memory = op["new_context_summary"] + update_vec_db_item.vector = self.embedder.embed([op["new_context_summary"]])[0] + self.vector_db.update(collection_name, op["target_id"], update_vec_db_item) return op["target_id"] elif op_type == "delete": self.vector_db.delete(collection_name, [op["target_id"]]) return None with ThreadPoolExecutor(max_workers=min(len(rsp["trace"]), 5)) as executor: - future_to_op = {executor.submit(execute_op, op): op for op in rsp["trace"]} + future_to_op = {executor.submit(execute_op, op, new_vec_db_item, retrieved_memories): op for op in rsp["trace"]} added_ids = [] for future in as_completed(future_to_op): result = future.result() @@ -172,6 +168,55 @@ def execute_op(op): return added_ids + def _update_memory_fine( + self, + new_memory: TextualMemoryItem, + retrieved_memories: list[MilvusVecDBItem], + collection_name: str, + preference_type: str, + ) -> str: + payload = new_memory.to_dict()["metadata"] + fields_to_remove = {"dialog_id", "dialog_str", "embedding"} + payload = {k: v for k, v in payload.items() if k not in fields_to_remove} + vec_db_item = MilvusVecDBItem( + id=new_memory.id, + memory=new_memory.memory, + vector=new_memory.metadata.embedding, + payload=payload, + ) + + new_mem_input = { + "memory": new_memory.memory, + "preference": new_memory.metadata.explicit_preference + if preference_type == "explicit_preference" + else new_memory.metadata.implicit_preference, + } + retrieved_mem_inputs = [ + { + "id": mem.id, + "memory": mem.memory, + "preference": mem.payload[preference_type], + } + for mem in retrieved_memories + ] + rsp = self._judge_update_or_add_fine( + new_mem=json.dumps(new_mem_input), + retrieved_mems=json.dumps(retrieved_mem_inputs) if retrieved_mem_inputs else "", + ) + need_update = rsp.get("need_update", False) if rsp else False + need_update = need_update if isinstance(need_update, bool) else need_update.lower() == "true" + if need_update: + payload[preference_type] = rsp["new_preference"] + vec_db_item.id = rsp["id"] + vec_db_item.memory = rsp["new_memory"] + vec_db_item.vector = self.embedder.embed([rsp["new_memory"]])[0] + + self.vector_db.update(collection_name, rsp["id"], vec_db_item) + return rsp["id"] + else: + self.vector_db.add(collection_name, [vec_db_item]) + return new_memory.id + def _update_memory_fast( self, new_memory: TextualMemoryItem, @@ -196,8 +241,9 @@ def _update_memory_fast( new_msg_str = new_memory.memory is_same = self._judge_update_or_add_fast(old_msg=old_msg_str, new_msg=new_msg_str) if is_same: - self.vector_db.delete(collection_name, [recall.id]) - self.vector_db.update(collection_name, new_memory.id, vec_db_item) + vec_db_item.id = recall.id + self.vector_db.update(collection_name, recall.id, vec_db_item) + self.vector_db.add(collection_name, [vec_db_item]) return new_memory.id def _update_memory( @@ -206,7 +252,7 @@ def _update_memory( retrieved_memories: list[MilvusVecDBItem], collection_name: str, preference_type: str, - update_mode: str = "op_trace", + update_mode: str = "fine", ) -> list[str] | str | None: """Update the memory. Args: @@ -214,7 +260,7 @@ def _update_memory( retrieved_memories: list[MilvusVecDBItem] collection_name: str preference_type: str - update_mode: str, "op_trace" or "fast" + update_mode: str, "op_trace", "fast" or "fine" """ if update_mode == "op_trace": return self._update_memory_op_trace( @@ -222,6 +268,8 @@ def _update_memory( ) elif update_mode == "fast": return self._update_memory_fast(new_memory, retrieved_memories, collection_name) + elif update_mode == "fine": + return self._update_memory_fine(new_memory, retrieved_memories, collection_name, preference_type) else: raise ValueError(f"Invalid update mode: {update_mode}") @@ -236,15 +284,16 @@ def _process_single_memory(self, memory: TextualMemoryItem) -> list[str] | str | collection_name = pref_type_collection_map[preference_type] search_results = self.vector_db.search( - memory.metadata.embedding, - collection_name, + query_vector=memory.metadata.embedding, + query=memory.memory, + collection_name=collection_name, top_k=5, filter={"user_id": memory.metadata.user_id}, ) search_results.sort(key=lambda x: x.score, reverse=True) return self._update_memory( - memory, search_results, collection_name, preference_type, update_mode="fast" + memory, search_results, collection_name, preference_type, update_mode="fine" ) except Exception as e: diff --git a/src/memos/memories/textual/prefer_text_memory/retrievers.py b/src/memos/memories/textual/prefer_text_memory/retrievers.py index 7f70bac3b..15463db6a 100644 --- a/src/memos/memories/textual/prefer_text_memory/retrievers.py +++ b/src/memos/memories/textual/prefer_text_memory/retrievers.py @@ -45,10 +45,10 @@ def retrieve( with ThreadPoolExecutor(max_workers=2) as executor: # Submit all search tasks future_explicit = executor.submit( - self.vector_db.search, query_embedding, "explicit_preference", top_k * 2, info + self.vector_db.search, query_embedding, query, "explicit_preference", top_k * 2, info ) future_implicit = executor.submit( - self.vector_db.search, query_embedding, "implicit_preference", top_k * 2, info + self.vector_db.search, query_embedding, query, "implicit_preference", top_k * 2, info ) # Wait for all results diff --git a/src/memos/templates/prefer_complete_prompt.py b/src/memos/templates/prefer_complete_prompt.py index d40b7b778..9062f535c 100644 --- a/src/memos/templates/prefer_complete_prompt.py +++ b/src/memos/templates/prefer_complete_prompt.py @@ -9,9 +9,9 @@ - When the user modifies or updates their preferences for the same topic or event, extract the complete evolution process of their preference changes, including both the original and updated preferences. Requirements: -1. Keep only the preferences explicitly mentioned by the user. Do not infer or assume. +1. Keep only the preferences explicitly mentioned by the user. Do not infer or assume. If the user mentions reasons for their preferences, include those reasons as well. 2. Output should be a list of concise natural language summaries and the corresponding context summary, context summary must contain complete information of the conversation fragment that the preference is mentioned. -3. If multiple preferences are mentioned within the same topic, you need to merge the preferences and context summary. +3. If multiple preferences are mentioned within the same topic, you need to merge the preferences and context summary. Avoid repetition and redundancy in the merged content. Conversation: {qa_pair} @@ -80,6 +80,35 @@ {new_information} """ +NAIVE_JUDGE_UPDATE_OR_ADD_PROMPT_FINE = """ +You are a preference memory comparison expert. Analyze if the new preference memory describes the same topic as any retrieved memories. At most one retrieved memory can match the new memory. + +**Task:** Compare the new preference memory with retrieved memories to determine if they discuss the same topic and whether an update is needed. + +**Criteria:** +- Same core topic = need to check if update is needed +- Different topics = no update needed +- If same topic but content has changed/evolved = update needed +- If same topic and content is identical = no update needed + +**Output JSON:** +```json +{ + "need_update": true/false, + "id": "ID of the memory being updated (empty string if no update needed)", + "new_memory": "Updated memory field with change description (empty string if no update needed)", + "new_preference": "Updated preference field (empty string if no update needed)", + "reasoning": "Brief explanation of the comparison" +} +``` + +**New preference memory:** +{new_memory} + +**Retrieved preference memories:** +{retrieved_memories} +""" + NAIVE_JUDGE_UPDATE_OR_ADD_PROMPT_OP_TRACE = """ # User Preference Memory Management Agent diff --git a/src/memos/vec_dbs/milvus.py b/src/memos/vec_dbs/milvus.py index fb19fd6ff..c0d1aa1b3 100644 --- a/src/memos/vec_dbs/milvus.py +++ b/src/memos/vec_dbs/milvus.py @@ -34,18 +34,28 @@ def __init__(self, config: MilvusVecDBConfig): def create_schema(self): """Create schema for the milvus collection.""" - from pymilvus import DataType + from pymilvus import DataType, Function, FunctionType schema = self.client.create_schema(auto_id=False, enable_dynamic_field=True) schema.add_field( field_name="id", datatype=DataType.VARCHAR, max_length=65535, is_primary=True ) - schema.add_field(field_name="memory", datatype=DataType.VARCHAR, max_length=65535) + analyzer_params = {"tokenizer": "standard", "filter": ["lowercase"]} + schema.add_field(field_name="memory", datatype=DataType.VARCHAR, max_length=65535, analyzer_params=analyzer_params, enable_match=True, enable_analyzer=True) schema.add_field( field_name="vector", datatype=DataType.FLOAT_VECTOR, dim=self.config.vector_dimension ) schema.add_field(field_name="payload", datatype=DataType.JSON) + schema.add_field(field_name="sparse_vector", datatype=DataType.SPARSE_FLOAT_VECTOR) + bm25_function = Function( + name="bm25", + function_type=FunctionType.BM25, + input_field_names=["memory"], + output_field_names="sparse_vector", + ) + schema.add_function(bm25_function) + return schema def create_index(self): @@ -54,6 +64,11 @@ def create_index(self): index_params.add_index( field_name="vector", index_type="FLAT", metric_type=self._get_metric_type() ) + index_params.add_index( + field_name="sparse_vector", + index_type="SPARSE_INVERTED_INDEX", + metric_type="BM25", + ) return index_params @@ -102,12 +117,85 @@ def collection_exists(self, name: str) -> bool: """Check if a collection exists.""" return self.client.has_collection(collection_name=name) + def _dense_search( + self, + collection_name: str, + query_vector: list[float], + top_k: int, + filter: str = "", + **kwargs: Any, + ) -> list[list[dict]]: + """Dense search for similar items in the database.""" + results = self.client.search( + collection_name=collection_name, + data=[query_vector], + limit=top_k, + filter=filter, + output_fields=["*"], + anns_field="vector", + ) + return results + + def _sparse_search( + self, + collection_name: str, + query: str, + top_k: int, + filter: str = "", + **kwargs: Any, + ) -> list[list[dict]]: + """Sparse search for similar items in the database.""" + results = self.client.search( + collection_name=collection_name, + data=[query], + limit=top_k, + filter=filter, + output_fields=["*"], + anns_field="sparse_vector", + ) + return results + + def _hybrid_search( + self, + collection_name: str, + query_vector: list[float], + query: str, + top_k: int, + filter: str | None = None, + ranker_type: str = "rrf", # rrf, weighted + sparse_weight=1.0, + dense_weight=1.0, + **kwargs: Any, + ) -> list[list[dict]]: + """Hybrid search for similar items in the database.""" + from pymilvus import AnnSearchRequest, RRFRanker, WeightedRanker + # Set up BM25 search request + expr = filter if filter else None + sparse_request = AnnSearchRequest( + data=[query], anns_field="sparse_vector", param={"metric_type": "BM25"}, limit=top_k, expr=expr + ) + # Set up dense vector search request + dense_request = AnnSearchRequest( + data=[query_vector], anns_field="vector", param={"metric_type": self._get_metric_type()}, limit=top_k, expr=expr + ) + ranker = RRFRanker() if ranker_type == "rrf" else WeightedRanker(sparse_weight, dense_weight) + results = self.client.hybrid_search( + collection_name=collection_name, + reqs=[sparse_request, dense_request], + ranker=ranker, + limit=top_k, + output_fields=["*"], + ) + return results + def search( self, query_vector: list[float], + query: str, collection_name: str, top_k: int, filter: dict[str, Any] | None = None, + search_type: str = "dense", # dense, sparse, hybrid ) -> list[MilvusVecDBItem]: """ Search for similar items in the database. @@ -124,13 +212,13 @@ def search( # Convert filter to Milvus expression expr = self._dict_to_expr(filter) if filter else "" - results = self.client.search( - collection_name=collection_name, - data=[query_vector], - limit=top_k, - filter=expr, - output_fields=["*"], # Return all fields - ) + search_func_map = { + "dense": self._dense_search, + "sparse": self._sparse_search, + "hybrid": self._hybrid_search, + } + + results = search_func_map[search_type](collection_name=collection_name, query_vector=query_vector, query=query, top_k=top_k, filter=expr) items = [] for hit in results[0]: @@ -138,7 +226,7 @@ def search( items.append( MilvusVecDBItem( - id=str(hit["id"]), + id=str(entity.get("id")), memory=entity.get("memory"), vector=entity.get("vector"), payload=entity.get("payload", {}), From 9d6cda91d251a698a93cc74f9ca5a3f4e3882445 Mon Sep 17 00:00:00 2001 From: "yuan.wang" Date: Mon, 27 Oct 2025 20:02:08 +0800 Subject: [PATCH 02/21] add dialog and modify spliter chunk --- .../textual/prefer_text_memory/adder.py | 28 ++++++++++++------- .../textual/prefer_text_memory/spliter.py | 3 +- src/memos/vec_dbs/item.py | 1 + src/memos/vec_dbs/milvus.py | 6 ++++ 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/src/memos/memories/textual/prefer_text_memory/adder.py b/src/memos/memories/textual/prefer_text_memory/adder.py index 41cf5e596..447ac46a3 100644 --- a/src/memos/memories/textual/prefer_text_memory/adder.py +++ b/src/memos/memories/textual/prefer_text_memory/adder.py @@ -110,6 +110,7 @@ def _update_memory_op_trace( new_vec_db_item = MilvusVecDBItem( id=new_memory.id, memory=new_memory.memory, + original_text=new_memory.metadata.dialog_str, vector=new_memory.metadata.embedding, payload=payload, ) @@ -142,15 +143,17 @@ def execute_op(op, new_mem_vec_db_item: MilvusVecDBItem, op_type = op["type"].lower() if op_type == "add": self.vector_db.add(collection_name, [new_mem_vec_db_item]) - return new_memory.id + return new_mem_vec_db_item.id elif op_type == "update": update_item = [mem for mem in retrieved_memories if mem.id == op["target_id"]] if not update_item: self.vector_db.add(collection_name, [new_mem_vec_db_item]) - return new_memory.id + return new_mem_vec_db_item.id update_vec_db_item = update_item[0] update_vec_db_item.payload[preference_type] = op["new_preference"] + update_vec_db_item.payload["updated_at"] = new_mem_vec_db_item.payload["updated_at"] update_vec_db_item.memory = op["new_context_summary"] + update_vec_db_item.original_text = new_mem_vec_db_item.original_text update_vec_db_item.vector = self.embedder.embed([op["new_context_summary"]])[0] self.vector_db.update(collection_name, op["target_id"], update_vec_db_item) return op["target_id"] @@ -181,6 +184,7 @@ def _update_memory_fine( vec_db_item = MilvusVecDBItem( id=new_memory.id, memory=new_memory.memory, + original_text=new_memory.metadata.dialog_str, vector=new_memory.metadata.embedding, payload=payload, ) @@ -205,17 +209,20 @@ def _update_memory_fine( ) need_update = rsp.get("need_update", False) if rsp else False need_update = need_update if isinstance(need_update, bool) else need_update.lower() == "true" - if need_update: - payload[preference_type] = rsp["new_preference"] - vec_db_item.id = rsp["id"] - vec_db_item.memory = rsp["new_memory"] - vec_db_item.vector = self.embedder.embed([rsp["new_memory"]])[0] - - self.vector_db.update(collection_name, rsp["id"], vec_db_item) + update_item = [mem for mem in retrieved_memories if mem.id == rsp["id"]] + if need_update and update_item: + update_vec_db_item = update_item[0] + update_vec_db_item.payload[preference_type] = rsp["new_preference"] + update_vec_db_item.payload["updated_at"] = vec_db_item.payload["updated_at"] + update_vec_db_item.memory = rsp["new_memory"] + update_vec_db_item.original_text = vec_db_item.original_text + update_vec_db_item.vector = self.embedder.embed([rsp["new_memory"]])[0] + + self.vector_db.update(collection_name, rsp["id"], update_vec_db_item) return rsp["id"] else: self.vector_db.add(collection_name, [vec_db_item]) - return new_memory.id + return vec_db_item.id def _update_memory_fast( self, @@ -229,6 +236,7 @@ def _update_memory_fast( vec_db_item = MilvusVecDBItem( id=new_memory.id, memory=new_memory.memory, + original_text=new_memory.metadata.dialog_str, vector=new_memory.metadata.embedding, payload=payload, ) diff --git a/src/memos/memories/textual/prefer_text_memory/spliter.py b/src/memos/memories/textual/prefer_text_memory/spliter.py index 59a6b0052..5d9911e79 100644 --- a/src/memos/memories/textual/prefer_text_memory/spliter.py +++ b/src/memos/memories/textual/prefer_text_memory/spliter.py @@ -87,7 +87,8 @@ def _split_with_overlap(self, data: MessageList) -> list[MessageList]: # overlap 1 turns (Q + A = 2) context = copy.deepcopy(chunk[-2:]) chunk = context - if chunk: + # avoid duplicate qa, and cardinal number + if chunk and len(chunk) > 3: chunks.append(chunk) return chunks diff --git a/src/memos/vec_dbs/item.py b/src/memos/vec_dbs/item.py index 081400f15..c6aa1c9c2 100644 --- a/src/memos/vec_dbs/item.py +++ b/src/memos/vec_dbs/item.py @@ -47,3 +47,4 @@ class MilvusVecDBItem(VecDBItem): """Represents a single item in the Milvus vector database.""" memory: str | None = Field(default=None, description="Memory string") + original_text: str | None = Field(default=None, description="Original text content") diff --git a/src/memos/vec_dbs/milvus.py b/src/memos/vec_dbs/milvus.py index c0d1aa1b3..a22490404 100644 --- a/src/memos/vec_dbs/milvus.py +++ b/src/memos/vec_dbs/milvus.py @@ -42,6 +42,7 @@ def create_schema(self): ) analyzer_params = {"tokenizer": "standard", "filter": ["lowercase"]} schema.add_field(field_name="memory", datatype=DataType.VARCHAR, max_length=65535, analyzer_params=analyzer_params, enable_match=True, enable_analyzer=True) + schema.add_field(field_name="original_text", datatype=DataType.VARCHAR, max_length=65535) schema.add_field( field_name="vector", datatype=DataType.FLOAT_VECTOR, dim=self.config.vector_dimension ) @@ -228,6 +229,7 @@ def search( MilvusVecDBItem( id=str(entity.get("id")), memory=entity.get("memory"), + original_text=entity.get("original_text"), vector=entity.get("vector"), payload=entity.get("payload", {}), score=1 - float(hit["distance"]), @@ -284,6 +286,7 @@ def get_by_id(self, collection_name: str, id: str) -> MilvusVecDBItem | None: return MilvusVecDBItem( id=entity["id"], memory=entity.get("memory"), + original_text=entity.get("original_text"), vector=entity.get("vector"), payload=payload, ) @@ -305,6 +308,7 @@ def get_by_ids(self, collection_name: str, ids: list[str]) -> list[MilvusVecDBIt MilvusVecDBItem( id=entity["id"], memory=entity.get("memory"), + original_text=entity.get("original_text"), vector=entity.get("vector"), payload=payload, ) @@ -352,6 +356,7 @@ def get_by_filter( MilvusVecDBItem( id=entity["id"], memory=entity.get("memory"), + original_text=entity.get("original_text"), vector=entity.get("vector"), payload=payload, ) @@ -409,6 +414,7 @@ def add(self, collection_name: str, data: list[MilvusVecDBItem | dict[str, Any]] entity = { "id": item.id, "memory": item.memory, + "original_text": item.original_text, "vector": item.vector, "payload": item.payload if item.payload else {}, } From 573199e5ea0e64911572a01cc253d7cbefd32b17 Mon Sep 17 00:00:00 2001 From: "yuan.wang" Date: Tue, 28 Oct 2025 16:11:27 +0800 Subject: [PATCH 03/21] optmize the update and retriever code --- src/memos/memories/textual/item.py | 2 +- .../textual/prefer_text_memory/adder.py | 12 +++---- .../textual/prefer_text_memory/extractor.py | 10 +++--- .../textual/prefer_text_memory/retrievers.py | 36 ++++++++++++++----- .../textual/prefer_text_memory/spliter.py | 10 +++--- src/memos/templates/prefer_complete_prompt.py | 30 ++++++++++------ 6 files changed, 65 insertions(+), 35 deletions(-) diff --git a/src/memos/memories/textual/item.py b/src/memos/memories/textual/item.py index 6d975cfd7..0c36c94c8 100644 --- a/src/memos/memories/textual/item.py +++ b/src/memos/memories/textual/item.py @@ -174,7 +174,7 @@ class PreferenceTextualMemoryMetadata(TextualMemoryMetadata): default="explicit_preference", description="Type of preference." ) dialog_id: str | None = Field(default=None, description="ID of the dialog.") - dialog_str: str | None = Field(default=None, description="String of the dialog.") + original_text: str | None = Field(default=None, description="String of the dialog.") embedding: list[float] | None = Field(default=None, description="Vector of the dialog.") explicit_preference: str | None = Field(default=None, description="Explicit preference.") created_at: str | None = Field(default=None, description="Timestamp of the dialog.") diff --git a/src/memos/memories/textual/prefer_text_memory/adder.py b/src/memos/memories/textual/prefer_text_memory/adder.py index 447ac46a3..e518f3299 100644 --- a/src/memos/memories/textual/prefer_text_memory/adder.py +++ b/src/memos/memories/textual/prefer_text_memory/adder.py @@ -105,12 +105,12 @@ def _update_memory_op_trace( preference_type: str, ) -> list[str] | str: payload = new_memory.to_dict()["metadata"] - fields_to_remove = {"dialog_id", "dialog_str", "embedding"} + fields_to_remove = {"dialog_id", "original_text", "embedding"} payload = {k: v for k, v in payload.items() if k not in fields_to_remove} new_vec_db_item = MilvusVecDBItem( id=new_memory.id, memory=new_memory.memory, - original_text=new_memory.metadata.dialog_str, + original_text=new_memory.metadata.original_text, vector=new_memory.metadata.embedding, payload=payload, ) @@ -179,12 +179,12 @@ def _update_memory_fine( preference_type: str, ) -> str: payload = new_memory.to_dict()["metadata"] - fields_to_remove = {"dialog_id", "dialog_str", "embedding"} + fields_to_remove = {"dialog_id", "original_text", "embedding"} payload = {k: v for k, v in payload.items() if k not in fields_to_remove} vec_db_item = MilvusVecDBItem( id=new_memory.id, memory=new_memory.memory, - original_text=new_memory.metadata.dialog_str, + original_text=new_memory.metadata.original_text, vector=new_memory.metadata.embedding, payload=payload, ) @@ -231,12 +231,12 @@ def _update_memory_fast( collection_name: str, ) -> str: payload = new_memory.to_dict()["metadata"] - fields_to_remove = {"dialog_id", "dialog_str", "embedding"} + fields_to_remove = {"dialog_id", "original_text", "embedding"} payload = {k: v for k, v in payload.items() if k not in fields_to_remove} vec_db_item = MilvusVecDBItem( id=new_memory.id, memory=new_memory.memory, - original_text=new_memory.metadata.dialog_str, + original_text=new_memory.metadata.original_text, vector=new_memory.metadata.embedding, payload=payload, ) diff --git a/src/memos/memories/textual/prefer_text_memory/extractor.py b/src/memos/memories/textual/prefer_text_memory/extractor.py index 460b31f4f..8bfbfbc94 100644 --- a/src/memos/memories/textual/prefer_text_memory/extractor.py +++ b/src/memos/memories/textual/prefer_text_memory/extractor.py @@ -43,7 +43,7 @@ def extract_basic_info(self, qa_pair: MessageList) -> dict[str, Any]: """Extract basic information from a QA pair (no LLM needed).""" basic_info = { "dialog_id": str(uuid.uuid4()), - "dialog_str": convert_messages_to_string(qa_pair), + "original_text": convert_messages_to_string(qa_pair), "created_at": datetime.now().isoformat(), } @@ -84,10 +84,10 @@ def _process_single_chunk_explicit( ) -> TextualMemoryItem | None: """Process a single chunk and return a TextualMemoryItem.""" basic_info = self.extract_basic_info(chunk) - if not basic_info["dialog_str"]: + if not basic_info["original_text"]: return None - explicit_pref = self.extract_explicit_preference(basic_info["dialog_str"]) + explicit_pref = self.extract_explicit_preference(basic_info["original_text"]) if not explicit_pref: return None @@ -113,9 +113,9 @@ def _process_single_chunk_implicit( self, chunk: MessageList, msg_type: str, info: dict[str, Any] ) -> TextualMemoryItem | None: basic_info = self.extract_basic_info(chunk) - if not basic_info["dialog_str"]: + if not basic_info["original_text"]: return None - implicit_pref = self.extract_implicit_preference(basic_info["dialog_str"]) + implicit_pref = self.extract_implicit_preference(basic_info["original_text"]) if not implicit_pref: return None diff --git a/src/memos/memories/textual/prefer_text_memory/retrievers.py b/src/memos/memories/textual/prefer_text_memory/retrievers.py index 15463db6a..b5db89a8b 100644 --- a/src/memos/memories/textual/prefer_text_memory/retrievers.py +++ b/src/memos/memories/textual/prefer_text_memory/retrievers.py @@ -3,6 +3,7 @@ from typing import Any from memos.memories.textual.item import PreferenceTextualMemoryMetadata, TextualMemoryItem +from memos.vec_dbs.item import MilvusVecDBItem class BaseRetriever(ABC): @@ -29,6 +30,26 @@ def __init__(self, llm_provider=None, embedder=None, reranker=None, vector_db=No self.vector_db = vector_db self.embedder = embedder + def _naive_reranker(self, query: str, prefs_mem: list[TextualMemoryItem], top_k: int, **kwargs: Any) -> list[TextualMemoryItem]: + if self.reranker: + prefs_mem = self.reranker.rerank(query, prefs_mem, top_k) + return [item for item, _ in prefs_mem] + return prefs_mem + + def _original_text_reranker(self, query: str, prefs_mem: list[TextualMemoryItem], prefs: list[MilvusVecDBItem], top_k: int, **kwargs: Any) -> list[TextualMemoryItem]: + if self.reranker: + from copy import deepcopy + prefs_mem_for_reranker = deepcopy(prefs_mem) + for pref_mem, pref in zip(prefs_mem_for_reranker, prefs): + pref_mem.memory = pref_mem.memory + "\n" + pref.original_text + prefs_mem_for_reranker = self.reranker.rerank(query, prefs_mem_for_reranker, top_k) + prefs_mem_for_reranker = [item for item, _ in prefs_mem_for_reranker] + prefs_ids = [item.id for item in prefs_mem_for_reranker] + prefs_dict = {item.id: item for item in prefs_mem} + return [prefs_dict[item_id] for item_id in prefs_ids if item_id in prefs_dict] + return prefs_mem + + def retrieve( self, query: str, top_k: int, info: dict[str, Any] | None = None ) -> list[TextualMemoryItem]: @@ -59,7 +80,7 @@ def retrieve( explicit_prefs.sort(key=lambda x: x.score, reverse=True) implicit_prefs.sort(key=lambda x: x.score, reverse=True) - explicit_prefs = [ + explicit_prefs_mem = [ TextualMemoryItem( id=pref.id, memory=pref.memory, @@ -69,7 +90,7 @@ def retrieve( if pref.payload["explicit_preference"] ] - implicit_prefs = [ + implicit_prefs_mem = [ TextualMemoryItem( id=pref.id, memory=pref.memory, @@ -79,10 +100,9 @@ def retrieve( if pref.payload["implicit_preference"] ] - if self.reranker: - explicit_prefs = self.reranker.rerank(query, explicit_prefs, top_k) - implicit_prefs = self.reranker.rerank(query, implicit_prefs, top_k) - explicit_prefs = [item for item, _ in explicit_prefs] - implicit_prefs = [item for item, _ in implicit_prefs] + reranker_map = {"naive": self._naive_reranker, "original_text": self._original_text_reranker} + reranker_func = reranker_map["naive"] + explicit_prefs_mem = reranker_func(query=query, prefs_mem=explicit_prefs_mem, prefs=explicit_prefs, top_k=top_k) + implicit_prefs_mem = reranker_func(query=query, prefs_mem=implicit_prefs_mem, prefs=implicit_prefs, top_k=top_k) - return explicit_prefs + implicit_prefs + return explicit_prefs_mem + implicit_prefs_mem diff --git a/src/memos/memories/textual/prefer_text_memory/spliter.py b/src/memos/memories/textual/prefer_text_memory/spliter.py index 5d9911e79..eafd08b03 100644 --- a/src/memos/memories/textual/prefer_text_memory/spliter.py +++ b/src/memos/memories/textual/prefer_text_memory/spliter.py @@ -79,16 +79,18 @@ def _split_with_overlap(self, data: MessageList) -> list[MessageList]: adjacent chunk with low duplicate rate""" chunks = [] chunk = [] - for item in data: + for i, item in enumerate(data): chunk.append(item) # 5 turns (Q + A = 10) each chunk if len(chunk) >= 10: chunks.append(chunk) # overlap 1 turns (Q + A = 2) - context = copy.deepcopy(chunk[-2:]) + if i + 1 < len(data): + context = copy.deepcopy(chunk[-2:]) + else: + context = [] chunk = context - # avoid duplicate qa, and cardinal number - if chunk and len(chunk) > 3: + if chunk and len(chunk) % 2 == 0: chunks.append(chunk) return chunks diff --git a/src/memos/templates/prefer_complete_prompt.py b/src/memos/templates/prefer_complete_prompt.py index 9062f535c..51b1f2eed 100644 --- a/src/memos/templates/prefer_complete_prompt.py +++ b/src/memos/templates/prefer_complete_prompt.py @@ -10,8 +10,8 @@ Requirements: 1. Keep only the preferences explicitly mentioned by the user. Do not infer or assume. If the user mentions reasons for their preferences, include those reasons as well. -2. Output should be a list of concise natural language summaries and the corresponding context summary, context summary must contain complete information of the conversation fragment that the preference is mentioned. -3. If multiple preferences are mentioned within the same topic, you need to merge the preferences and context summary. Avoid repetition and redundancy in the merged content. +2. Output should be a list of entries concise natural language summaries and the corresponding context summary, context summary must contain complete information of the conversation fragment that the preference is mentioned. +3. If multiple preferences are mentioned within the same topic or domain, you MUST combine them into a single entry, keep each entry information complete. Conversation: {qa_pair} @@ -81,24 +81,32 @@ """ NAIVE_JUDGE_UPDATE_OR_ADD_PROMPT_FINE = """ -You are a preference memory comparison expert. Analyze if the new preference memory describes the same topic as any retrieved memories. At most one retrieved memory can match the new memory. +You are a preference memory comparison expert. Analyze if the new preference memory describes the same topic as any retrieved memories by considering BOTH the memory field and preference field. At most one retrieved memory can match the new memory. **Task:** Compare the new preference memory with retrieved memories to determine if they discuss the same topic and whether an update is needed. -**Criteria:** -- Same core topic = need to check if update is needed -- Different topics = no update needed +**Comparison Criteria:** +- **Memory field**: Compare the core topics, scenarios, and contexts described +- **Preference field**: Compare the actual preference statements, choices, and attitudes expressed +- **Same topic**: Both memory AND preference content relate to the same subject matter +- **Different topics**: Either memory OR preference content differs significantly +- **Content evolution**: Same topic but preference has changed/evolved or memory has been updated +- **Identical content**: Both memory and preference fields are essentially the same + +**Decision Logic:** +- Same core topic (both memory and preference) = need to check if update is needed +- Different topics (either memory or preference differs) = no update needed - If same topic but content has changed/evolved = update needed -- If same topic and content is identical = no update needed +- If same topic and content is identical = update needed **Output JSON:** ```json { "need_update": true/false, "id": "ID of the memory being updated (empty string if no update needed)", - "new_memory": "Updated memory field with change description (empty string if no update needed)", - "new_preference": "Updated preference field (empty string if no update needed)", - "reasoning": "Brief explanation of the comparison" + "new_memory": "Updated memory field with merged/evolved memory content (empty string if no update needed)", + "new_preference": "Updated preference field with merged/evolved preference content (empty string if no update needed)", + "reasoning": "Brief explanation of the comparison considering both memory and preference fields" } ``` @@ -137,7 +145,7 @@ 1. Analyze each retrieved memory and determine its relationship to the new memory: - **Unrelated** → perform `"ADD"` (insert as a new independent memory); - - **Related** → perform `"UPDATE"` (refine, supplement, or merge both the `preference` and the `context_summary`); + - **Related** → perform `"UPDATE"` (refine, supplement, or merge both the `preference` and the `context_summary`, while preserving change history trajectory information); - **Conflicting or outdated** → perform `"DELETE"` (remove obsolete or contradictory memory). 2. If multiple retrieved memories describe the same preference theme, merge them into one updated memory entry, combining both their `preference` information and their `context_summary` in a coherent and concise way. From 1c350512ba852e2cd6699119f6cd0ad1207ebeee Mon Sep 17 00:00:00 2001 From: "yuan.wang" Date: Tue, 28 Oct 2025 20:42:13 +0800 Subject: [PATCH 04/21] modify pref field --- evaluation/scripts/PrefEval/pref_memos.py | 6 +++--- evaluation/scripts/locomo/locomo_search.py | 4 ++-- evaluation/scripts/longmemeval/lme_search.py | 2 +- evaluation/scripts/personamem/pm_ingestion.py | 4 ++-- evaluation/scripts/personamem/pm_search.py | 2 +- evaluation/scripts/utils/client.py | 1 + src/memos/api/routers/server_router.py | 12 +++++------- 7 files changed, 15 insertions(+), 16 deletions(-) diff --git a/evaluation/scripts/PrefEval/pref_memos.py b/evaluation/scripts/PrefEval/pref_memos.py index 7336d4612..fc358dc36 100644 --- a/evaluation/scripts/PrefEval/pref_memos.py +++ b/evaluation/scripts/PrefEval/pref_memos.py @@ -53,9 +53,9 @@ def add_memory_for_line( if os.getenv("PRE_SPLIT_CHUNK", "false").lower() == "true": for chunk_start in range(0, len(conversation), turns_add * 2): chunk = conversation[chunk_start : chunk_start + turns_add * 2] - mem_client.add(messages=chunk, user_id=user_id, conv_id=None) + mem_client.add(messages=chunk, user_id=user_id, conv_id=None, batch_size=2) else: - mem_client.add(messages=conversation, user_id=user_id, conv_id=None) + mem_client.add(messages=conversation, user_id=user_id, conv_id=None, batch_size=2) end_time_add = time.monotonic() add_duration = end_time_add - start_time_add @@ -98,7 +98,7 @@ def search_memory_for_line(line_data: tuple, mem_client, top_k_value: int) -> di f"- {entry.get('memory', '')}" for entry in relevant_memories["text_mem"][0]["memories"] ) - + f"\n{relevant_memories['pref_mem']}" + + f"\n{relevant_memories['pref_string']}" ) memory_tokens_used = len(tokenizer.encode(memories_str)) diff --git a/evaluation/scripts/locomo/locomo_search.py b/evaluation/scripts/locomo/locomo_search.py index 1ddf0d933..0b610d574 100644 --- a/evaluation/scripts/locomo/locomo_search.py +++ b/evaluation/scripts/locomo/locomo_search.py @@ -107,11 +107,11 @@ def memos_api_search( speaker_a_context = ( "\n".join([i["memory"] for i in search_a_results["text_mem"][0]["memories"]]) - + f"\n{search_a_results['pref_mem']}" + + f"\n{search_a_results['pref_string']}" ) speaker_b_context = ( "\n".join([i["memory"] for i in search_b_results["text_mem"][0]["memories"]]) - + f"\n{search_b_results['pref_mem']}" + + f"\n{search_b_results['pref_string']}" ) context = TEMPLATE_MEMOS.format( diff --git a/evaluation/scripts/longmemeval/lme_search.py b/evaluation/scripts/longmemeval/lme_search.py index 60b2146f6..89c02aaea 100644 --- a/evaluation/scripts/longmemeval/lme_search.py +++ b/evaluation/scripts/longmemeval/lme_search.py @@ -46,7 +46,7 @@ def memos_search(client, query, user_id, top_k): results = client.search(query=query, user_id=user_id, top_k=top_k) context = ( "\n".join([i["memory"] for i in results["text_mem"][0]["memories"]]) - + f"\n{results['pref_mem']}" + + f"\n{results['pref_string']}" ) context = MEMOS_CONTEXT_TEMPLATE.format(user_id=user_id, memories=context) duration_ms = (time() - start) * 1000 diff --git a/evaluation/scripts/personamem/pm_ingestion.py b/evaluation/scripts/personamem/pm_ingestion.py index 5204b5c2a..cab0fbeb5 100644 --- a/evaluation/scripts/personamem/pm_ingestion.py +++ b/evaluation/scripts/personamem/pm_ingestion.py @@ -31,10 +31,10 @@ def ingest_session(session, user_id, session_id, frame, client): if os.getenv("PRE_SPLIT_CHUNK") == "true": for i in range(0, len(session), 10): messages = session[i : i + 10] - client.add(messages=messages, user_id=user_id, conv_id=session_id) + client.add(messages=messages, user_id=user_id, conv_id=session_id, batch_size=2) print(f"[{frame}] ✅ Session [{session_id}]: Ingested {len(messages)} messages") else: - client.add(messages=session, user_id=user_id, conv_id=session_id) + client.add(messages=session, user_id=user_id, conv_id=session_id, batch_size=2) print(f"[{frame}] ✅ Session [{session_id}]: Ingested {len(session)} messages") elif frame == "memobase": for _idx, msg in enumerate(session): diff --git a/evaluation/scripts/personamem/pm_search.py b/evaluation/scripts/personamem/pm_search.py index c18e05623..441474c7c 100644 --- a/evaluation/scripts/personamem/pm_search.py +++ b/evaluation/scripts/personamem/pm_search.py @@ -84,7 +84,7 @@ def memos_search(client, user_id, query, top_k): results = client.search(query=query, user_id=user_id, top_k=top_k) search_memories = ( "\n".join(item["memory"] for cube in results["text_mem"] for item in cube["memories"]) - + f"\n{results['pref_mem']}" + + f"\n{results['pref_string']}" ) context = MEMOS_CONTEXT_TEMPLATE.format(user_id=user_id, memories=search_memories) diff --git a/evaluation/scripts/utils/client.py b/evaluation/scripts/utils/client.py index 4117cba56..33a1d6223 100644 --- a/evaluation/scripts/utils/client.py +++ b/evaluation/scripts/utils/client.py @@ -182,6 +182,7 @@ def search(self, query, user_id, top_k): "conversation_id": "", "top_k": top_k, "mode": "mixture", + "handle_pref_mem": True, }, ensure_ascii=False, ) diff --git a/src/memos/api/routers/server_router.py b/src/memos/api/routers/server_router.py index f50d3ad75..4c348ca69 100644 --- a/src/memos/api/routers/server_router.py +++ b/src/memos/api/routers/server_router.py @@ -303,18 +303,15 @@ def _post_process_pref_mem( mem_cube_id: str, handle_pref_mem: bool, ): - if os.getenv("RETURN_ORIGINAL_PREF_MEM", "false").lower() == "true" and pref_formatted_mem: - memories_result["prefs"] = [] - memories_result["prefs"].append( + if handle_pref_mem: + memories_result["pref_mem"].append( { "cube_id": mem_cube_id, "memories": pref_formatted_mem, } ) - - if handle_pref_mem: pref_instruction: str = instruct_completion(pref_formatted_mem) - memories_result["pref_mem"] = pref_instruction + memories_result["pref_string"] = pref_instruction return memories_result @@ -333,7 +330,8 @@ def search_memories(search_req: APISearchRequest): "text_mem": [], "act_mem": [], "para_mem": [], - "pref_mem": "", + "pref_mem": [], + "pref_string": "", } search_mode = search_req.mode From 9fdbb7d4f0ee54dec74449b81bedb35f3235eb3e Mon Sep 17 00:00:00 2001 From: "yuan.wang" Date: Wed, 29 Oct 2025 16:18:50 +0800 Subject: [PATCH 05/21] add pref mem update srategy --- .../textual/prefer_text_memory/adder.py | 169 ++++++--- .../textual/prefer_text_memory/extractor.py | 17 +- src/memos/templates/instruction_completion.py | 23 +- src/memos/templates/prefer_complete_prompt.py | 358 ++++++++++++++++-- 4 files changed, 480 insertions(+), 87 deletions(-) diff --git a/src/memos/memories/textual/prefer_text_memory/adder.py b/src/memos/memories/textual/prefer_text_memory/adder.py index e518f3299..e1e3a624b 100644 --- a/src/memos/memories/textual/prefer_text_memory/adder.py +++ b/src/memos/memories/textual/prefer_text_memory/adder.py @@ -4,6 +4,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from typing import Any +from datetime import datetime from memos.log import get_logger from memos.memories.textual.item import TextualMemoryItem from memos.templates.prefer_complete_prompt import ( @@ -81,11 +82,11 @@ def _judge_update_or_add_fine( return None def _judge_update_or_add_trace_op( - self, new_mem: str, retrieved_mems: str + self, new_mems: str, retrieved_mems: str ) -> dict[str, Any] | None: if not retrieved_mems: return None - prompt = NAIVE_JUDGE_UPDATE_OR_ADD_PROMPT_OP_TRACE.replace("{new_memory}", new_mem).replace( + prompt = NAIVE_JUDGE_UPDATE_OR_ADD_PROMPT_OP_TRACE.replace("{new_memories}", new_mems).replace( "{retrieved_memories}", retrieved_mems ) try: @@ -99,28 +100,34 @@ def _judge_update_or_add_trace_op( def _update_memory_op_trace( self, - new_memory: TextualMemoryItem, + new_memories: list[TextualMemoryItem], retrieved_memories: list[MilvusVecDBItem], collection_name: str, preference_type: str, ) -> list[str] | str: - payload = new_memory.to_dict()["metadata"] - fields_to_remove = {"dialog_id", "original_text", "embedding"} - payload = {k: v for k, v in payload.items() if k not in fields_to_remove} - new_vec_db_item = MilvusVecDBItem( - id=new_memory.id, - memory=new_memory.memory, - original_text=new_memory.metadata.original_text, - vector=new_memory.metadata.embedding, - payload=payload, - ) + # create new vec db items + new_vec_db_items: list[MilvusVecDBItem] = [] + for new_memory in new_memories: + payload = new_memory.to_dict()["metadata"] + fields_to_remove = {"dialog_id", "original_text", "embedding"} + payload = {k: v for k, v in payload.items() if k not in fields_to_remove} + new_vec_db_item = MilvusVecDBItem( + id=new_memory.id, + memory=new_memory.memory, + original_text=new_memory.metadata.original_text, + vector=new_memory.metadata.embedding, + payload=payload, + ) + new_vec_db_items.append(new_vec_db_item) - new_mem_input = { - "context_summary": new_memory.memory, - "preference": new_memory.metadata.explicit_preference - if preference_type == "explicit_preference" - else new_memory.metadata.implicit_preference, - } + new_mem_inputs = [ + { + "id": new_memory.id, + "context_summary": new_memory.memory, + "preference": new_memory.payload[preference_type], + } + for new_memory in new_vec_db_items + ] retrieved_mem_inputs = [ { "id": mem.id, @@ -131,38 +138,43 @@ def _update_memory_op_trace( ] rsp = self._judge_update_or_add_trace_op( - new_mem=json.dumps(new_mem_input), + new_mems=json.dumps(new_mem_inputs), retrieved_mems=json.dumps(retrieved_mem_inputs) if retrieved_mem_inputs else "", ) if not rsp: - self.vector_db.add(collection_name, [new_vec_db_item]) - return new_memory.id - - def execute_op(op, new_mem_vec_db_item: MilvusVecDBItem, - retrieved_memories: list[MilvusVecDBItem]) -> str | None: + with ThreadPoolExecutor(max_workers=min(len(new_vec_db_items), 5)) as executor: + futures = {executor.submit(self.vector_db.add, collection_name, [db_item]): db_item for db_item in new_vec_db_items} + for future in as_completed(futures): + result = future.result() + return [db_item.id for db_item in new_vec_db_items] + + new_mem_db_item_map = {db_item.id: db_item for db_item in new_vec_db_items} + retrieved_mem_db_item_map = {db_item.id: db_item for db_item in retrieved_memories} + def execute_op(op, new_mem_db_item_map: dict[str, MilvusVecDBItem], + retrieved_mem_db_item_map: dict[str, MilvusVecDBItem]) -> str | None: op_type = op["type"].lower() if op_type == "add": - self.vector_db.add(collection_name, [new_mem_vec_db_item]) - return new_mem_vec_db_item.id + if op["target_id"] in new_mem_db_item_map: + self.vector_db.add(collection_name, [new_mem_db_item_map[op["target_id"]]]) + return new_mem_db_item_map[op["target_id"]].id + return None elif op_type == "update": - update_item = [mem for mem in retrieved_memories if mem.id == op["target_id"]] - if not update_item: - self.vector_db.add(collection_name, [new_mem_vec_db_item]) - return new_mem_vec_db_item.id - update_vec_db_item = update_item[0] - update_vec_db_item.payload[preference_type] = op["new_preference"] - update_vec_db_item.payload["updated_at"] = new_mem_vec_db_item.payload["updated_at"] - update_vec_db_item.memory = op["new_context_summary"] - update_vec_db_item.original_text = new_mem_vec_db_item.original_text - update_vec_db_item.vector = self.embedder.embed([op["new_context_summary"]])[0] - self.vector_db.update(collection_name, op["target_id"], update_vec_db_item) - return op["target_id"] + if op["target_id"] in retrieved_mem_db_item_map: + update_mem_db_item = retrieved_mem_db_item_map[op["target_id"]] + update_mem_db_item.payload[preference_type] = op["new_preference"] + update_mem_db_item.payload["updated_at"] = datetime.now().isoformat() + update_mem_db_item.memory = op["new_context_summary"] + update_mem_db_item.original_text = op["new_context_summary"] + update_mem_db_item.vector = self.embedder.embed([op["new_context_summary"]])[0] + self.vector_db.update(collection_name, op["target_id"], update_mem_db_item) + return op["target_id"] + return None elif op_type == "delete": self.vector_db.delete(collection_name, [op["target_id"]]) return None with ThreadPoolExecutor(max_workers=min(len(rsp["trace"]), 5)) as executor: - future_to_op = {executor.submit(execute_op, op, new_vec_db_item, retrieved_memories): op for op in rsp["trace"]} + future_to_op = {executor.submit(execute_op, op, new_mem_db_item_map, retrieved_mem_db_item_map): op for op in rsp["trace"]} added_ids = [] for future in as_completed(future_to_op): result = future.result() @@ -268,13 +280,9 @@ def _update_memory( retrieved_memories: list[MilvusVecDBItem] collection_name: str preference_type: str - update_mode: str, "op_trace", "fast" or "fine" + update_mode: str, "fast" or "fine" """ - if update_mode == "op_trace": - return self._update_memory_op_trace( - new_memory, retrieved_memories, collection_name, preference_type - ) - elif update_mode == "fast": + if update_mode == "fast": return self._update_memory_fast(new_memory, retrieved_memories, collection_name) elif update_mode == "fine": return self._update_memory_fine(new_memory, retrieved_memories, collection_name, preference_type) @@ -308,18 +316,43 @@ def _process_single_memory(self, memory: TextualMemoryItem) -> list[str] | str | logger.error(f"Error processing memory {memory.id}: {e}") return None - def add( - self, - memories: list[TextualMemoryItem | dict[str, Any]], - max_workers: int = 8, - *args, - **kwargs, - ) -> list[str]: - """Add the instruct preference memories using thread pool for acceleration.""" - if not memories: - return [] + def process_memory_batch(self, memories: list[TextualMemoryItem], *args, **kwargs) -> list[str]: + pref_type_collection_map = { + "explicit_preference": "explicit_preference", + "implicit_preference": "implicit_preference", + } + + explicit_new_mems = [] + implicit_new_mems = [] + explicit_recalls = [] + implicit_recalls = [] - added_ids = [] + for memory in memories: + preference_type = memory.metadata.preference_type + collection_name = pref_type_collection_map[preference_type] + search_results = self.vector_db.search( + query_vector=memory.metadata.embedding, + query=memory.memory, + collection_name=collection_name, + top_k=5, + filter={"user_id": memory.metadata.user_id}, + ) + if preference_type == "explicit_preference": + explicit_recalls.extend(search_results) + explicit_new_mems.append(memory) + elif preference_type == "implicit_preference": + implicit_recalls.extend(search_results) + implicit_new_mems.append(memory) + + explicit_recalls = list({recall.id: recall for recall in explicit_recalls}.values()) + implicit_recalls = list({recall.id: recall for recall in implicit_recalls}.values()) + + explicit_added_ids = self._update_memory_op_trace(explicit_new_mems, explicit_recalls, pref_type_collection_map["explicit_preference"], "explicit_preference") + implicit_added_ids = self._update_memory_op_trace(implicit_new_mems, implicit_recalls, pref_type_collection_map["implicit_preference"], "implicit_preference") + return explicit_added_ids + implicit_added_ids + + def process_memory_single(self, memories: list[TextualMemoryItem], max_workers: int = 8, *args, **kwargs) -> list[str]: + added_ids: list[str] = [] with ThreadPoolExecutor(max_workers=min(max_workers, len(memories))) as executor: future_to_memory = { executor.submit(self._process_single_memory, memory): memory for memory in memories @@ -337,5 +370,25 @@ def add( memory = future_to_memory[future] logger.error(f"Error processing memory {memory.id}: {e}") continue - return added_ids + + + def add( + self, + memories: list[TextualMemoryItem | dict[str, Any]], + max_workers: int = 8, + *args, + **kwargs, + ) -> list[str]: + """Add the instruct preference memories using thread pool for acceleration.""" + if not memories: + return [] + + process_map = { + "single": self.process_memory_single, + "batch": self.process_memory_batch, + } + + process_func = process_map["single"] + return process_func(memories, max_workers) + diff --git a/src/memos/memories/textual/prefer_text_memory/extractor.py b/src/memos/memories/textual/prefer_text_memory/extractor.py index 8bfbfbc94..fdfe4a611 100644 --- a/src/memos/memories/textual/prefer_text_memory/extractor.py +++ b/src/memos/memories/textual/prefer_text_memory/extractor.py @@ -13,8 +13,11 @@ from memos.templates.prefer_complete_prompt import ( NAIVE_EXPLICIT_PREFERENCE_EXTRACT_PROMPT, NAIVE_IMPLICIT_PREFERENCE_EXTRACT_PROMPT, + NAIVE_EXPLICIT_PREFERENCE_EXTRACT_PROMPT_ZH, + NAIVE_IMPLICIT_PREFERENCE_EXTRACT_PROMPT_ZH, ) from memos.types import MessageList +from memos.mem_reader.simple_struct import detect_lang logger = get_logger(__name__) @@ -52,7 +55,12 @@ def extract_basic_info(self, qa_pair: MessageList) -> dict[str, Any]: def extract_explicit_preference(self, qa_pair: MessageList | str) -> dict[str, Any] | None: """Extract explicit preference from a QA pair.""" qa_pair_str = convert_messages_to_string(qa_pair) if isinstance(qa_pair, list) else qa_pair - prompt = NAIVE_EXPLICIT_PREFERENCE_EXTRACT_PROMPT.replace("{qa_pair}", qa_pair_str) + lang = detect_lang(qa_pair_str) + _map = { + "zh": NAIVE_EXPLICIT_PREFERENCE_EXTRACT_PROMPT_ZH, + "en": NAIVE_EXPLICIT_PREFERENCE_EXTRACT_PROMPT, + } + prompt = _map[lang].replace("{qa_pair}", qa_pair_str) try: response = self.llm_provider.generate([{"role": "user", "content": prompt}]) @@ -68,7 +76,12 @@ def extract_implicit_preference(self, qa_pair: MessageList | str) -> dict[str, A if not qa_pair: return None qa_pair_str = convert_messages_to_string(qa_pair) if isinstance(qa_pair, list) else qa_pair - prompt = NAIVE_IMPLICIT_PREFERENCE_EXTRACT_PROMPT.replace("{qa_pair}", qa_pair_str) + lang = detect_lang(qa_pair_str) + _map = { + "zh": NAIVE_IMPLICIT_PREFERENCE_EXTRACT_PROMPT_ZH, + "en": NAIVE_IMPLICIT_PREFERENCE_EXTRACT_PROMPT, + } + prompt = _map[lang].replace("{qa_pair}", qa_pair_str) try: response = self.llm_provider.generate([{"role": "user", "content": prompt}]) diff --git a/src/memos/templates/instruction_completion.py b/src/memos/templates/instruction_completion.py index 7ad0fe190..c2a7f58c7 100644 --- a/src/memos/templates/instruction_completion.py +++ b/src/memos/templates/instruction_completion.py @@ -1,6 +1,7 @@ from typing import Any -from memos.templates.prefer_complete_prompt import PREF_INSTRUCTIONS +from memos.mem_reader.simple_struct import detect_lang +from memos.templates.prefer_complete_prompt import PREF_INSTRUCTIONS, PREF_INSTRUCTIONS_ZH def instruct_completion( @@ -33,11 +34,25 @@ def instruct_completion( else "" ) + _prompt_map = { + "zh": PREF_INSTRUCTIONS_ZH, + "en": PREF_INSTRUCTIONS, + } + _remove_exp_map = { + "zh": "显式偏好 > ", + "en": "explicit preference > ", + } + _remove_imp_map = { + "zh": "隐式偏好 > ", + "en": "implicit preference > ", + } + lang = detect_lang(explicit_pref_str + implicit_pref_str) + if not explicit_pref_str and not implicit_pref_str: return "" if not explicit_pref_str: - return implicit_pref_str + "\n" + PREF_INSTRUCTIONS.replace("explicit preferences > ", "") + return implicit_pref_str + "\n" + _prompt_map[lang].replace(_remove_exp_map[lang], "") if not implicit_pref_str: - return explicit_pref_str + "\n" + PREF_INSTRUCTIONS.replace("implicit preferences > ", "") + return explicit_pref_str + "\n" + _prompt_map[lang].replace(_remove_imp_map[lang], "") - return explicit_pref_str + "\n" + implicit_pref_str + "\n" + PREF_INSTRUCTIONS + return explicit_pref_str + "\n" + implicit_pref_str + "\n" + _prompt_map[lang] diff --git a/src/memos/templates/prefer_complete_prompt.py b/src/memos/templates/prefer_complete_prompt.py index 51b1f2eed..db28ef4f9 100644 --- a/src/memos/templates/prefer_complete_prompt.py +++ b/src/memos/templates/prefer_complete_prompt.py @@ -29,6 +29,37 @@ """ +NAIVE_EXPLICIT_PREFERENCE_EXTRACT_PROMPT_ZH = """ +你是一个偏好提取助手。 +请从以下对话中提取用户明确提及的偏好。 + +注意事项： +- 偏好是指用户对某事物的明确态度或选择，不仅限于"喜欢/不喜欢/想要/不想要/偏好"等词汇。 +- 包括但不限于用户明确表达的任何倾向、渴望、拒绝或优先级，这些都算作显式偏好。 +- 重点提取用户在查询中的偏好。不要从助手的回复中提取偏好，除非用户明确同意或认可助手的建议。 +- 当用户针对同一主题或事件修改或更新其偏好时，提取其偏好变化的完整演变过程，包括原始偏好和更新后的偏好。 + +要求： +1. 只保留用户明确提到的偏好，不要推断或假设。如果用户提到了偏好的原因，也要包含这些原因。 +2. 输出应该是一个条目列表，包含简洁的自然语言摘要和相应的上下文摘要，上下文摘要必须包含提到偏好的对话片段的完整信息。 +3. 如果在同一主题或领域内提到了多个偏好，你必须将它们合并为一个条目，保持每个条目信息完整。 + +对话： +{qa_pair} + +找出所有显式偏好。如果没有找到显式偏好，返回[]。仅输出JSON： +```json +[ + { + "explicit_preference": "偏好的简短自然语言摘要", + "context_summary": "对应的上下文摘要，即对应对话的摘要，不要遗漏任何场景信息", + "reasoning": "寻找显式偏好的推理过程" + }, +] +``` +""" + + NAIVE_IMPLICIT_PREFERENCE_EXTRACT_PROMPT = """ You are a preference inference assistant. Please extract **implicit preferences** from the following conversation (preferences that the user did not explicitly state but can be reasonably inferred from context, behavior, frequency, comparisons, exclusions, or scenario choices). @@ -39,10 +70,9 @@ Requirements: 1. Only make inferences when there is sufficient evidence in the conversation; avoid unsupported or far-fetched guesses. -2. Output a concise natural language statement; do not use lists, categories, or include the reasoning process. -3. Inferred implicit preferences must not conflict with explicit preferences. -4. For implicit_preference: only output the preference statement itself; do not include any extra explanation, reasoning, or confidence information. Put all reasoning and explanation in the reasoning field. -5. If no implicit preference can be reasonably inferred, leave the implicit_preference field empty (do not output anything else). +2. Inferred implicit preferences must not conflict with explicit preferences. +3. For implicit_preference: only output the preference statement itself; do not include any extra explanation, reasoning, or confidence information. Put all reasoning and explanation in the reasoning field. +4. If no implicit preference can be reasonably inferred, leave the implicit_preference field empty (do not output anything else). Conversation: {qa_pair} @@ -59,6 +89,35 @@ """ +NAIVE_IMPLICIT_PREFERENCE_EXTRACT_PROMPT_ZH = """ +你是一个偏好推理助手。请从以下对话中提取**隐式偏好** +（用户没有明确表述，但可以从上下文、行为、频率、比较、排除或场景选择中合理推断出的偏好）。 + +注意事项： +- 隐式偏好是指用户未直接表达，但可以从对话中的事实线索合理推断出的倾向或选择。 +- 不要将明确陈述的偏好视为隐式偏好；此提示仅用于推断未直接提及的偏好。 + +要求： +1. 仅在对话中有充分证据时进行推断；避免无根据或牵强的猜测。 +2. 推断的隐式偏好不得与显式偏好冲突。 +3. 对于 implicit_preference：仅输出偏好陈述本身；不要包含任何额外的解释、推理或置信度信息。将所有推理和解释放在 reasoning 字段中。 +4. 如果无法合理推断出隐式偏好，则将 implicit_preference 字段留空（不要输出其他任何内容）。 + +对话： +{qa_pair} + +输出格式： +```json +{ + "implicit_preference": "从对话中合理推断出的隐式偏好的简洁自然语言陈述，或空字符串", + "context_summary": "对应的上下文摘要，即对应对话的摘要，不要遗漏任何场景信息", + "reasoning": "简要解释隐式偏好的推理过程" +} +``` +除JSON外不要输出任何其他内容。 +""" + + NAIVE_JUDGE_UPDATE_OR_ADD_PROMPT = """ You are a content comparison expert. Now you are given old and new information, each containing a question, answer topic name and topic description. Please judge whether these two information express the **same question or core content**, regardless of expression differences, details or example differences. The judgment criteria are as follows: @@ -80,6 +139,29 @@ {new_information} """ + +NAIVE_JUDGE_UPDATE_OR_ADD_PROMPT_ZH = """ +你是一个内容比较专家。现在给你旧信息和新信息，每个信息都包含问题、答案主题名称和主题描述。 +请判断这两个信息是否表达**相同的问题或核心内容**，不考虑表达差异、细节或示例差异。判断标准如下： + +- 核心内容一致，即要解决的问题本质、目标或核心概念相同，算作"相同"。 +- 表达方式不同、示例不同，但核心含义一致，也算作"相同"。 +- 如果问题目标、涉及的概念或解决思路不同，则算作"不同"。 + +请输出JSON格式： +{ + "is_same": true/false, + "reasoning": "简要解释判断依据，突出核心内容是否一致" +} + +**旧信息：** +{old_information} + +**新信息：** +{new_information} +""" + + NAIVE_JUDGE_UPDATE_OR_ADD_PROMPT_FINE = """ You are a preference memory comparison expert. Analyze if the new preference memory describes the same topic as any retrieved memories by considering BOTH the memory field and preference field. At most one retrieved memory can match the new memory. @@ -118,6 +200,44 @@ """ +NAIVE_JUDGE_UPDATE_OR_ADD_PROMPT_FINE_ZH = """ +你是一个偏好记忆比较专家。通过同时考虑 memory 字段和 preference 字段，分析新的偏好记忆是否与任何召回记忆描述相同的主题。最多只有一个召回记忆可以与新记忆匹配。 + +**任务：** 比较新的偏好记忆与召回记忆，以确定它们是否讨论相同的主题以及是否需要更新。 + +**比较标准：** +- **Memory 字段**：比较所描述的核心主题、场景和上下文 +- **Preference 字段**：比较表达的实际偏好陈述、选择和态度 +- **相同主题**：memory 和 preference 内容都涉及相同的主题 +- **不同主题**：memory 或 preference 内容有显著差异 +- **内容演变**：相同主题但偏好已改变/演变或记忆已更新 +- **内容相同**：memory 和 preference 字段本质上相同 + +**决策逻辑：** +- 核心主题相同（memory 和 preference 都相同）= 需要检查是否需要更新 +- 主题不同（memory 或 preference 有差异）= 不需要更新 +- 如果主题相同但内容已改变/演变 = 需要更新 +- 如果主题相同且内容完全相同 = 需要更新 + +**输出 JSON：** +```json +{ + "need_update": true/false, + "id": "正在更新的记忆的ID（如果不需要更新则为空字符串）", + "new_memory": "合并/演变后的更新 memory 字段（如果不需要更新则为空字符串）", + "new_preference": "合并/演变后的更新 preference 字段（如果不需要更新则为空字符串）", + "reasoning": "简要解释比较结果，同时考虑 memory 和 preference 字段" +} +``` + +**新的偏好记忆：** +{new_memory} + +**召回的偏好记忆：** +{retrieved_memories} +""" + + NAIVE_JUDGE_UPDATE_OR_ADD_PROMPT_OP_TRACE = """ # User Preference Memory Management Agent @@ -131,32 +251,168 @@ When updating a preference, you should also integrate and update the corresponding `context_summary` to ensure both fields stay semantically consistent. -You must produce a complete **operation trace**, showing which memory entries (identified by unique IDs) should be **added**, **updated**, or **deleted**, and then output the **final memory state** after all operations. +You must produce a complete **operation trace**, showing which memory entries (identified by unique IDs) should be **added**, **updated**, or **deleted**. ## Input Format -New preference memory (new_memory): -{new_memory} +New preference memories (new_memories): +{new_memories} Retrieved preference memories (retrieved_memories): {retrieved_memories} +## Task Instructions + +1. For each new memory, analyze its relationship with the retrieved memories: + - If a new memory is **unrelated** to all retrieved memories → perform `"ADD"` (insert as a new independent memory); + - If a new memory is **related** to one or more retrieved memories → perform `"UPDATE"` on those related retrieved memories (refine, supplement, or merge both the `preference` and the `context_summary`, while preserving change history trajectory information); + - If one or more retrieved memories are merged into one updated memory → perform `"DELETE"` on those retrieved memories. + +2. **Important**: Only retrieved memories that are related to the new memories should be updated or deleted. Retrieved memories that are unrelated to any new memory must be preserved. + +3. If multiple retrieved memories describe the same preference theme, merge them into one updated memory entry, combining both their `preference` information and their `context_summary` in a coherent and concise way. + +4. Output a structured list of **operation traces**, each explicitly stating: + - which memory (by ID) is affected, + - what operation is performed, + - the before/after `preference` and `context_summary`, + - and the reasoning behind it. + +## Output Format (JSON) + +{ + "trace": [ + { + "op_id": "op_1", + "type": "ADD" | "UPDATE" | "DELETE", + "target_id": "(the old memory ID; null if ADD)", + "old_preference": "(the old preference text; null if ADD)", + "old_context_summary": "(the old context summary; null if ADD)", + "new_preference": "(the updated or newly created preference, if applicable)", + "new_context_summary": "(the updated or newly created context summary, if applicable)", + "reason": "(brief natural-language explanation for the decision)" + } + ] +} +## Output Requirements + +- The output **must** be valid JSON. +- Each operation must include both `preference` and `context_summary` updates where applicable. +- Each operation must include a clear `reason`. +- Multiple retrieved memories may be merged into one unified updated memory. +- Do **not** include any explanatory text outside the JSON. +""" + + +NAIVE_JUDGE_UPDATE_OR_ADD_PROMPT_OP_TRACE_ZH = """ +# 用户偏好记忆管理代理 + +你是一个**用户偏好记忆管理代理**。 +你的目标是通过分析新的偏好信息并确定如何更新现有记忆，来维护用户的长期**偏好记忆库**。 + +每个记忆条目包含三个字段： +- **id**：记忆的唯一标识符。 +- **context_summary**：从中提取偏好的对话或情境的事实摘要。 +- **preference**：描述用户偏好或倾向的提取陈述。 + +更新偏好时，你还应该整合并更新相应的 `context_summary`，以确保两个字段保持语义一致。 + +你必须生成完整的**操作跟踪**，显示应该**添加**、**更新**或**删除**哪些记忆条目（通过唯一 ID 标识）。 + +## 输入格式 + +新的偏好记忆 (new_memories): +{new_memories} + +召回的偏好记忆 (retrieved_memories): +{retrieved_memories} +## 任务说明 + +1. 对于每个新记忆，分析其与召回记忆的关系： + - 如果新记忆与所有召回记忆**无关** → 执行 `"ADD"`（作为新的独立记忆插入）； + - 如果新记忆与一个或多个召回记忆**相关** → 对这些相关的召回记忆执行 `"UPDATE"`（细化、补充或合并 `preference` 和 `context_summary`，同时保留变化历史轨迹信息）； + - 如果一个或多个召回记忆被合并到一个更新的记忆中 → 对这些召回记忆执行 `"DELETE"`。 + +2. **重要**：只有与新记忆相关的召回记忆才应该被更新或删除。与任何新记忆都无关的召回记忆必须保留。 + +3. 如果多个召回记忆描述相同的偏好主题，将它们合并为一个更新的记忆条目，以连贯简洁的方式结合它们的 `preference` 信息和 `context_summary`。 + +4. 输出结构化的**操作跟踪**列表，每个操作明确说明： + - 受影响的记忆（通过 ID）； + - 执行的操作类型； + - 更新前后的 `preference` 和 `context_summary`； + - 以及决策的原因。 + +## 输出格式 (JSON) + +{ + "trace": [ + { + "op_id": "op_1", + "type": "ADD" | "UPDATE" | "DELETE", + "target_id": "（旧记忆 ID；如果是 ADD 则为 null）", + "old_preference": "（旧的偏好文本；如果是 ADD 则为 null）", + "old_context_summary": "（旧的上下文摘要；如果是 ADD 则为 null）", + "new_preference": "（更新或新创建的偏好，如果适用）", + "new_context_summary": "（更新或新创建的上下文摘要，如果适用）", + "reason": "（决策的简要自然语言解释）" + } + ] +} + +## 输出要求 + +- 输出**必须**是有效的 JSON。 +- 每个操作必须包含 `preference` 和 `context_summary` 的更新（如果适用）。 +- 每个操作必须包含清晰的 `reason`。 +- 多个召回记忆可以合并为一个统一的更新记忆。 +- **不要**在 JSON 之外包含任何解释性文本。 +""" + + +NAIVE_JUDGE_UPDATE_OR_ADD_PROMPT_OP_TRACE_WITH_ONE_SHOT = """ +# User Preference Memory Management Agent + +You are a **User Preference Memory Management Agent**. +Your goal is to maintain a user's long-term **preference memory base** by analyzing new preference information and determining how it should update existing memories. + +Each memory entry contains three fields: +- **id**: a unique identifier for the memory. +- **context_summary**: a factual summary of the dialogue or situation from which the preference was extracted. +- **preference**: the extracted statement describing the user's preference or tendency. + +When updating a preference, you should also integrate and update the corresponding `context_summary` to ensure both fields stay semantically consistent. + +You must produce a complete **operation trace**, showing which memory entries (identified by unique IDs) should be **added**, **updated**, or **deleted**, and then output the **final memory state** after all operations. + +## Input Format + +New preference memories (new_memories): +{new_memories} + +Retrieved preference memories (retrieved_memories): +{retrieved_memories} ## Task Instructions -1. Analyze each retrieved memory and determine its relationship to the new memory: - - **Unrelated** → perform `"ADD"` (insert as a new independent memory); - - **Related** → perform `"UPDATE"` (refine, supplement, or merge both the `preference` and the `context_summary`, while preserving change history trajectory information); - - **Conflicting or outdated** → perform `"DELETE"` (remove obsolete or contradictory memory). +1. For each new memory, analyze its relationship with the retrieved memories: + - If a new memory is **unrelated** to all retrieved memories → perform `"ADD"` (insert as a new independent memory); + - If a new memory is **related** to one or more retrieved memories → perform `"UPDATE"` on those related retrieved memories (refine, supplement, or merge both the `preference` and the `context_summary`, while preserving change history trajectory information); + - If one or more retrieved memories are merged into one updated memory → perform `"DELETE"` on those retrieved memories. -2. If multiple retrieved memories describe the same preference theme, merge them into one updated memory entry, combining both their `preference` information and their `context_summary` in a coherent and concise way. +2. **Important**: Only retrieved memories that are related to the new memories should be updated or deleted. Retrieved memories that are unrelated to any new memory must be preserved as-is in the final state. -3. Output a structured list of **operation traces**, each explicitly stating: +3. If multiple retrieved memories describe the same preference theme, merge them into one updated memory entry, combining both their `preference` information and their `context_summary` in a coherent and concise way. + +4. Output a structured list of **operation traces**, each explicitly stating: - which memory (by ID) is affected, - what operation is performed, - the before/after `preference` and `context_summary`, - and the reasoning behind it. -4. Output the **final memory state (after_update_state)**, representing the complete preference memory base after applying all operations. +5. Output the **final memory state (after_update_state)**, representing the complete preference memory base after applying all operations. This must include: + - All newly added memories (from ADD operations) + - All updated memories (from UPDATE operations) + - All unrelated retrieved memories that were preserved unchanged ## Output Format (JSON) @@ -185,11 +441,24 @@ ## Example **Input:** -new_memory: -{ - "context_summary": "During a recent chat about study habits, the user mentioned that he often studies in quiet coffee shops and has started preferring lattes over Americanos, which he only drinks occasionally.", - "preference": "User now prefers lattes but occasionally drinks Americanos; he also enjoys studying in quiet coffee shops." -} +new_memories: +[ + { + "id": "new_id1", + "context_summary": "During a recent chat about study habits, the user mentioned that he often studies in quiet coffee shops and has started preferring lattes over Americanos, which he only drinks occasionally.", + "preference": "User now prefers lattes but occasionally drinks Americanos; he also enjoys studying in quiet coffee shops." + }, + { + "id": "new_id2", + "context_summary": "The user mentioned in a conversation about beverages that he has recently started enjoying green tea in the morning.", + "preference": "User now enjoys drinking green tea in the morning." + }, + { + "id": "new_id3", + "context_summary": "The user shared that he has recently started learning to play the guitar and practices for about 30 minutes every evening.", + "preference": "User enjoys playing guitar and practices regularly in the evenings." + } +] retrieved_memories: [ @@ -212,6 +481,11 @@ "id": "id4", "context_summary": "The user noted he doesn't drink tea very often.", "preference": "User has no particular interest in tea." + }, + { + "id": "id5", + "context_summary": "The user mentioned he enjoys running in the park on weekends.", + "preference": "User likes running outdoors on weekends." } ] @@ -226,7 +500,7 @@ "old_context_summary": "The user previously said he likes coffee in general.", "new_preference": "User likes coffee, especially lattes, but occasionally drinks Americanos.", "new_context_summary": "The user discussed his coffee habits, stating he now prefers lattes but only occasionally drinks Americanos", - "reason": "The new memory refines and expands the coffee preference and context while preserving frequency semantics ('occasionally')." + "reason": "New memory new_id1 refines and expands the coffee preference and context while preserving frequency semantics ('occasionally')." }, { "op_id": "op_2", @@ -246,7 +520,27 @@ "old_context_summary": "The user said he often works from home.", "new_preference": "User now prefers studying in quiet coffee shops instead of working from home.", "new_context_summary": "The user mentioned shifting from working at home to studying in quiet cafes, reflecting a new preferred environment.", - "reason": "The preference has changed for the working environment." + "reason": "New memory new_id1 indicates a preference change for the working environment." + }, + { + "op_id": "op_4", + "type": "UPDATE", + "target_id": "id4", + "old_preference": "User has no particular interest in tea.", + "old_context_summary": "The user noted he doesn't drink tea very often.", + "new_preference": "The user does not drink tea very often before, but now enjoys drinking green tea in the morning.", + "new_context_summary": "The user mentioned that he has recently started enjoying green tea in the morning.", + "reason": "New memory new_id2 indicates a preference change for tea consumption." + }, + { + "op_id": "op_5", + "type": "ADD", + "target_id": "new_id3", + "old_preference": null, + "old_context_summary": null, + "new_preference": "User enjoys playing guitar and practices regularly in the evenings.", + "new_context_summary": "The user shared that he has recently started learning to play the guitar and practices for about 30 minutes every evening.", + "reason": "This is a completely new preference unrelated to any existing memories, so it should be added as a new entry." } ], "after_update_state": [ @@ -262,8 +556,18 @@ }, { "id": "id4", - "context_summary": "The user noted he doesn't drink tea very often.", - "preference": "User has no particular interest in tea." + "context_summary": "The user mentioned that he has recently started enjoying green tea in the morning.", + "preference": "The user does not drink tea very often before, but now enjoys drinking green tea in the morning." + }, + { + "id": "id5", + "context_summary": "The user mentioned he enjoys running in the park on weekends.", + "preference": "User likes running outdoors on weekends." + }, + { + "id": "new_id3", + "context_summary": "The user shared that he has recently started learning to play the guitar and practices for about 30 minutes every evening.", + "preference": "User enjoys playing guitar and practices regularly in the evenings." } ] } @@ -285,3 +589,11 @@ Your response must not violate any of the user's preferences, whether explicit or implicit, and briefly explain why you answer this way to avoid conflicts. When encountering preference conflicts, the priority is: explicit preference > implicit preference > plaintext memory. """ + + +PREF_INSTRUCTIONS_ZH = """ +# 注意： +明文记忆是事实的摘要，而偏好记忆是用户偏好的摘要。 +你的回复不得违反用户的任何偏好，无论是显式偏好还是隐式偏好，并简要解释你为什么这样回答以避免冲突。 +当遇到偏好冲突时，优先级为：显式偏好 > 隐式偏好 > 明文记忆。 +""" From 53342d45be1559bf7deb8e21cf0a3bb417341762 Mon Sep 17 00:00:00 2001 From: "yuan.wang" Date: Wed, 29 Oct 2025 16:19:47 +0800 Subject: [PATCH 06/21] add pref mem update srategy --- evaluation/scripts/utils/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evaluation/scripts/utils/client.py b/evaluation/scripts/utils/client.py index 33a1d6223..e1bdd54e9 100644 --- a/evaluation/scripts/utils/client.py +++ b/evaluation/scripts/utils/client.py @@ -182,7 +182,7 @@ def search(self, query, user_id, top_k): "conversation_id": "", "top_k": top_k, "mode": "mixture", - "handle_pref_mem": True, + "handle_pref_mem": False, }, ensure_ascii=False, ) From c2332846d6899c0ca8e083a592c2f8c749362b65 Mon Sep 17 00:00:00 2001 From: "yuan.wang" Date: Wed, 29 Oct 2025 16:49:21 +0800 Subject: [PATCH 07/21] fix bug in pre_commit --- docker/requirements.txt | 2 +- .../textual/prefer_text_memory/adder.py | 68 ++++++++++++------- .../textual/prefer_text_memory/extractor.py | 4 +- .../textual/prefer_text_memory/retrievers.py | 44 +++++++++--- .../textual/prefer_text_memory/spliter.py | 5 +- src/memos/reranker/cosine_local.py | 2 +- src/memos/reranker/noop.py | 4 +- src/memos/templates/prefer_complete_prompt.py | 2 +- src/memos/vec_dbs/milvus.py | 36 ++++++++-- 9 files changed, 119 insertions(+), 48 deletions(-) diff --git a/docker/requirements.txt b/docker/requirements.txt index d20c0b36e..4846f1832 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -157,4 +157,4 @@ volcengine-python-sdk==4.0.6 watchfiles==1.1.0 websockets==15.0.1 xlrd==2.0.2 -xlsxwriter==3.2.5 \ No newline at end of file +xlsxwriter==3.2.5 diff --git a/src/memos/memories/textual/prefer_text_memory/adder.py b/src/memos/memories/textual/prefer_text_memory/adder.py index 10f1045dc..052ae30c2 100644 --- a/src/memos/memories/textual/prefer_text_memory/adder.py +++ b/src/memos/memories/textual/prefer_text_memory/adder.py @@ -2,16 +2,16 @@ from abc import ABC, abstractmethod from concurrent.futures import as_completed +from datetime import datetime from typing import Any -from datetime import datetime from memos.context.context import ContextThreadPoolExecutor from memos.log import get_logger from memos.memories.textual.item import TextualMemoryItem from memos.templates.prefer_complete_prompt import ( NAIVE_JUDGE_UPDATE_OR_ADD_PROMPT, - NAIVE_JUDGE_UPDATE_OR_ADD_PROMPT_OP_TRACE, NAIVE_JUDGE_UPDATE_OR_ADD_PROMPT_FINE, + NAIVE_JUDGE_UPDATE_OR_ADD_PROMPT_OP_TRACE, ) from memos.vec_dbs.item import MilvusVecDBItem @@ -65,9 +65,7 @@ def _judge_update_or_add_fast(self, old_msg: str, new_msg: str) -> bool: # Fallback to simple string comparison return old_msg == new_msg - def _judge_update_or_add_fine( - self, new_mem: str, retrieved_mems: str - ) -> dict[str, Any] | None: + def _judge_update_or_add_fine(self, new_mem: str, retrieved_mems: str) -> dict[str, Any] | None: if not retrieved_mems: return None prompt = NAIVE_JUDGE_UPDATE_OR_ADD_PROMPT_FINE.replace("{new_memory}", new_mem).replace( @@ -87,9 +85,9 @@ def _judge_update_or_add_trace_op( ) -> dict[str, Any] | None: if not retrieved_mems: return None - prompt = NAIVE_JUDGE_UPDATE_OR_ADD_PROMPT_OP_TRACE.replace("{new_memories}", new_mems).replace( - "{retrieved_memories}", retrieved_mems - ) + prompt = NAIVE_JUDGE_UPDATE_OR_ADD_PROMPT_OP_TRACE.replace( + "{new_memories}", new_mems + ).replace("{retrieved_memories}", retrieved_mems) try: response = self.llm_provider.generate([{"role": "user", "content": prompt}]) response = response.strip().replace("```json", "").replace("```", "").strip() @@ -143,16 +141,23 @@ def _update_memory_op_trace( retrieved_mems=json.dumps(retrieved_mem_inputs) if retrieved_mem_inputs else "", ) if not rsp: - with ThreadPoolExecutor(max_workers=min(len(new_vec_db_items), 5)) as executor: - futures = {executor.submit(self.vector_db.add, collection_name, [db_item]): db_item for db_item in new_vec_db_items} + with ContextThreadPoolExecutor(max_workers=min(len(new_vec_db_items), 5)) as executor: + futures = { + executor.submit(self.vector_db.add, collection_name, [db_item]): db_item + for db_item in new_vec_db_items + } for future in as_completed(futures): result = future.result() return [db_item.id for db_item in new_vec_db_items] new_mem_db_item_map = {db_item.id: db_item for db_item in new_vec_db_items} retrieved_mem_db_item_map = {db_item.id: db_item for db_item in retrieved_memories} - def execute_op(op, new_mem_db_item_map: dict[str, MilvusVecDBItem], - retrieved_mem_db_item_map: dict[str, MilvusVecDBItem]) -> str | None: + + def execute_op( + op, + new_mem_db_item_map: dict[str, MilvusVecDBItem], + retrieved_mem_db_item_map: dict[str, MilvusVecDBItem], + ) -> str | None: op_type = op["type"].lower() if op_type == "add": if op["target_id"] in new_mem_db_item_map: @@ -175,7 +180,10 @@ def execute_op(op, new_mem_db_item_map: dict[str, MilvusVecDBItem], return None with ContextThreadPoolExecutor(max_workers=min(len(rsp["trace"]), 5)) as executor: - future_to_op = {executor.submit(execute_op, op, new_mem_db_item_map, retrieved_mem_db_item_map): op for op in rsp["trace"]} + future_to_op = { + executor.submit(execute_op, op, new_mem_db_item_map, retrieved_mem_db_item_map): op + for op in rsp["trace"] + } added_ids = [] for future in as_completed(future_to_op): result = future.result() @@ -221,7 +229,9 @@ def _update_memory_fine( retrieved_mems=json.dumps(retrieved_mem_inputs) if retrieved_mem_inputs else "", ) need_update = rsp.get("need_update", False) if rsp else False - need_update = need_update if isinstance(need_update, bool) else need_update.lower() == "true" + need_update = ( + need_update if isinstance(need_update, bool) else need_update.lower() == "true" + ) update_item = [mem for mem in retrieved_memories if mem.id == rsp["id"]] if need_update and update_item: update_vec_db_item = update_item[0] @@ -286,7 +296,9 @@ def _update_memory( if update_mode == "fast": return self._update_memory_fast(new_memory, retrieved_memories, collection_name) elif update_mode == "fine": - return self._update_memory_fine(new_memory, retrieved_memories, collection_name, preference_type) + return self._update_memory_fine( + new_memory, retrieved_memories, collection_name, preference_type + ) else: raise ValueError(f"Invalid update mode: {update_mode}") @@ -319,9 +331,9 @@ def _process_single_memory(self, memory: TextualMemoryItem) -> list[str] | str | def process_memory_batch(self, memories: list[TextualMemoryItem], *args, **kwargs) -> list[str]: pref_type_collection_map = { - "explicit_preference": "explicit_preference", - "implicit_preference": "implicit_preference", - } + "explicit_preference": "explicit_preference", + "implicit_preference": "implicit_preference", + } explicit_new_mems = [] implicit_new_mems = [] @@ -348,11 +360,23 @@ def process_memory_batch(self, memories: list[TextualMemoryItem], *args, **kwarg explicit_recalls = list({recall.id: recall for recall in explicit_recalls}.values()) implicit_recalls = list({recall.id: recall for recall in implicit_recalls}.values()) - explicit_added_ids = self._update_memory_op_trace(explicit_new_mems, explicit_recalls, pref_type_collection_map["explicit_preference"], "explicit_preference") - implicit_added_ids = self._update_memory_op_trace(implicit_new_mems, implicit_recalls, pref_type_collection_map["implicit_preference"], "implicit_preference") + explicit_added_ids = self._update_memory_op_trace( + explicit_new_mems, + explicit_recalls, + pref_type_collection_map["explicit_preference"], + "explicit_preference", + ) + implicit_added_ids = self._update_memory_op_trace( + implicit_new_mems, + implicit_recalls, + pref_type_collection_map["implicit_preference"], + "implicit_preference", + ) return explicit_added_ids + implicit_added_ids - def process_memory_single(self, memories: list[TextualMemoryItem], max_workers: int = 8, *args, **kwargs) -> list[str]: + def process_memory_single( + self, memories: list[TextualMemoryItem], max_workers: int = 8, *args, **kwargs + ) -> list[str]: added_ids: list[str] = [] with ContextThreadPoolExecutor(max_workers=min(max_workers, len(memories))) as executor: future_to_memory = { @@ -373,7 +397,6 @@ def process_memory_single(self, memories: list[TextualMemoryItem], max_workers: continue return added_ids - def add( self, memories: list[TextualMemoryItem | dict[str, Any]], @@ -392,4 +415,3 @@ def add( process_func = process_map["single"] return process_func(memories, max_workers) - diff --git a/src/memos/memories/textual/prefer_text_memory/extractor.py b/src/memos/memories/textual/prefer_text_memory/extractor.py index 82979a72f..61629b38a 100644 --- a/src/memos/memories/textual/prefer_text_memory/extractor.py +++ b/src/memos/memories/textual/prefer_text_memory/extractor.py @@ -8,17 +8,17 @@ from memos.context.context import ContextThreadPoolExecutor from memos.log import get_logger +from memos.mem_reader.simple_struct import detect_lang from memos.memories.textual.item import PreferenceTextualMemoryMetadata, TextualMemoryItem from memos.memories.textual.prefer_text_memory.spliter import Splitter from memos.memories.textual.prefer_text_memory.utils import convert_messages_to_string from memos.templates.prefer_complete_prompt import ( NAIVE_EXPLICIT_PREFERENCE_EXTRACT_PROMPT, - NAIVE_IMPLICIT_PREFERENCE_EXTRACT_PROMPT, NAIVE_EXPLICIT_PREFERENCE_EXTRACT_PROMPT_ZH, + NAIVE_IMPLICIT_PREFERENCE_EXTRACT_PROMPT, NAIVE_IMPLICIT_PREFERENCE_EXTRACT_PROMPT_ZH, ) from memos.types import MessageList -from memos.mem_reader.simple_struct import detect_lang logger = get_logger(__name__) diff --git a/src/memos/memories/textual/prefer_text_memory/retrievers.py b/src/memos/memories/textual/prefer_text_memory/retrievers.py index 7215b7666..f09d646b1 100644 --- a/src/memos/memories/textual/prefer_text_memory/retrievers.py +++ b/src/memos/memories/textual/prefer_text_memory/retrievers.py @@ -30,17 +30,27 @@ def __init__(self, llm_provider=None, embedder=None, reranker=None, vector_db=No self.vector_db = vector_db self.embedder = embedder - def _naive_reranker(self, query: str, prefs_mem: list[TextualMemoryItem], top_k: int, **kwargs: Any) -> list[TextualMemoryItem]: + def _naive_reranker( + self, query: str, prefs_mem: list[TextualMemoryItem], top_k: int, **kwargs: Any + ) -> list[TextualMemoryItem]: if self.reranker: prefs_mem = self.reranker.rerank(query, prefs_mem, top_k) return [item for item, _ in prefs_mem] return prefs_mem - def _original_text_reranker(self, query: str, prefs_mem: list[TextualMemoryItem], prefs: list[MilvusVecDBItem], top_k: int, **kwargs: Any) -> list[TextualMemoryItem]: + def _original_text_reranker( + self, + query: str, + prefs_mem: list[TextualMemoryItem], + prefs: list[MilvusVecDBItem], + top_k: int, + **kwargs: Any, + ) -> list[TextualMemoryItem]: if self.reranker: from copy import deepcopy + prefs_mem_for_reranker = deepcopy(prefs_mem) - for pref_mem, pref in zip(prefs_mem_for_reranker, prefs): + for pref_mem, pref in zip(prefs_mem_for_reranker, prefs, strict=False): pref_mem.memory = pref_mem.memory + "\n" + pref.original_text prefs_mem_for_reranker = self.reranker.rerank(query, prefs_mem_for_reranker, top_k) prefs_mem_for_reranker = [item for item, _ in prefs_mem_for_reranker] @@ -48,7 +58,6 @@ def _original_text_reranker(self, query: str, prefs_mem: list[TextualMemoryItem] prefs_dict = {item.id: item for item in prefs_mem} return [prefs_dict[item_id] for item_id in prefs_ids if item_id in prefs_dict] return prefs_mem - def retrieve( self, query: str, top_k: int, info: dict[str, Any] | None = None @@ -66,10 +75,20 @@ def retrieve( with ContextThreadPoolExecutor(max_workers=2) as executor: # Submit all search tasks future_explicit = executor.submit( - self.vector_db.search, query_embedding, query, "explicit_preference", top_k * 2, info + self.vector_db.search, + query_embedding, + query, + "explicit_preference", + top_k * 2, + info, ) future_implicit = executor.submit( - self.vector_db.search, query_embedding, query, "implicit_preference", top_k * 2, info + self.vector_db.search, + query_embedding, + query, + "implicit_preference", + top_k * 2, + info, ) # Wait for all results @@ -100,9 +119,16 @@ def retrieve( if pref.payload["implicit_preference"] ] - reranker_map = {"naive": self._naive_reranker, "original_text": self._original_text_reranker} + reranker_map = { + "naive": self._naive_reranker, + "original_text": self._original_text_reranker, + } reranker_func = reranker_map["naive"] - explicit_prefs_mem = reranker_func(query=query, prefs_mem=explicit_prefs_mem, prefs=explicit_prefs, top_k=top_k) - implicit_prefs_mem = reranker_func(query=query, prefs_mem=implicit_prefs_mem, prefs=implicit_prefs, top_k=top_k) + explicit_prefs_mem = reranker_func( + query=query, prefs_mem=explicit_prefs_mem, prefs=explicit_prefs, top_k=top_k + ) + implicit_prefs_mem = reranker_func( + query=query, prefs_mem=implicit_prefs_mem, prefs=implicit_prefs, top_k=top_k + ) return explicit_prefs_mem + implicit_prefs_mem diff --git a/src/memos/memories/textual/prefer_text_memory/spliter.py b/src/memos/memories/textual/prefer_text_memory/spliter.py index eafd08b03..3059d611b 100644 --- a/src/memos/memories/textual/prefer_text_memory/spliter.py +++ b/src/memos/memories/textual/prefer_text_memory/spliter.py @@ -85,10 +85,7 @@ def _split_with_overlap(self, data: MessageList) -> list[MessageList]: if len(chunk) >= 10: chunks.append(chunk) # overlap 1 turns (Q + A = 2) - if i + 1 < len(data): - context = copy.deepcopy(chunk[-2:]) - else: - context = [] + context = copy.deepcopy(chunk[-2:]) if i + 1 < len(data) else [] chunk = context if chunk and len(chunk) % 2 == 0: chunks.append(chunk) diff --git a/src/memos/reranker/cosine_local.py b/src/memos/reranker/cosine_local.py index 38ace458f..318cd744a 100644 --- a/src/memos/reranker/cosine_local.py +++ b/src/memos/reranker/cosine_local.py @@ -4,9 +4,9 @@ from typing import TYPE_CHECKING from memos.log import get_logger +from memos.utils import timed from .base import BaseReranker -from memos.utils import timed if TYPE_CHECKING: diff --git a/src/memos/reranker/noop.py b/src/memos/reranker/noop.py index 4f6ba0438..04250bef7 100644 --- a/src/memos/reranker/noop.py +++ b/src/memos/reranker/noop.py @@ -2,9 +2,11 @@ from typing import TYPE_CHECKING -from .base import BaseReranker from memos.utils import timed +from .base import BaseReranker + + if TYPE_CHECKING: from memos.memories.textual.item import TextualMemoryItem diff --git a/src/memos/templates/prefer_complete_prompt.py b/src/memos/templates/prefer_complete_prompt.py index db28ef4f9..b98e65d54 100644 --- a/src/memos/templates/prefer_complete_prompt.py +++ b/src/memos/templates/prefer_complete_prompt.py @@ -177,7 +177,7 @@ **Decision Logic:** - Same core topic (both memory and preference) = need to check if update is needed -- Different topics (either memory or preference differs) = no update needed +- Different topics (either memory or preference differs) = no update needed - If same topic but content has changed/evolved = update needed - If same topic and content is identical = update needed diff --git a/src/memos/vec_dbs/milvus.py b/src/memos/vec_dbs/milvus.py index a22490404..e50c8ce18 100644 --- a/src/memos/vec_dbs/milvus.py +++ b/src/memos/vec_dbs/milvus.py @@ -41,7 +41,14 @@ def create_schema(self): field_name="id", datatype=DataType.VARCHAR, max_length=65535, is_primary=True ) analyzer_params = {"tokenizer": "standard", "filter": ["lowercase"]} - schema.add_field(field_name="memory", datatype=DataType.VARCHAR, max_length=65535, analyzer_params=analyzer_params, enable_match=True, enable_analyzer=True) + schema.add_field( + field_name="memory", + datatype=DataType.VARCHAR, + max_length=65535, + analyzer_params=analyzer_params, + enable_match=True, + enable_analyzer=True, + ) schema.add_field(field_name="original_text", datatype=DataType.VARCHAR, max_length=65535) schema.add_field( field_name="vector", datatype=DataType.FLOAT_VECTOR, dim=self.config.vector_dimension @@ -170,16 +177,27 @@ def _hybrid_search( ) -> list[list[dict]]: """Hybrid search for similar items in the database.""" from pymilvus import AnnSearchRequest, RRFRanker, WeightedRanker + # Set up BM25 search request expr = filter if filter else None sparse_request = AnnSearchRequest( - data=[query], anns_field="sparse_vector", param={"metric_type": "BM25"}, limit=top_k, expr=expr + data=[query], + anns_field="sparse_vector", + param={"metric_type": "BM25"}, + limit=top_k, + expr=expr, ) # Set up dense vector search request dense_request = AnnSearchRequest( - data=[query_vector], anns_field="vector", param={"metric_type": self._get_metric_type()}, limit=top_k, expr=expr + data=[query_vector], + anns_field="vector", + param={"metric_type": self._get_metric_type()}, + limit=top_k, + expr=expr, + ) + ranker = ( + RRFRanker() if ranker_type == "rrf" else WeightedRanker(sparse_weight, dense_weight) ) - ranker = RRFRanker() if ranker_type == "rrf" else WeightedRanker(sparse_weight, dense_weight) results = self.client.hybrid_search( collection_name=collection_name, reqs=[sparse_request, dense_request], @@ -196,7 +214,7 @@ def search( collection_name: str, top_k: int, filter: dict[str, Any] | None = None, - search_type: str = "dense", # dense, sparse, hybrid + search_type: str = "dense", # dense, sparse, hybrid ) -> list[MilvusVecDBItem]: """ Search for similar items in the database. @@ -219,7 +237,13 @@ def search( "hybrid": self._hybrid_search, } - results = search_func_map[search_type](collection_name=collection_name, query_vector=query_vector, query=query, top_k=top_k, filter=expr) + results = search_func_map[search_type]( + collection_name=collection_name, + query_vector=query_vector, + query=query, + top_k=top_k, + filter=expr, + ) items = [] for hit in results[0]: From 85e46c617e56bafb5bde1bc70f9aa1309dd4456d Mon Sep 17 00:00:00 2001 From: "yuan.wang" Date: Thu, 30 Oct 2025 16:59:28 +0800 Subject: [PATCH 08/21] modify pref filed --- evaluation/scripts/PrefEval/pref_memos.py | 2 +- evaluation/scripts/locomo/locomo_search.py | 4 ++-- evaluation/scripts/longmemeval/lme_search.py | 2 +- evaluation/scripts/personamem/pm_search.py | 2 +- evaluation/scripts/utils/client.py | 12 +++++++----- src/memos/api/config.py | 2 +- src/memos/api/product_models.py | 3 ++- src/memos/api/routers/server_router.py | 13 +++++++------ src/memos/templates/instruction_completion.py | 16 +++++++++++----- 9 files changed, 33 insertions(+), 23 deletions(-) diff --git a/evaluation/scripts/PrefEval/pref_memos.py b/evaluation/scripts/PrefEval/pref_memos.py index fc358dc36..f2102a7b1 100644 --- a/evaluation/scripts/PrefEval/pref_memos.py +++ b/evaluation/scripts/PrefEval/pref_memos.py @@ -98,7 +98,7 @@ def search_memory_for_line(line_data: tuple, mem_client, top_k_value: int) -> di f"- {entry.get('memory', '')}" for entry in relevant_memories["text_mem"][0]["memories"] ) - + f"\n{relevant_memories['pref_string']}" + + f"\n{relevant_memories.get('pref_string', '')}" ) memory_tokens_used = len(tokenizer.encode(memories_str)) diff --git a/evaluation/scripts/locomo/locomo_search.py b/evaluation/scripts/locomo/locomo_search.py index 0b610d574..24f6149ec 100644 --- a/evaluation/scripts/locomo/locomo_search.py +++ b/evaluation/scripts/locomo/locomo_search.py @@ -107,11 +107,11 @@ def memos_api_search( speaker_a_context = ( "\n".join([i["memory"] for i in search_a_results["text_mem"][0]["memories"]]) - + f"\n{search_a_results['pref_string']}" + + f"\n{search_a_results.get('pref_string', '')}" ) speaker_b_context = ( "\n".join([i["memory"] for i in search_b_results["text_mem"][0]["memories"]]) - + f"\n{search_b_results['pref_string']}" + + f"\n{search_b_results.get('pref_string', '')}" ) context = TEMPLATE_MEMOS.format( diff --git a/evaluation/scripts/longmemeval/lme_search.py b/evaluation/scripts/longmemeval/lme_search.py index 89c02aaea..8e0e3c5c2 100644 --- a/evaluation/scripts/longmemeval/lme_search.py +++ b/evaluation/scripts/longmemeval/lme_search.py @@ -46,7 +46,7 @@ def memos_search(client, query, user_id, top_k): results = client.search(query=query, user_id=user_id, top_k=top_k) context = ( "\n".join([i["memory"] for i in results["text_mem"][0]["memories"]]) - + f"\n{results['pref_string']}" + + f"\n{results.get('pref_string', '')}" ) context = MEMOS_CONTEXT_TEMPLATE.format(user_id=user_id, memories=context) duration_ms = (time() - start) * 1000 diff --git a/evaluation/scripts/personamem/pm_search.py b/evaluation/scripts/personamem/pm_search.py index 441474c7c..9c30db876 100644 --- a/evaluation/scripts/personamem/pm_search.py +++ b/evaluation/scripts/personamem/pm_search.py @@ -84,7 +84,7 @@ def memos_search(client, user_id, query, top_k): results = client.search(query=query, user_id=user_id, top_k=top_k) search_memories = ( "\n".join(item["memory"] for cube in results["text_mem"] for item in cube["memories"]) - + f"\n{results['pref_string']}" + + f"\n{results.get('pref_string', '')}" ) context = MEMOS_CONTEXT_TEMPLATE.format(user_id=user_id, memories=search_memories) diff --git a/evaluation/scripts/utils/client.py b/evaluation/scripts/utils/client.py index e1bdd54e9..1e2648b3f 100644 --- a/evaluation/scripts/utils/client.py +++ b/evaluation/scripts/utils/client.py @@ -181,8 +181,9 @@ def search(self, query, user_id, top_k): "mem_cube_id": user_id, "conversation_id": "", "top_k": top_k, - "mode": "mixture", - "handle_pref_mem": False, + "mode": "fast", + "include_preference": True, + "pref_top_k": 6, }, ensure_ascii=False, ) @@ -344,9 +345,10 @@ def wait_for_completion(self, task_id): query = "杭州西湖有什么" top_k = 5 - # MEMOBASE - client = MemobaseClient() + # MEMOS-API + client = MemosApiClient() for m in messages: m["created_at"] = iso_date - client.add(messages, user_id) + client.add(messages, user_id, user_id) memories = client.search(query, user_id, top_k) + print(memories) diff --git a/src/memos/api/config.py b/src/memos/api/config.py index 6de013313..e9541d8a1 100644 --- a/src/memos/api/config.py +++ b/src/memos/api/config.py @@ -114,7 +114,7 @@ def get_preference_memory_config() -> dict[str, Any]: return { "backend": "pref_text", "config": { - "extractor_llm": {"backend": "openai", "config": APIConfig.get_openai_config()}, + "extractor_llm": APIConfig.get_memreader_config(), "vector_db": { "backend": "milvus", "config": APIConfig.get_milvus_config(), diff --git a/src/memos/api/product_models.py b/src/memos/api/product_models.py index dd2fde22b..0412754c3 100644 --- a/src/memos/api/product_models.py +++ b/src/memos/api/product_models.py @@ -180,7 +180,8 @@ class APISearchRequest(BaseRequest): operation: list[PermissionDict] | None = Field( None, description="operation ids for multi cubes" ) - handle_pref_mem: bool = Field(False, description="Whether to handle preference memory") + include_preference: bool = Field(True, description="Whether to handle preference memory") + pref_top_k: int = Field(6, description="Number of preference results to return") class APIADDRequest(BaseRequest): diff --git a/src/memos/api/routers/server_router.py b/src/memos/api/routers/server_router.py index bb98f04ba..3851c377f 100644 --- a/src/memos/api/routers/server_router.py +++ b/src/memos/api/routers/server_router.py @@ -301,17 +301,18 @@ def _post_process_pref_mem( memories_result: list[dict[str, Any]], pref_formatted_mem: list[dict[str, Any]], mem_cube_id: str, - handle_pref_mem: bool, + include_preference: bool, ): - if handle_pref_mem: + if include_preference: memories_result["pref_mem"].append( { "cube_id": mem_cube_id, "memories": pref_formatted_mem, } ) - pref_instruction: str = instruct_completion(pref_formatted_mem) + pref_instruction, pref_note = instruct_completion(pref_formatted_mem) memories_result["pref_string"] = pref_instruction + memories_result["pref_note"] = pref_note return memories_result @@ -331,7 +332,7 @@ def search_memories(search_req: APISearchRequest): "act_mem": [], "para_mem": [], "pref_mem": [], - "pref_string": "", + "pref_note": "", } search_mode = search_req.mode @@ -359,7 +360,7 @@ def _search_pref(): return [] results = naive_mem_cube.pref_mem.search( query=search_req.query, - top_k=search_req.top_k, + top_k=6, info={ "user_id": search_req.user_id, "session_id": search_req.session_id, @@ -382,7 +383,7 @@ def _search_pref(): ) memories_result = _post_process_pref_mem( - memories_result, pref_formatted_memories, search_req.mem_cube_id, search_req.handle_pref_mem + memories_result, pref_formatted_memories, search_req.mem_cube_id, search_req.include_preference ) return SearchResponse( diff --git a/src/memos/templates/instruction_completion.py b/src/memos/templates/instruction_completion.py index c2a7f58c7..acd110930 100644 --- a/src/memos/templates/instruction_completion.py +++ b/src/memos/templates/instruction_completion.py @@ -6,7 +6,7 @@ def instruct_completion( memories: list[dict[str, Any]] | None = None, -) -> str: +) -> [str, str]: """Create instruction following the preferences.""" explicit_pref = [] implicit_pref = [] @@ -49,10 +49,16 @@ def instruct_completion( lang = detect_lang(explicit_pref_str + implicit_pref_str) if not explicit_pref_str and not implicit_pref_str: - return "" + return "", "" if not explicit_pref_str: - return implicit_pref_str + "\n" + _prompt_map[lang].replace(_remove_exp_map[lang], "") + pref_note = _prompt_map[lang].replace(_remove_exp_map[lang], "") + pref_string = implicit_pref_str + "\n" + pref_note + return pref_string, pref_note if not implicit_pref_str: - return explicit_pref_str + "\n" + _prompt_map[lang].replace(_remove_imp_map[lang], "") + pref_note = _prompt_map[lang].replace(_remove_imp_map[lang], "") + pref_string = explicit_pref_str + "\n" + pref_note + return pref_string, pref_note - return explicit_pref_str + "\n" + implicit_pref_str + "\n" + _prompt_map[lang] + pref_note = _prompt_map[lang] + pref_string = explicit_pref_str + "\n" + implicit_pref_str + "\n" + pref_note + return pref_string, pref_note From 64dad1ca7571878625c9b098c5fdccd4195e950c Mon Sep 17 00:00:00 2001 From: "yuan.wang" Date: Thu, 30 Oct 2025 17:29:00 +0800 Subject: [PATCH 09/21] fix bug --- src/memos/api/routers/server_router.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/memos/api/routers/server_router.py b/src/memos/api/routers/server_router.py index 3851c377f..59c347359 100644 --- a/src/memos/api/routers/server_router.py +++ b/src/memos/api/routers/server_router.py @@ -360,7 +360,7 @@ def _search_pref(): return [] results = naive_mem_cube.pref_mem.search( query=search_req.query, - top_k=6, + top_k=search_req.pref_top_k, info={ "user_id": search_req.user_id, "session_id": search_req.session_id, From 032573b353b9b13af8f9b141fa3d72d9fae123d3 Mon Sep 17 00:00:00 2001 From: "yuan.wang" Date: Thu, 30 Oct 2025 17:46:01 +0800 Subject: [PATCH 10/21] fix pre_commit --- src/memos/api/routers/server_router.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/memos/api/routers/server_router.py b/src/memos/api/routers/server_router.py index b2a4bce6a..e255b1a48 100644 --- a/src/memos/api/routers/server_router.py +++ b/src/memos/api/routers/server_router.py @@ -406,7 +406,10 @@ def _search_pref(): ) memories_result = _post_process_pref_mem( - memories_result, pref_formatted_memories, search_req.mem_cube_id, search_req.include_preference + memories_result, + pref_formatted_memories, + search_req.mem_cube_id, + search_req.include_preference, ) return SearchResponse( From e2a0335aa0b631a77d04b5e25a573e36a15859c0 Mon Sep 17 00:00:00 2001 From: "yuan.wang" Date: Thu, 30 Oct 2025 19:45:28 +0800 Subject: [PATCH 11/21] fix bug in adder --- evaluation/README.md | 2 +- evaluation/scripts/PrefEval/pref_mem0.py | 2 +- evaluation/scripts/PrefEval/pref_memobase.py | 3 +- evaluation/scripts/PrefEval/pref_memos.py | 3 +- evaluation/scripts/PrefEval/pref_memu.py | 3 +- .../scripts/PrefEval/pref_supermemory.py | 3 +- evaluation/scripts/PrefEval/pref_zep.py | 3 +- evaluation/scripts/personamem/pm_ingestion.py | 62 +++++++++++++------ evaluation/scripts/personamem/pm_metric.py | 15 ++++- evaluation/scripts/personamem/pm_responses.py | 37 ++++++++--- evaluation/scripts/personamem/pm_search.py | 37 ++++++++--- evaluation/scripts/run_prefeval_eval.sh | 2 +- .../textual/prefer_text_memory/adder.py | 8 ++- 13 files changed, 130 insertions(+), 50 deletions(-) diff --git a/evaluation/README.md b/evaluation/README.md index ba8c7a0cc..8683c60b2 100644 --- a/evaluation/README.md +++ b/evaluation/README.md @@ -84,4 +84,4 @@ get `questions_32k.csv` and `shared_contexts_32k.jsonl` from https://huggingface # Specify the model and memory backend you want to use (e.g., mem0, zep, etc.) # If you want to use MIRIX, edit the the configuration in ./scripts/personamem/config.yaml ./scripts/run_pm_eval.sh -``` \ No newline at end of file +``` diff --git a/evaluation/scripts/PrefEval/pref_mem0.py b/evaluation/scripts/PrefEval/pref_mem0.py index 214068567..300e0ede3 100644 --- a/evaluation/scripts/PrefEval/pref_mem0.py +++ b/evaluation/scripts/PrefEval/pref_mem0.py @@ -56,7 +56,7 @@ def add_memory_for_line( for idx, _ in enumerate(conversation[::2]): msg_idx = idx * 2 - record_id = f"{lib}_user_pref_eval_{i}_{version}_{str(msg_idx)}" + record_id = f"{lib}_user_pref_eval_{i}_{version}_{msg_idx!s}" timestamp_add = int(time.time() * 100) if record_id not in success_records: diff --git a/evaluation/scripts/PrefEval/pref_memobase.py b/evaluation/scripts/PrefEval/pref_memobase.py index e99b10520..776642657 100644 --- a/evaluation/scripts/PrefEval/pref_memobase.py +++ b/evaluation/scripts/PrefEval/pref_memobase.py @@ -12,6 +12,7 @@ from openai import OpenAI from tqdm import tqdm + ROOT_DIR = os.path.dirname( os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) ) @@ -68,7 +69,7 @@ def add_memory_for_line( ) for idx, _ in enumerate(conversation[::2]): msg_idx = idx * 2 - record_id = f"{lib}_user_pref_eval_{i}_{version}_{str(msg_idx)}" + record_id = f"{lib}_user_pref_eval_{i}_{version}_{msg_idx!s}" if record_id not in success_records: mem_client.add(messages=conversation[msg_idx : msg_idx + 2], user_id=user_id) diff --git a/evaluation/scripts/PrefEval/pref_memos.py b/evaluation/scripts/PrefEval/pref_memos.py index 4a21e3af0..bbe1788b5 100644 --- a/evaluation/scripts/PrefEval/pref_memos.py +++ b/evaluation/scripts/PrefEval/pref_memos.py @@ -12,6 +12,7 @@ from openai import OpenAI from tqdm import tqdm + ROOT_DIR = os.path.dirname( os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) ) @@ -49,7 +50,7 @@ def add_memory_for_line( for idx, _ in enumerate(conversation[::2]): msg_idx = idx * 2 - record_id = f"{lib}_user_pref_eval_{i}_{version}_{str(msg_idx)}" + record_id = f"{lib}_user_pref_eval_{i}_{version}_{msg_idx!s}" if record_id not in success_records: mem_client.add( diff --git a/evaluation/scripts/PrefEval/pref_memu.py b/evaluation/scripts/PrefEval/pref_memu.py index 4c37db7b7..00c411eb7 100644 --- a/evaluation/scripts/PrefEval/pref_memu.py +++ b/evaluation/scripts/PrefEval/pref_memu.py @@ -14,6 +14,7 @@ from openai import OpenAI from tqdm import tqdm + ROOT_DIR = os.path.dirname( os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) ) @@ -56,7 +57,7 @@ def add_memory_for_line( for idx, _ in enumerate(conversation[::2]): msg_idx = idx * 2 - record_id = f"{lib}_user_pref_eval_{i}_{version}_{str(msg_idx)}" + record_id = f"{lib}_user_pref_eval_{i}_{version}_{msg_idx!s}" if record_id not in success_records: mem_client.add( diff --git a/evaluation/scripts/PrefEval/pref_supermemory.py b/evaluation/scripts/PrefEval/pref_supermemory.py index 68963e2af..7386bc462 100644 --- a/evaluation/scripts/PrefEval/pref_supermemory.py +++ b/evaluation/scripts/PrefEval/pref_supermemory.py @@ -12,6 +12,7 @@ from openai import OpenAI from tqdm import tqdm + ROOT_DIR = os.path.dirname( os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) ) @@ -54,7 +55,7 @@ def add_memory_for_line( for idx, _ in enumerate(conversation[::2]): msg_idx = idx * 2 - record_id = f"{lib}_user_pref_eval_{i}_{version}_{str(msg_idx)}" + record_id = f"{lib}_user_pref_eval_{i}_{version}_{msg_idx!s}" if record_id not in success_records: mem_client.add( diff --git a/evaluation/scripts/PrefEval/pref_zep.py b/evaluation/scripts/PrefEval/pref_zep.py index be98c6ba9..8a4d50558 100644 --- a/evaluation/scripts/PrefEval/pref_zep.py +++ b/evaluation/scripts/PrefEval/pref_zep.py @@ -14,6 +14,7 @@ from openai import OpenAI from tqdm import tqdm + ROOT_DIR = os.path.dirname( os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) ) @@ -56,7 +57,7 @@ def add_memory_for_line( for idx, _ in enumerate(conversation[::2]): msg_idx = idx * 2 - record_id = f"{lib}_user_pref_eval_{i}_{version}_{str(msg_idx)}" + record_id = f"{lib}_user_pref_eval_{i}_{version}_{msg_idx!s}" if record_id not in success_records: mem_client.add( diff --git a/evaluation/scripts/personamem/pm_ingestion.py b/evaluation/scripts/personamem/pm_ingestion.py index fdbf43528..b960aa157 100644 --- a/evaluation/scripts/personamem/pm_ingestion.py +++ b/evaluation/scripts/personamem/pm_ingestion.py @@ -10,6 +10,7 @@ from tqdm import tqdm + sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -171,7 +172,9 @@ def ingest_conv(row_data, context, version, conv_idx, frame, success_records, f) client = MemosApiOnlineClient() try: - ingest_session(session=context, user_id=user_id, session_id=conv_idx, frame=frame, client=client) + ingest_session( + session=context, user_id=user_id, session_id=conv_idx, frame=frame, client=client + ) print(f"✅ Ingestion of conversation {conv_idx} completed") print("=" * 80) @@ -187,10 +190,9 @@ def main(frame, version, num_workers=2, clear=False): os.makedirs(f"results/pm/{frame}-{version}/", exist_ok=True) record_file = f"results/pm/{frame}-{version}/success_records.txt" - if clear: - if os.path.exists(record_file): - os.remove(record_file) - print("🧹 Cleared progress records") + if clear and os.path.exists(record_file): + os.remove(record_file) + print("🧹 Cleared progress records") print("\n" + "=" * 80) print(f"🚀 PERSONAMEM INGESTION - {frame.upper()} v{version}".center(80)) @@ -205,15 +207,20 @@ def main(frame, version, num_workers=2, clear=False): success_records = set() if os.path.exists(record_file): - with open(record_file, "r") as f: - success_records = set(line.strip() for line in f) - print(f"📊 Found {len(success_records)} completed conversations, {total_rows - len(success_records)} remaining") + with open(record_file) as f: + success_records = {line.strip() for line in f} + print( + f"📊 Found {len(success_records)} completed conversations, {total_rows - len(success_records)} remaining" + ) start_time = datetime.now() all_data = list(load_rows_with_context(question_csv_path, context_jsonl_path)) - pending_data = [(idx, row_data, context) for idx, (row_data, context) in enumerate(all_data) - if str(idx) not in success_records] + pending_data = [ + (idx, row_data, context) + for idx, (row_data, context) in enumerate(all_data) + if str(idx) not in success_records + ] if not pending_data: print("✅ All conversations have been processed!") @@ -232,16 +239,16 @@ def main(frame, version, num_workers=2, clear=False): conv_idx=idx, frame=frame, success_records=success_records, - f=f + f=f, ) futures.append(future) completed_count = 0 for future in tqdm( - as_completed(futures), total=len(futures), desc="Processing conversations" + as_completed(futures), total=len(futures), desc="Processing conversations" ): try: - result = future.result() + future.result() completed_count += 1 except Exception as exc: print(f"\n❌ Conversation generated an exception: {exc}") @@ -261,13 +268,28 @@ def main(frame, version, num_workers=2, clear=False): if __name__ == "__main__": parser = argparse.ArgumentParser(description="PersonaMem Ingestion Script") - parser.add_argument("--lib", type=str, - choices=["memos-api-online", "mem0", "mem0_graph", "memos-api", "memobase", "memu", - "supermemory", "zep"], - default='memos-api') - parser.add_argument("--version", type=str, default="default", help="Version of the evaluation framework.") - parser.add_argument("--workers", type=int, default=3, help="Number of parallel workers for processing users.") + parser.add_argument( + "--lib", + type=str, + choices=[ + "memos-api-online", + "mem0", + "mem0_graph", + "memos-api", + "memobase", + "memu", + "supermemory", + "zep", + ], + default="memos-api", + ) + parser.add_argument( + "--version", type=str, default="default", help="Version of the evaluation framework." + ) + parser.add_argument( + "--workers", type=int, default=3, help="Number of parallel workers for processing users." + ) parser.add_argument("--clear", action="store_true", help="Clear progress and start fresh") args = parser.parse_args() - main(frame=args.lib, version=args.version, num_workers=args.workers, clear=args.clear) \ No newline at end of file + main(frame=args.lib, version=args.version, num_workers=args.workers, clear=args.clear) diff --git a/evaluation/scripts/personamem/pm_metric.py b/evaluation/scripts/personamem/pm_metric.py index b9d10a576..4c93ec0c6 100644 --- a/evaluation/scripts/personamem/pm_metric.py +++ b/evaluation/scripts/personamem/pm_metric.py @@ -353,12 +353,23 @@ def print_summary(results): parser.add_argument( "--lib", type=str, - choices=["zep", "mem0", "mem0_graph", "memos-api", "memos-api-online", "memobase", "memu", "supermemory"], + choices=[ + "zep", + "mem0", + "mem0_graph", + "memos-api", + "memos-api-online", + "memobase", + "memu", + "supermemory", + ], required=True, help="Memory library to evaluate", default="memos-api", ) - parser.add_argument("--version", type=str, default="default", help="Evaluation framework version") + parser.add_argument( + "--version", type=str, default="default", help="Evaluation framework version" + ) args = parser.parse_args() lib, version = args.lib, args.version diff --git a/evaluation/scripts/personamem/pm_responses.py b/evaluation/scripts/personamem/pm_responses.py index 2e41b4140..171b5af1a 100644 --- a/evaluation/scripts/personamem/pm_responses.py +++ b/evaluation/scripts/personamem/pm_responses.py @@ -10,6 +10,7 @@ from openai import OpenAI from tqdm import tqdm + sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import re @@ -153,9 +154,9 @@ def main(frame, version, num_runs=3, num_workers=4): future_to_user_id[future] = user_id for future in tqdm( - as_completed(future_to_user_id), - total=len(future_to_user_id), - desc="📝 Generating responses", + as_completed(future_to_user_id), + total=len(future_to_user_id), + desc="📝 Generating responses", ): user_id = future_to_user_id[future] try: @@ -184,12 +185,30 @@ def main(frame, version, num_runs=3, num_workers=4): if __name__ == "__main__": parser = argparse.ArgumentParser(description="PersonaMem Response Generation Script") - parser.add_argument("--lib", type=str, - choices=["memos-api-online", "zep", "mem0", "mem0_graph", "memos-api", "memobase", "memu", - "supermemory"], default='memos-api') - parser.add_argument("--version", type=str, default="default", help="Version of the evaluation framework.") - parser.add_argument("--num_runs", type=int, default=3, help="Number of runs for LLM-as-a-Judge evaluation.") - parser.add_argument("--workers", type=int, default=10, help="Number of worker threads to use for processing.") + parser.add_argument( + "--lib", + type=str, + choices=[ + "memos-api-online", + "zep", + "mem0", + "mem0_graph", + "memos-api", + "memobase", + "memu", + "supermemory", + ], + default="memos-api", + ) + parser.add_argument( + "--version", type=str, default="default", help="Version of the evaluation framework." + ) + parser.add_argument( + "--num_runs", type=int, default=3, help="Number of runs for LLM-as-a-Judge evaluation." + ) + parser.add_argument( + "--workers", type=int, default=10, help="Number of worker threads to use for processing." + ) args = parser.parse_args() main(frame=args.lib, version=args.version, num_runs=args.num_runs, num_workers=args.workers) diff --git a/evaluation/scripts/personamem/pm_search.py b/evaluation/scripts/personamem/pm_search.py index 13ed659d2..80a65e09b 100644 --- a/evaluation/scripts/personamem/pm_search.py +++ b/evaluation/scripts/personamem/pm_search.py @@ -3,10 +3,12 @@ import json import os import sys + from collections import defaultdict from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime from time import time + from tqdm import tqdm @@ -232,6 +234,7 @@ def process_user(row_data, conv_idx, frame, version, top_k=20): context, duration_ms = memobase_search(client, question, user_id, top_k) elif frame == "memos-api-online": from utils.client import MemosApiOnlineClient + client = MemosApiOnlineClient() print("🔌 Using memos-api-online client for search...") context, duration_ms = memos_search(client, question, user_id, top_k) @@ -253,7 +256,7 @@ def process_user(row_data, conv_idx, frame, version, top_k=20): os.makedirs(f"results/pm/{frame}-{version}/tmp", exist_ok=True) with open( - f"results/pm/{frame}-{version}/tmp/{frame}_pm_search_results_{conv_idx}.json", "w" + f"results/pm/{frame}-{version}/tmp/{frame}_pm_search_results_{conv_idx}.json", "w" ) as f: json.dump(search_results, f, indent=4) print(f"💾 Search results for conversation {conv_idx} saved...") @@ -304,7 +307,7 @@ def main(frame, version, top_k=20, num_workers=2): } for future in tqdm( - as_completed(future_to_idx), total=len(future_to_idx), desc="Processing conversations" + as_completed(future_to_idx), total=len(future_to_idx), desc="Processing conversations" ): idx = future_to_idx[future] try: @@ -333,13 +336,29 @@ def main(frame, version, top_k=20, num_workers=2): if __name__ == "__main__": parser = argparse.ArgumentParser(description="PersonaMem Search Script") - parser.add_argument("--lib", type=str, - choices=["memos-api-online", "mem0", "mem0_graph", "memos-api", "memobase", "memu", - "supermemory"], - default='memos-api') - parser.add_argument("--version", type=str, default="default", help="Version of the evaluation framework.") - parser.add_argument("--top_k", type=int, default=20, help="Number of top results to retrieve from the search.") - parser.add_argument("--workers", type=int, default=3, help="Number of parallel workers for processing users.") + parser.add_argument( + "--lib", + type=str, + choices=[ + "memos-api-online", + "mem0", + "mem0_graph", + "memos-api", + "memobase", + "memu", + "supermemory", + ], + default="memos-api", + ) + parser.add_argument( + "--version", type=str, default="default", help="Version of the evaluation framework." + ) + parser.add_argument( + "--top_k", type=int, default=20, help="Number of top results to retrieve from the search." + ) + parser.add_argument( + "--workers", type=int, default=3, help="Number of parallel workers for processing users." + ) args = parser.parse_args() diff --git a/evaluation/scripts/run_prefeval_eval.sh b/evaluation/scripts/run_prefeval_eval.sh index 129382ebf..6f5f3b7b0 100755 --- a/evaluation/scripts/run_prefeval_eval.sh +++ b/evaluation/scripts/run_prefeval_eval.sh @@ -143,4 +143,4 @@ fi echo "" echo "--- PrefEval Pipeline completed successfully! ---" -echo "Final results are in $RESPONSE_FILE" \ No newline at end of file +echo "Final results are in $RESPONSE_FILE" diff --git a/src/memos/memories/textual/prefer_text_memory/adder.py b/src/memos/memories/textual/prefer_text_memory/adder.py index 052ae30c2..8d00ae81d 100644 --- a/src/memos/memories/textual/prefer_text_memory/adder.py +++ b/src/memos/memories/textual/prefer_text_memory/adder.py @@ -232,8 +232,12 @@ def _update_memory_fine( need_update = ( need_update if isinstance(need_update, bool) else need_update.lower() == "true" ) - update_item = [mem for mem in retrieved_memories if mem.id == rsp["id"]] - if need_update and update_item: + update_item = ( + [mem for mem in retrieved_memories if mem.id == rsp["id"]] + if rsp and "id" in rsp + else [] + ) + if need_update and update_item and rsp: update_vec_db_item = update_item[0] update_vec_db_item.payload[preference_type] = rsp["new_preference"] update_vec_db_item.payload["updated_at"] = vec_db_item.payload["updated_at"] From 30d0ed4a4c60428a6750dafb337c4d16112359da Mon Sep 17 00:00:00 2001 From: "yuan.wang" Date: Thu, 30 Oct 2025 21:27:54 +0800 Subject: [PATCH 12/21] fast --- src/memos/memories/textual/prefer_text_memory/adder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/memos/memories/textual/prefer_text_memory/adder.py b/src/memos/memories/textual/prefer_text_memory/adder.py index 8d00ae81d..c8eea3cd4 100644 --- a/src/memos/memories/textual/prefer_text_memory/adder.py +++ b/src/memos/memories/textual/prefer_text_memory/adder.py @@ -326,7 +326,7 @@ def _process_single_memory(self, memory: TextualMemoryItem) -> list[str] | str | search_results.sort(key=lambda x: x.score, reverse=True) return self._update_memory( - memory, search_results, collection_name, preference_type, update_mode="fine" + memory, search_results, collection_name, preference_type, update_mode="fast" ) except Exception as e: From 0e531f09e9753600873b3f1fce04019ddb928297 Mon Sep 17 00:00:00 2001 From: "yuan.wang" Date: Fri, 31 Oct 2025 15:18:08 +0800 Subject: [PATCH 13/21] modify pref and adder mode --- .../textual/prefer_text_memory/adder.py | 3 +- src/memos/templates/prefer_complete_prompt.py | 30 ++++++++++++++----- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/memos/memories/textual/prefer_text_memory/adder.py b/src/memos/memories/textual/prefer_text_memory/adder.py index c8eea3cd4..b0bbc0f7b 100644 --- a/src/memos/memories/textual/prefer_text_memory/adder.py +++ b/src/memos/memories/textual/prefer_text_memory/adder.py @@ -1,4 +1,5 @@ import json +import os from abc import ABC, abstractmethod from concurrent.futures import as_completed @@ -326,7 +327,7 @@ def _process_single_memory(self, memory: TextualMemoryItem) -> list[str] | str | search_results.sort(key=lambda x: x.score, reverse=True) return self._update_memory( - memory, search_results, collection_name, preference_type, update_mode="fast" + memory, search_results, collection_name, preference_type, update_mode=os.getenv("PREFERENCE_ADDER_MODE", "fast") ) except Exception as e: diff --git a/src/memos/templates/prefer_complete_prompt.py b/src/memos/templates/prefer_complete_prompt.py index b98e65d54..ec06af27f 100644 --- a/src/memos/templates/prefer_complete_prompt.py +++ b/src/memos/templates/prefer_complete_prompt.py @@ -62,17 +62,24 @@ NAIVE_IMPLICIT_PREFERENCE_EXTRACT_PROMPT = """ You are a preference inference assistant. Please extract **implicit preferences** from the following conversation -(preferences that the user did not explicitly state but can be reasonably inferred from context, behavior, frequency, comparisons, exclusions, or scenario choices). +(preferences that the user did not explicitly state but can be reasonably inferred from their underlying motivations, behavioral patterns, decision-making logic, and latent needs). Notes: -- Implicit preferences refer to user inclinations or choices that are not directly expressed, but can be reasonably inferred from factual cues in the conversation. +- Implicit preferences refer to user inclinations or choices that are not directly expressed, but can be deeply inferred by analyzing: + * **Hidden motivations**: What underlying needs or goals might drive the user's behavior? + * **Behavioral patterns**: What recurring patterns or tendencies can be observed? + * **Decision-making logic**: What reasoning or trade-offs might the user be considering? + * **Latent preferences**: What preferences might the user have but haven't yet articulated? + * **Contextual signals**: What do the user's choices, comparisons, exclusions, or scenario selections reveal about their deeper preferences? - Do not treat explicitly stated preferences as implicit preferences; this prompt is only for inferring preferences that are not directly mentioned. +- Go beyond surface-level facts to understand the user's hidden possibilities and underlying logic. Requirements: 1. Only make inferences when there is sufficient evidence in the conversation; avoid unsupported or far-fetched guesses. 2. Inferred implicit preferences must not conflict with explicit preferences. 3. For implicit_preference: only output the preference statement itself; do not include any extra explanation, reasoning, or confidence information. Put all reasoning and explanation in the reasoning field. -4. If no implicit preference can be reasonably inferred, leave the implicit_preference field empty (do not output anything else). +4. In the reasoning field, explicitly explain the underlying logic and hidden motivations you identified. +5. If no implicit preference can be reasonably inferred, leave the implicit_preference field empty (do not output anything else). Conversation: {qa_pair} @@ -82,7 +89,7 @@ { "implicit_preference": "A concise natural language statement of the implicit preferences reasonably inferred from the conversation, or an empty string", "context_summary": "The corresponding context summary, which is a summary of the corresponding conversation, do not lack any scenario information", - "reasoning": "Briefly explain the reasoning process for the implicit preference" + "reasoning": "Explain the underlying logic, hidden motivations, and behavioral patterns that led to this inference" } ``` Don't output anything except the JSON. @@ -91,17 +98,24 @@ NAIVE_IMPLICIT_PREFERENCE_EXTRACT_PROMPT_ZH = """ 你是一个偏好推理助手。请从以下对话中提取**隐式偏好** -（用户没有明确表述，但可以从上下文、行为、频率、比较、排除或场景选择中合理推断出的偏好）。 +（用户没有明确表述，但可以通过分析其潜在动机、行为模式、决策逻辑和隐藏需求深度推断出的偏好）。 注意事项： -- 隐式偏好是指用户未直接表达，但可以从对话中的事实线索合理推断出的倾向或选择。 +- 隐式偏好是指用户未直接表达，但可以通过深入分析以下方面推断出的倾向或选择： + * **隐藏动机**：什么样的潜在需求或目标可能驱动用户的行为？ + * **行为模式**：可以观察到什么样的重复模式或倾向？ + * **决策逻辑**：用户可能在考虑什么样的推理或权衡？ + * **潜在偏好**：用户可能有但尚未明确表达的偏好是什么？ + * **情境信号**：用户的选择、比较、排除或场景选择揭示了什么样的深层偏好？ - 不要将明确陈述的偏好视为隐式偏好；此提示仅用于推断未直接提及的偏好。 +- 超越表面事实，理解用户的隐藏可能性和背后的逻辑。 要求： 1. 仅在对话中有充分证据时进行推断；避免无根据或牵强的猜测。 2. 推断的隐式偏好不得与显式偏好冲突。 3. 对于 implicit_preference：仅输出偏好陈述本身；不要包含任何额外的解释、推理或置信度信息。将所有推理和解释放在 reasoning 字段中。 -4. 如果无法合理推断出隐式偏好，则将 implicit_preference 字段留空（不要输出其他任何内容）。 +4. 在 reasoning 字段中，明确解释你识别出的底层逻辑和隐藏动机。 +5. 如果无法合理推断出隐式偏好，则将 implicit_preference 字段留空（不要输出其他任何内容）。 对话： {qa_pair} @@ -111,7 +125,7 @@ { "implicit_preference": "从对话中合理推断出的隐式偏好的简洁自然语言陈述，或空字符串", "context_summary": "对应的上下文摘要，即对应对话的摘要，不要遗漏任何场景信息", - "reasoning": "简要解释隐式偏好的推理过程" + "reasoning": "解释推断出该偏好的底层逻辑、隐藏动机和行为模式" } ``` 除JSON外不要输出任何其他内容。 From f0f463c042e1a468cf2a9deb3b5b0ab0b6c51e89 Mon Sep 17 00:00:00 2001 From: "yuan.wang" Date: Fri, 31 Oct 2025 15:19:36 +0800 Subject: [PATCH 14/21] modify code --- src/memos/memories/textual/prefer_text_memory/adder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/memos/memories/textual/prefer_text_memory/adder.py b/src/memos/memories/textual/prefer_text_memory/adder.py index b0bbc0f7b..204a2fbd3 100644 --- a/src/memos/memories/textual/prefer_text_memory/adder.py +++ b/src/memos/memories/textual/prefer_text_memory/adder.py @@ -288,7 +288,7 @@ def _update_memory( retrieved_memories: list[MilvusVecDBItem], collection_name: str, preference_type: str, - update_mode: str = "fine", + update_mode: str = "fast", ) -> list[str] | str | None: """Update the memory. Args: From a47af8306a3ee7a505a3e96f1721025ae86445d8 Mon Sep 17 00:00:00 2001 From: "yuan.wang" Date: Fri, 31 Oct 2025 15:25:35 +0800 Subject: [PATCH 15/21] make pre_commit --- src/memos/memories/textual/prefer_text_memory/adder.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/memos/memories/textual/prefer_text_memory/adder.py b/src/memos/memories/textual/prefer_text_memory/adder.py index 204a2fbd3..ce0282a23 100644 --- a/src/memos/memories/textual/prefer_text_memory/adder.py +++ b/src/memos/memories/textual/prefer_text_memory/adder.py @@ -327,7 +327,11 @@ def _process_single_memory(self, memory: TextualMemoryItem) -> list[str] | str | search_results.sort(key=lambda x: x.score, reverse=True) return self._update_memory( - memory, search_results, collection_name, preference_type, update_mode=os.getenv("PREFERENCE_ADDER_MODE", "fast") + memory, + search_results, + collection_name, + preference_type, + update_mode=os.getenv("PREFERENCE_ADDER_MODE", "fast"), ) except Exception as e: From ff15da753581d4930a339e0814a9991af8ecb7df Mon Sep 17 00:00:00 2001 From: "yuan.wang" Date: Fri, 31 Oct 2025 23:03:17 +0800 Subject: [PATCH 16/21] fix pref_string for memos online api --- evaluation/scripts/utils/client.py | 35 +++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/evaluation/scripts/utils/client.py b/evaluation/scripts/utils/client.py index ea0caa307..50ea1e201 100644 --- a/evaluation/scripts/utils/client.py +++ b/evaluation/scripts/utils/client.py @@ -234,6 +234,8 @@ def search(self, query, user_id, top_k): "user_id": user_id, "memory_limit_number": top_k, "mode": os.getenv("SEARCH_MODE", "fast"), + "include_preference": True, + "pref_top_k": 6, } ) @@ -243,10 +245,23 @@ def search(self, query, user_id, top_k): response = requests.request("POST", url, data=payload, headers=self.headers) assert response.status_code == 200, response.text assert json.loads(response.text)["message"] == "ok", response.text - res = json.loads(response.text)["data"]["memory_detail_list"] - for i in res: + text_mem_res = json.loads(response.text)["data"]["memory_detail_list"] + pref_mem_res = json.loads(response.text)["data"]["preference_detail_list"] + for i in text_mem_res: i.update({"memory": i.pop("memory_value")}) - return {"text_mem": [{"memories": res}], "pref_str": ""} + + explicit_prefs = [p['preference'] for p in pref_mem_res if p.get('preference_type', '') == 'explicit_preference'] + implicit_prefs = [p['preference'] for p in pref_mem_res if p.get('preference_type', '') == 'implicit_preference'] + + pref_parts = [] + if explicit_prefs: + pref_parts.append("Explicit Preference:\n" + "\n".join(f"{i + 1}. {p}" for i, p in enumerate(explicit_prefs))) + if implicit_prefs: + pref_parts.append("Implicit Preference:\n" + "\n".join(f"{i + 1}. {p}" for i, p in enumerate(implicit_prefs))) + + pref_string = "\n".join(pref_parts) + + return {"text_mem": [{"memories": text_mem_res}], "pref_string": pref_string} except Exception as e: if attempt < max_retries - 1: time.sleep(2**attempt) @@ -336,19 +351,23 @@ def wait_for_completion(self, task_id): if __name__ == "__main__": messages = [ - {"role": "user", "content": "杭州西湖有什么好玩的"}, - {"role": "assistant", "content": "杭州西湖有好多松鼠，还有断桥"}, + # {"role": "user", "content": "杭州西湖有什么好玩的,我喜欢动物"}, + # {"role": "assistant", "content": "杭州西湖有好多松鼠, 你喜欢动物的话可以去看松鼠"}, + {"role": "user", "content": "我暑假定好去广州旅游，住宿的话有哪些连锁酒店可选？"}, + {"role": "assistant", "content": "您可以考虑【七天、全季、希尔顿】等等"}, + {"role": "user", "content": "我选七天"}, + {"role": "assistant", "content": "好的，有其他问题再问我。"}, ] - user_id = "test_user" + user_id = "test_user2" iso_date = "2023-05-01T00:00:00.000Z" timestamp = 1682899200 query = "杭州西湖有什么" top_k = 5 # MEMOS-API - client = MemosApiClient() + client = MemosApiOnlineClient() for m in messages: m["created_at"] = iso_date - client.add(messages, user_id, user_id) + # client.add(messages, user_id, user_id) memories = client.search(query, user_id, top_k) print(memories) From 7a272cead27afcc1e2db1483d4e27798b73c59f3 Mon Sep 17 00:00:00 2001 From: "yuan.wang" Date: Fri, 31 Oct 2025 23:11:35 +0800 Subject: [PATCH 17/21] modify code --- evaluation/scripts/utils/client.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/evaluation/scripts/utils/client.py b/evaluation/scripts/utils/client.py index 50ea1e201..ed45c3386 100644 --- a/evaluation/scripts/utils/client.py +++ b/evaluation/scripts/utils/client.py @@ -351,23 +351,19 @@ def wait_for_completion(self, task_id): if __name__ == "__main__": messages = [ - # {"role": "user", "content": "杭州西湖有什么好玩的,我喜欢动物"}, - # {"role": "assistant", "content": "杭州西湖有好多松鼠, 你喜欢动物的话可以去看松鼠"}, - {"role": "user", "content": "我暑假定好去广州旅游，住宿的话有哪些连锁酒店可选？"}, - {"role": "assistant", "content": "您可以考虑【七天、全季、希尔顿】等等"}, - {"role": "user", "content": "我选七天"}, - {"role": "assistant", "content": "好的，有其他问题再问我。"}, + {"role": "user", "content": "杭州西湖有什么好玩的,我喜欢动物"}, + {"role": "assistant", "content": "杭州西湖有好多松鼠, 你喜欢动物的话可以去看松鼠"}, ] - user_id = "test_user2" + user_id = "test_user" iso_date = "2023-05-01T00:00:00.000Z" timestamp = 1682899200 query = "杭州西湖有什么" top_k = 5 # MEMOS-API - client = MemosApiOnlineClient() + client = MemosApiClient() for m in messages: m["created_at"] = iso_date - # client.add(messages, user_id, user_id) + client.add(messages, user_id, user_id) memories = client.search(query, user_id, top_k) print(memories) From 5876ddd4d78919f2de9d7b5a9e33a006d8431696 Mon Sep 17 00:00:00 2001 From: "yuan.wang" Date: Fri, 31 Oct 2025 23:13:10 +0800 Subject: [PATCH 18/21] modify code --- evaluation/scripts/utils/client.py | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/evaluation/scripts/utils/client.py b/evaluation/scripts/utils/client.py index ed45c3386..04a7bfb1c 100644 --- a/evaluation/scripts/utils/client.py +++ b/evaluation/scripts/utils/client.py @@ -250,17 +250,31 @@ def search(self, query, user_id, top_k): for i in text_mem_res: i.update({"memory": i.pop("memory_value")}) - explicit_prefs = [p['preference'] for p in pref_mem_res if p.get('preference_type', '') == 'explicit_preference'] - implicit_prefs = [p['preference'] for p in pref_mem_res if p.get('preference_type', '') == 'implicit_preference'] - + explicit_prefs = [ + p["preference"] + for p in pref_mem_res + if p.get("preference_type", "") == "explicit_preference" + ] + implicit_prefs = [ + p["preference"] + for p in pref_mem_res + if p.get("preference_type", "") == "implicit_preference" + ] + pref_parts = [] if explicit_prefs: - pref_parts.append("Explicit Preference:\n" + "\n".join(f"{i + 1}. {p}" for i, p in enumerate(explicit_prefs))) + pref_parts.append( + "Explicit Preference:\n" + + "\n".join(f"{i + 1}. {p}" for i, p in enumerate(explicit_prefs)) + ) if implicit_prefs: - pref_parts.append("Implicit Preference:\n" + "\n".join(f"{i + 1}. {p}" for i, p in enumerate(implicit_prefs))) - + pref_parts.append( + "Implicit Preference:\n" + + "\n".join(f"{i + 1}. {p}" for i, p in enumerate(implicit_prefs)) + ) + pref_string = "\n".join(pref_parts) - + return {"text_mem": [{"memories": text_mem_res}], "pref_string": pref_string} except Exception as e: if attempt < max_retries - 1: @@ -352,7 +366,7 @@ def wait_for_completion(self, task_id): if __name__ == "__main__": messages = [ {"role": "user", "content": "杭州西湖有什么好玩的,我喜欢动物"}, - {"role": "assistant", "content": "杭州西湖有好多松鼠, 你喜欢动物的话可以去看松鼠"}, + {"role": "assistant", "content": "杭州西湖有好多松鼠，还有断桥"}, ] user_id = "test_user" iso_date = "2023-05-01T00:00:00.000Z" From 670a92ecc8429a62830a066e690eced308b34684 Mon Sep 17 00:00:00 2001 From: "yuan.wang" Date: Fri, 31 Oct 2025 23:13:48 +0800 Subject: [PATCH 19/21] modify code --- evaluation/scripts/utils/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evaluation/scripts/utils/client.py b/evaluation/scripts/utils/client.py index 04a7bfb1c..3b02dcd0c 100644 --- a/evaluation/scripts/utils/client.py +++ b/evaluation/scripts/utils/client.py @@ -365,7 +365,7 @@ def wait_for_completion(self, task_id): if __name__ == "__main__": messages = [ - {"role": "user", "content": "杭州西湖有什么好玩的,我喜欢动物"}, + {"role": "user", "content": "杭州西湖有什么好玩的"}, {"role": "assistant", "content": "杭州西湖有好多松鼠，还有断桥"}, ] user_id = "test_user" From 7e485bf320c35111d119aee1dff21f5911d69e20 Mon Sep 17 00:00:00 2001 From: "yuan.wang" Date: Fri, 31 Oct 2025 23:29:32 +0800 Subject: [PATCH 20/21] pre comimt --- evaluation/scripts/utils/client.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/evaluation/scripts/utils/client.py b/evaluation/scripts/utils/client.py index 3b02dcd0c..815bc5d8d 100644 --- a/evaluation/scripts/utils/client.py +++ b/evaluation/scripts/utils/client.py @@ -250,6 +250,13 @@ def search(self, query, user_id, top_k): for i in text_mem_res: i.update({"memory": i.pop("memory_value")}) + pref_instructions = """ +# Note: +Plaintext memory are summaries of facts, while preference memories are summaries of user preferences. +Your response must not violate any of the user's preferences, whether explicit or implicit, and briefly explain why you answer this way to avoid conflicts. +When encountering preference conflicts, the priority is: explicit preference > implicit preference > plaintext memory. +""" + explicit_prefs = [ p["preference"] for p in pref_mem_res @@ -273,7 +280,7 @@ def search(self, query, user_id, top_k): + "\n".join(f"{i + 1}. {p}" for i, p in enumerate(implicit_prefs)) ) - pref_string = "\n".join(pref_parts) + pref_string = "\n".join(pref_parts) + pref_instructions return {"text_mem": [{"memories": text_mem_res}], "pref_string": pref_string} except Exception as e: From 785c96d18be7aea5d17cc3636962cbe612d8dda1 Mon Sep 17 00:00:00 2001 From: "yuan.wang" Date: Sat, 1 Nov 2025 09:16:59 +0800 Subject: [PATCH 21/21] modify code --- evaluation/scripts/utils/client.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/evaluation/scripts/utils/client.py b/evaluation/scripts/utils/client.py index 815bc5d8d..9aa527903 100644 --- a/evaluation/scripts/utils/client.py +++ b/evaluation/scripts/utils/client.py @@ -247,16 +247,10 @@ def search(self, query, user_id, top_k): assert json.loads(response.text)["message"] == "ok", response.text text_mem_res = json.loads(response.text)["data"]["memory_detail_list"] pref_mem_res = json.loads(response.text)["data"]["preference_detail_list"] + preference_note = json.loads(response.text)["data"]["preference_note"] for i in text_mem_res: i.update({"memory": i.pop("memory_value")}) - pref_instructions = """ -# Note: -Plaintext memory are summaries of facts, while preference memories are summaries of user preferences. -Your response must not violate any of the user's preferences, whether explicit or implicit, and briefly explain why you answer this way to avoid conflicts. -When encountering preference conflicts, the priority is: explicit preference > implicit preference > plaintext memory. -""" - explicit_prefs = [ p["preference"] for p in pref_mem_res @@ -280,7 +274,7 @@ def search(self, query, user_id, top_k): + "\n".join(f"{i + 1}. {p}" for i, p in enumerate(implicit_prefs)) ) - pref_string = "\n".join(pref_parts) + pref_instructions + pref_string = "\n".join(pref_parts) + preference_note return {"text_mem": [{"memories": text_mem_res}], "pref_string": pref_string} except Exception as e: