From e63bdc58a765f58b110019bc8897c12579594e6d Mon Sep 17 00:00:00 2001 From: Alex Yang Date: Fri, 30 Aug 2024 15:58:30 -0700 Subject: [PATCH 01/13] feat: support convert from python side --- packages/core/package.json | 1 + packages/core/src/schema/index.ts | 1 + packages/core/src/schema/python.ts | 72 +++++++++++++++++++++++++ packages/core/tests/schema-node.test.ts | 68 ++++++++++++++++++++++- pnpm-lock.yaml | 38 +++++++------ 5 files changed, 159 insertions(+), 21 deletions(-) create mode 100644 packages/core/src/schema/python.ts diff --git a/packages/core/package.json b/packages/core/package.json index 4a51d2c15d..c717403532 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -132,6 +132,7 @@ "devDependencies": { "ajv": "^8.17.1", "bunchee": "5.3.2", + "change-case": "^5.4.4", "natural": "^8.0.1" }, "dependencies": { diff --git a/packages/core/src/schema/index.ts b/packages/core/src/schema/index.ts index a38ba9975b..10be838720 100644 --- a/packages/core/src/schema/index.ts +++ b/packages/core/src/schema/index.ts @@ -1,4 +1,5 @@ export * from "./node"; +export { fromPythonDocStore } from "./python"; export { FileReader, TransformComponent, type BaseReader } from "./type"; export { EngineResponse } from "./type/engine–response"; export * from "./zod"; diff --git a/packages/core/src/schema/python.ts b/packages/core/src/schema/python.ts new file mode 100644 index 0000000000..c5bced815d --- /dev/null +++ b/packages/core/src/schema/python.ts @@ -0,0 +1,72 @@ +/** + * Python adapter for the schema. + */ +import { jsonToNode, ObjectType } from "./node"; + +export const TYPE_KEY = "__type__"; +export const DATA_KEY = "__data__"; + +async function camelCaseJson(json: Record) { + const { camelCase } = await import("change-case"); + return Object.entries(json).reduce( + (acc, [key, value]) => { + acc[ + camelCase(key, { + suffixCharacters: "_", + }) + ] = value; + return acc; + }, + {} as Record, + ); +} + +const PYTHON_TO_JS_TYPE_MAP = { + "1": ObjectType.TEXT, + "2": ObjectType.IMAGE, + "3": ObjectType.INDEX, + "4": ObjectType.DOCUMENT, +}; + +async function fromPythonImpl(data: Record) { + const convertedJson = await camelCaseJson(data); + if (convertedJson.relationships) { + for (const [key, value] of Object.entries(convertedJson.relationships)) { + if (typeof value === "object" && value !== null) { + convertedJson.relationships[key] = await camelCaseJson(value); + } else if (Array.isArray(value)) { + convertedJson.relationships[key] = await Promise.all( + value.map((v) => camelCaseJson(v)), + ); + } + } + } + return convertedJson; +} + +export async function fromPythonDocStore({ + [TYPE_KEY]: type, + [DATA_KEY]: data, +}: { + [TYPE_KEY]: string; + [DATA_KEY]: Record; +}) { + if (!(type in PYTHON_TO_JS_TYPE_MAP)) { + throw new Error(""); + } + const objectType = + PYTHON_TO_JS_TYPE_MAP[type as keyof typeof PYTHON_TO_JS_TYPE_MAP]; + const convertedJson = fromPythonImpl(data); + return jsonToNode(convertedJson, objectType); +} + +export async function fromPythonVectorStore(json: Record) { + const convertedJson = await fromPythonImpl(json); + const type = convertedJson["metadata"]?.["_node_type"]; + if (!(type in PYTHON_TO_JS_TYPE_MAP)) { + throw new Error(""); + } + const objectType = + PYTHON_TO_JS_TYPE_MAP[type as keyof typeof PYTHON_TO_JS_TYPE_MAP]; + return jsonToNode(convertedJson, objectType); +} diff --git a/packages/core/tests/schema-node.test.ts b/packages/core/tests/schema-node.test.ts index 7827fcbd68..472ced3e40 100644 --- a/packages/core/tests/schema-node.test.ts +++ b/packages/core/tests/schema-node.test.ts @@ -1,6 +1,72 @@ -import { Document, TextNode } from "@llamaindex/core/schema"; +import { + Document, + ObjectType, + TextNode, + fromPythonDocStore, +} from "@llamaindex/core/schema"; import { beforeEach, describe, expect, test } from "vitest"; +describe("Python", () => { + test("from python doc store", async () => { + const node = await fromPythonDocStore({ + __data__: { + id_: "e86be4a7-2ad0-4c3c-937b-3140f562e7a7", + embedding: null, + metadata: {}, + excluded_embed_metadata_keys: [], + excluded_llm_metadata_keys: [], + relationships: { + "1": { + node_id: "e1fe8fd0-f470-40cd-bc2e-be3a220cef94", + node_type: "4", + metadata: {}, + hash: "191a8fdcf068d3ac831da23cde07a92efe1432243c7f628d1009aa2ecdf6cb03", + class_name: "RelatedNodeInfo", + }, + }, + text: "This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test.", + mimetype: "text/plain", + start_char_idx: 0, + end_char_idx: 1599, + text_template: "{metadata_str}\n\n{content}", + metadata_template: "{key}: {value}", + metadata_seperator: "\n", + class_name: "TextNode", + }, + __type__: "1", + }); + expect(node).toMatchInlineSnapshot(` + { + "embedding": null, + "endCharIdx": 1599, + "excludedEmbedMetadataKeys": [], + "excludedLlmMetadataKeys": [], + "hash": "e+X0sEK9rsmaY1GwfOQnloy69/UrsmG7mhihpwPPfS4=", + "id_": "e86be4a7-2ad0-4c3c-937b-3140f562e7a7", + "metadata": {}, + "metadataSeparator": " + ", + "relationships": { + "1": { + "className": "RelatedNodeInfo", + "hash": "191a8fdcf068d3ac831da23cde07a92efe1432243c7f628d1009aa2ecdf6cb03", + "metadata": {}, + "nodeId": "e1fe8fd0-f470-40cd-bc2e-be3a220cef94", + "nodeType": "4", + }, + }, + "text": "This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test.", + "textTemplate": "{metadata_str} + + {content}", + "type": "TEXT", + } + `); + expect(node.id_).toBe("e86be4a7-2ad0-4c3c-937b-3140f562e7a7"); + expect(node.type).toBe(ObjectType.TEXT); + }); +}); + describe("Document", () => { let document: Document; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0dc87238ce..e2da5993b2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -162,7 +162,7 @@ importers: version: link:../packages/llamaindex mongodb: specifier: ^6.7.0 - version: 6.8.0(@aws-sdk/credential-providers@3.637.0(@aws-sdk/client-sso-oidc@3.637.0(@aws-sdk/client-sts@3.637.0))) + version: 6.8.0(@aws-sdk/credential-providers@3.637.0) pathe: specifier: ^1.1.2 version: 1.1.2 @@ -380,6 +380,9 @@ importers: bunchee: specifier: 5.3.2 version: 5.3.2(typescript@5.5.4) + change-case: + specifier: ^5.4.4 + version: 5.4.4 natural: specifier: ^8.0.1 version: 8.0.1(@aws-sdk/credential-providers@3.637.0) @@ -567,7 +570,7 @@ importers: version: 2.0.0 mongodb: specifier: ^6.7.0 - version: 6.8.0(@aws-sdk/credential-providers@3.637.0(@aws-sdk/client-sso-oidc@3.637.0(@aws-sdk/client-sts@3.637.0))) + version: 6.8.0(@aws-sdk/credential-providers@3.637.0) notion-md-crawler: specifier: ^1.0.0 version: 1.0.0(encoding@0.1.13) @@ -4927,6 +4930,9 @@ packages: resolution: {integrity: sha512-dLitG79d+GV1Nb/VYcCDFivJeK1hiukt9QjRNVOsUtTy1rR1YJsmpGGTZ3qJos+uw7WmWF4wUwBd9jxjocFC2w==} engines: {node: ^12.17.0 || ^14.13 || >=16.0.0} + change-case@5.4.4: + resolution: {integrity: sha512-HRQyTk2/YPEkt9TnUPbOpr64Uw3KOicFWPVBb+xiHvd6eBx/qPr9xqfBFDT8P2vWsvvz4jbEkfDe71W3VyNu2w==} + char-regex@1.0.2: resolution: {integrity: sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==} engines: {node: '>=10'} @@ -11920,7 +11926,7 @@ snapshots: '@babel/helpers': 7.25.6 '@babel/parser': 7.25.6 '@babel/template': 7.25.0 - '@babel/traverse': 7.25.6 + '@babel/traverse': 7.23.2 '@babel/types': 7.25.6 convert-source-map: 2.0.0 debug: 4.3.6 @@ -11964,7 +11970,7 @@ snapshots: '@babel/helper-optimise-call-expression': 7.24.7 '@babel/helper-replace-supers': 7.25.0(@babel/core@7.25.2) '@babel/helper-skip-transparent-expression-wrappers': 7.24.7 - '@babel/traverse': 7.25.6 + '@babel/traverse': 7.23.2 semver: 6.3.1 transitivePeerDependencies: - supports-color @@ -12002,7 +12008,7 @@ snapshots: '@babel/helper-member-expression-to-functions@7.24.8': dependencies: - '@babel/traverse': 7.25.6 + '@babel/traverse': 7.23.2 '@babel/types': 7.25.6 transitivePeerDependencies: - supports-color @@ -12035,7 +12041,7 @@ snapshots: '@babel/core': 7.25.2 '@babel/helper-annotate-as-pure': 7.24.7 '@babel/helper-wrap-function': 7.25.0 - '@babel/traverse': 7.25.6 + '@babel/traverse': 7.23.2 transitivePeerDependencies: - supports-color @@ -12044,7 +12050,7 @@ snapshots: '@babel/core': 7.25.2 '@babel/helper-member-expression-to-functions': 7.24.8 '@babel/helper-optimise-call-expression': 7.24.7 - '@babel/traverse': 7.25.6 + '@babel/traverse': 7.23.2 transitivePeerDependencies: - supports-color @@ -12057,7 +12063,7 @@ snapshots: '@babel/helper-skip-transparent-expression-wrappers@7.24.7': dependencies: - '@babel/traverse': 7.25.6 + '@babel/traverse': 7.23.2 '@babel/types': 7.25.6 transitivePeerDependencies: - supports-color @@ -16565,6 +16571,8 @@ snapshots: chalk@5.3.0: {} + change-case@5.4.4: {} + char-regex@1.0.2: {} character-entities-html4@2.1.0: {} @@ -17719,16 +17727,6 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-module-utils@2.8.2(@typescript-eslint/parser@8.3.0(eslint@8.57.0)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint@8.57.0): - dependencies: - debug: 3.2.7 - optionalDependencies: - '@typescript-eslint/parser': 8.3.0(eslint@8.57.0)(typescript@5.5.4) - eslint: 8.57.0 - eslint-import-resolver-node: 0.3.9 - transitivePeerDependencies: - - supports-color - eslint-plugin-import@2.29.1(@typescript-eslint/parser@8.3.0(eslint@8.57.0)(typescript@5.5.4))(eslint@8.57.0): dependencies: array-includes: 3.1.8 @@ -17739,7 +17737,7 @@ snapshots: doctrine: 2.1.0 eslint: 8.57.0 eslint-import-resolver-node: 0.3.9 - eslint-module-utils: 2.8.2(@typescript-eslint/parser@8.3.0(eslint@8.57.0)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint@8.57.0) + eslint-module-utils: 2.8.2(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.5.4))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0) hasown: 2.0.2 is-core-module: 2.15.1 is-glob: 4.0.3 @@ -20467,7 +20465,7 @@ snapshots: optionalDependencies: '@aws-sdk/credential-providers': 3.637.0(@aws-sdk/client-sso-oidc@3.637.0(@aws-sdk/client-sts@3.637.0)) - mongodb@6.8.0(@aws-sdk/credential-providers@3.637.0(@aws-sdk/client-sso-oidc@3.637.0(@aws-sdk/client-sts@3.637.0))): + mongodb@6.8.0(@aws-sdk/credential-providers@3.637.0): dependencies: '@mongodb-js/saslprep': 1.1.7 bson: 6.8.0 From a0df6a07bc8e2fd70285c0749015ababf85f9e8e Mon Sep 17 00:00:00 2001 From: Alex Yang Date: Fri, 30 Aug 2024 16:11:11 -0700 Subject: [PATCH 02/13] fix: replace --- packages/core/src/schema/index.ts | 2 +- packages/core/src/schema/node.ts | 10 +++---- packages/core/src/schema/python.ts | 14 +++++---- .../src/ingestion/IngestionCache.ts | 2 +- .../src/storage/docStore/KVDocumentStore.ts | 2 +- .../llamaindex/src/storage/docStore/utils.ts | 30 ++----------------- .../src/storage/vectorStore/utils.ts | 2 +- 7 files changed, 19 insertions(+), 43 deletions(-) diff --git a/packages/core/src/schema/index.ts b/packages/core/src/schema/index.ts index 10be838720..e99243f7c7 100644 --- a/packages/core/src/schema/index.ts +++ b/packages/core/src/schema/index.ts @@ -1,5 +1,5 @@ export * from "./node"; -export { fromPythonDocStore } from "./python"; +export { DATA_KEY, TYPE_KEY, fromPythonDocStore, type DocJson } from "./python"; export { FileReader, TransformComponent, type BaseReader } from "./type"; export { EngineResponse } from "./type/engine–response"; export * from "./zod"; diff --git a/packages/core/src/schema/node.ts b/packages/core/src/schema/node.ts index 50ab9a9733..f000da3731 100644 --- a/packages/core/src/schema/node.ts +++ b/packages/core/src/schema/node.ts @@ -10,11 +10,11 @@ export enum NodeRelationship { } export enum ObjectType { - TEXT = "TEXT", - IMAGE = "IMAGE", - INDEX = "INDEX", - DOCUMENT = "DOCUMENT", - IMAGE_DOCUMENT = "IMAGE_DOCUMENT", + TEXT = "1", + IMAGE = "2", + INDEX = "3", + DOCUMENT = "4", + IMAGE_DOCUMENT = "5", // Python side doesn't have this enum } export enum MetadataMode { diff --git a/packages/core/src/schema/python.ts b/packages/core/src/schema/python.ts index c5bced815d..b0c7cf9d4d 100644 --- a/packages/core/src/schema/python.ts +++ b/packages/core/src/schema/python.ts @@ -28,6 +28,11 @@ const PYTHON_TO_JS_TYPE_MAP = { "4": ObjectType.DOCUMENT, }; +export type DocJson = { + [TYPE_KEY]: string; + [DATA_KEY]: string; +}; + async function fromPythonImpl(data: Record) { const convertedJson = await camelCaseJson(data); if (convertedJson.relationships) { @@ -47,20 +52,17 @@ async function fromPythonImpl(data: Record) { export async function fromPythonDocStore({ [TYPE_KEY]: type, [DATA_KEY]: data, -}: { - [TYPE_KEY]: string; - [DATA_KEY]: Record; -}) { +}: DocJson) { if (!(type in PYTHON_TO_JS_TYPE_MAP)) { throw new Error(""); } const objectType = PYTHON_TO_JS_TYPE_MAP[type as keyof typeof PYTHON_TO_JS_TYPE_MAP]; - const convertedJson = fromPythonImpl(data); + const convertedJson = fromPythonImpl(JSON.parse(data)); return jsonToNode(convertedJson, objectType); } -export async function fromPythonVectorStore(json: Record) { +export async function fromPythonNode(json: Record) { const convertedJson = await fromPythonImpl(json); const type = convertedJson["metadata"]?.["_node_type"]; if (!(type in PYTHON_TO_JS_TYPE_MAP)) { diff --git a/packages/llamaindex/src/ingestion/IngestionCache.ts b/packages/llamaindex/src/ingestion/IngestionCache.ts index 353e565f85..44bfd69a25 100644 --- a/packages/llamaindex/src/ingestion/IngestionCache.ts +++ b/packages/llamaindex/src/ingestion/IngestionCache.ts @@ -63,6 +63,6 @@ export class IngestionCache { if (!json || !json[this.nodesKey] || !Array.isArray(json[this.nodesKey])) { return undefined; } - return json[this.nodesKey].map((doc: any) => jsonToDoc(doc)); + return Promise.all(json[this.nodesKey].map((doc: any) => jsonToDoc(doc))); } } diff --git a/packages/llamaindex/src/storage/docStore/KVDocumentStore.ts b/packages/llamaindex/src/storage/docStore/KVDocumentStore.ts index b239c4c69b..b563197d76 100644 --- a/packages/llamaindex/src/storage/docStore/KVDocumentStore.ts +++ b/packages/llamaindex/src/storage/docStore/KVDocumentStore.ts @@ -29,7 +29,7 @@ export class KVDocumentStore extends BaseDocumentStore { for (const key in jsonDict) { const value = jsonDict[key]; if (isValidDocJson(value)) { - docs[key] = jsonToDoc(value); + docs[key] = await jsonToDoc(value); } else { console.warn(`Invalid JSON for docId ${key}`); } diff --git a/packages/llamaindex/src/storage/docStore/utils.ts b/packages/llamaindex/src/storage/docStore/utils.ts index 1735b81eea..d79426c48b 100644 --- a/packages/llamaindex/src/storage/docStore/utils.ts +++ b/packages/llamaindex/src/storage/docStore/utils.ts @@ -1,5 +1,5 @@ import type { BaseNode } from "@llamaindex/core/schema"; -import { Document, ObjectType, TextNode } from "@llamaindex/core/schema"; +import { ObjectType, fromPythonDocStore } from "@llamaindex/core/schema"; const TYPE_KEY = "__type__"; const DATA_KEY = "__data__"; @@ -25,30 +25,4 @@ export function docToJson(doc: BaseNode): DocJson { }; } -export function jsonToDoc(docDict: DocJson): BaseNode { - const docType = docDict[TYPE_KEY]; - const dataDict = JSON.parse(docDict[DATA_KEY]); - let doc: BaseNode; - - if (docType === ObjectType.DOCUMENT) { - doc = new Document({ - text: dataDict.text, - id_: dataDict.id_, - embedding: dataDict.embedding, - hash: dataDict.hash, - metadata: dataDict.metadata, - }); - } else if (docType === ObjectType.TEXT) { - doc = new TextNode({ - text: dataDict.text, - id_: dataDict.id_, - hash: dataDict.hash, - metadata: dataDict.metadata, - relationships: dataDict.relationships, - }); - } else { - throw new Error(`Unknown doc type: ${docType}`); - } - - return doc; -} +export const jsonToDoc = fromPythonDocStore; diff --git a/packages/llamaindex/src/storage/vectorStore/utils.ts b/packages/llamaindex/src/storage/vectorStore/utils.ts index da40eeb9b8..9fa6d14ae9 100644 --- a/packages/llamaindex/src/storage/vectorStore/utils.ts +++ b/packages/llamaindex/src/storage/vectorStore/utils.ts @@ -29,7 +29,7 @@ export function nodeToMetadata( } metadata["_node_content"] = JSON.stringify(rest); - metadata["_node_type"] = node.constructor.name.replace("_", ""); // remove leading underscore to be compatible with Python + metadata["_node_type"] = node.type; metadata["document_id"] = node.sourceNode?.nodeId || "None"; metadata["doc_id"] = node.sourceNode?.nodeId || "None"; From 41ad521ccbaaa1ee5f2b6d56dcfdaeff174ee822 Mon Sep 17 00:00:00 2001 From: Alex Yang Date: Fri, 30 Aug 2024 21:50:27 -0700 Subject: [PATCH 03/13] feat: add test --- .../src/storage/vectorStore/utils.ts | 2 +- packages/llamaindex/tests/VectorStore.test.ts | 41 ++++++++++++++++++- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/packages/llamaindex/src/storage/vectorStore/utils.ts b/packages/llamaindex/src/storage/vectorStore/utils.ts index 9fa6d14ae9..2484d17661 100644 --- a/packages/llamaindex/src/storage/vectorStore/utils.ts +++ b/packages/llamaindex/src/storage/vectorStore/utils.ts @@ -29,7 +29,7 @@ export function nodeToMetadata( } metadata["_node_content"] = JSON.stringify(rest); - metadata["_node_type"] = node.type; + metadata["_node_type"] = node.constructor.name.replace("_", ""); metadata["document_id"] = node.sourceNode?.nodeId || "None"; metadata["doc_id"] = node.sourceNode?.nodeId || "None"; diff --git a/packages/llamaindex/tests/VectorStore.test.ts b/packages/llamaindex/tests/VectorStore.test.ts index f44444d773..5efc070b9d 100644 --- a/packages/llamaindex/tests/VectorStore.test.ts +++ b/packages/llamaindex/tests/VectorStore.test.ts @@ -1,4 +1,10 @@ -import { Document, MetadataMode } from "@llamaindex/core/schema"; +import { + Document, + ImageNode, + IndexNode, + MetadataMode, + TextNode, +} from "@llamaindex/core/schema"; import { metadataDictToNode, nodeToMetadata, @@ -46,3 +52,36 @@ describe("Testing VectorStore utils", () => { }).toThrow(); }); }); + +describe("vector store utilities", () => { + test("nodeToMetadata", () => { + { + const node = new Document({ + text: "text", + }); + const metadata = nodeToMetadata(node); + expect(metadata["_node_type"]).toBe("Document"); + } + { + const node = new TextNode({ + text: "text", + }); + const metadata = nodeToMetadata(node); + expect(metadata["_node_type"]).toBe("TextNode"); + } + { + const node = new IndexNode({ + indexId: "indexId", + }); + const metadata = nodeToMetadata(node); + expect(metadata["_node_type"]).toBe("IndexNode"); + } + { + const node = new ImageNode({ + image: "image", + }); + const metadata = nodeToMetadata(node); + expect(metadata["_node_type"]).toBe("ImageNode"); + } + }); +}); From 0450c8c6307a9a919354462eb66d05972eab6d54 Mon Sep 17 00:00:00 2001 From: Alex Yang Date: Tue, 3 Sep 2024 09:40:59 -0700 Subject: [PATCH 04/13] fix: hash logic --- packages/core/src/schema/node.ts | 7 ++- packages/core/src/schema/python.ts | 2 +- packages/core/tests/schema-node.test.ts | 57 +++++++++++++------------ 3 files changed, 36 insertions(+), 30 deletions(-) diff --git a/packages/core/src/schema/node.ts b/packages/core/src/schema/node.ts index f000da3731..bfb038585b 100644 --- a/packages/core/src/schema/node.ts +++ b/packages/core/src/schema/node.ts @@ -85,6 +85,9 @@ export abstract class BaseNode { this.excludedLlmMetadataKeys = excludedLlmMetadataKeys ?? []; this.relationships = relationships ?? {}; this.embedding = embedding; + if (hash !== undefined) { + this.hash = hash; + } } abstract get type(): ObjectType; @@ -224,10 +227,10 @@ export class TextNode extends BaseNode { init; this.text = text ?? ""; this.textTemplate = textTemplate ?? ""; - if (startCharIdx) { + if (startCharIdx !== undefined) { this.startCharIdx = startCharIdx; } - if (endCharIdx) { + if (endCharIdx !== undefined) { this.endCharIdx = endCharIdx; } this.metadataSeparator = metadataSeparator ?? "\n"; diff --git a/packages/core/src/schema/python.ts b/packages/core/src/schema/python.ts index b0c7cf9d4d..8808183bea 100644 --- a/packages/core/src/schema/python.ts +++ b/packages/core/src/schema/python.ts @@ -58,7 +58,7 @@ export async function fromPythonDocStore({ } const objectType = PYTHON_TO_JS_TYPE_MAP[type as keyof typeof PYTHON_TO_JS_TYPE_MAP]; - const convertedJson = fromPythonImpl(JSON.parse(data)); + const convertedJson = await fromPythonImpl(JSON.parse(data)); return jsonToNode(convertedJson, objectType); } diff --git a/packages/core/tests/schema-node.test.ts b/packages/core/tests/schema-node.test.ts index 472ced3e40..57c31461a9 100644 --- a/packages/core/tests/schema-node.test.ts +++ b/packages/core/tests/schema-node.test.ts @@ -9,7 +9,7 @@ import { beforeEach, describe, expect, test } from "vitest"; describe("Python", () => { test("from python doc store", async () => { const node = await fromPythonDocStore({ - __data__: { + __data__: JSON.stringify({ id_: "e86be4a7-2ad0-4c3c-937b-3140f562e7a7", embedding: null, metadata: {}, @@ -32,36 +32,39 @@ describe("Python", () => { metadata_template: "{key}: {value}", metadata_seperator: "\n", class_name: "TextNode", - }, + }), __type__: "1", }); + expect(node.startCharIdx).toBe(0); + expect(node.endCharIdx).toBe(1599); expect(node).toMatchInlineSnapshot(` - { - "embedding": null, - "endCharIdx": 1599, - "excludedEmbedMetadataKeys": [], - "excludedLlmMetadataKeys": [], - "hash": "e+X0sEK9rsmaY1GwfOQnloy69/UrsmG7mhihpwPPfS4=", - "id_": "e86be4a7-2ad0-4c3c-937b-3140f562e7a7", - "metadata": {}, - "metadataSeparator": " - ", - "relationships": { - "1": { - "className": "RelatedNodeInfo", - "hash": "191a8fdcf068d3ac831da23cde07a92efe1432243c7f628d1009aa2ecdf6cb03", - "metadata": {}, - "nodeId": "e1fe8fd0-f470-40cd-bc2e-be3a220cef94", - "nodeType": "4", - }, - }, - "text": "This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test.", - "textTemplate": "{metadata_str} + { + "embedding": null, + "endCharIdx": 1599, + "excludedEmbedMetadataKeys": [], + "excludedLlmMetadataKeys": [], + "hash": "RCOOuYzMV1p6fQdVbg1750LB9BbqECFI0IkEriAyZYc=", + "id_": "e86be4a7-2ad0-4c3c-937b-3140f562e7a7", + "metadata": {}, + "metadataSeparator": " + ", + "relationships": { + "1": { + "className": "RelatedNodeInfo", + "hash": "191a8fdcf068d3ac831da23cde07a92efe1432243c7f628d1009aa2ecdf6cb03", + "metadata": {}, + "nodeId": "e1fe8fd0-f470-40cd-bc2e-be3a220cef94", + "nodeType": "4", + }, + }, + "startCharIdx": 0, + "text": "This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test. This is a test.", + "textTemplate": "{metadata_str} - {content}", - "type": "TEXT", - } - `); + {content}", + "type": "1", + } + `); expect(node.id_).toBe("e86be4a7-2ad0-4c3c-937b-3140f562e7a7"); expect(node.type).toBe(ObjectType.TEXT); }); From d7bda0c31b64e8923eb44692c79756962fcbac0d Mon Sep 17 00:00:00 2001 From: Alex Yang Date: Tue, 3 Sep 2024 09:46:16 -0700 Subject: [PATCH 05/13] fix: hash logic --- packages/core/src/schema/node.ts | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/packages/core/src/schema/node.ts b/packages/core/src/schema/node.ts index bfb038585b..10ced53b01 100644 --- a/packages/core/src/schema/node.ts +++ b/packages/core/src/schema/node.ts @@ -236,18 +236,10 @@ export class TextNode extends BaseNode { this.metadataSeparator = metadataSeparator ?? "\n"; } - /** - * Generate a hash of the text node. - * The ID is not part of the hash as it can change independent of content. - * @returns - */ generateHash() { const hashFunction = createSHA256(); - hashFunction.update(`type=${this.type}`); - hashFunction.update( - `startCharIdx=${this.startCharIdx} endCharIdx=${this.endCharIdx}`, - ); - hashFunction.update(this.getContent(MetadataMode.ALL)); + const docIdentity = this.text + JSON.stringify(this.metadata); + hashFunction.update(docIdentity); return hashFunction.digest(); } From 5e1b9b280922ee70c3c823fe11fc490867b5a232 Mon Sep 17 00:00:00 2001 From: Alex Yang Date: Tue, 3 Sep 2024 09:50:51 -0700 Subject: [PATCH 06/13] fix: hash logic --- packages/core/tests/schema-node.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/tests/schema-node.test.ts b/packages/core/tests/schema-node.test.ts index 57c31461a9..eaaef13d5f 100644 --- a/packages/core/tests/schema-node.test.ts +++ b/packages/core/tests/schema-node.test.ts @@ -99,7 +99,7 @@ describe("TextNode", () => { test("should generate a hash", () => { expect(node.hash).toMatchInlineSnapshot( - `"nTSKdUTYqR52MPv/brvb4RTGeqedTEqG9QN8KSAj2Do="`, + `"oznYDHYUGHArYnhRy9lj63IvEt/rNg1EH5EjwtPU/Pc="`, ); }); From b39fc052a65ae0d64c70ff76281270736942ddeb Mon Sep 17 00:00:00 2001 From: Alex Yang Date: Tue, 3 Sep 2024 09:56:19 -0700 Subject: [PATCH 07/13] fix: toJSON --- packages/core/src/schema/node.ts | 6 ++---- packages/core/tests/schema-node.test.ts | 6 +----- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/packages/core/src/schema/node.ts b/packages/core/src/schema/node.ts index 10ced53b01..2e45877964 100644 --- a/packages/core/src/schema/node.ts +++ b/packages/core/src/schema/node.ts @@ -179,14 +179,12 @@ export abstract class BaseNode { toJSON(): Record { return { ...this, - type: this.type, - // hash is an accessor property, so it's not included in the rest operator - hash: this.hash, + // no `type` and `hash` here to align with Python side }; } clone(): BaseNode { - return jsonToNode(this.toMutableJSON()) as BaseNode; + return jsonToNode(this.toMutableJSON(), this.type); } /** diff --git a/packages/core/tests/schema-node.test.ts b/packages/core/tests/schema-node.test.ts index eaaef13d5f..c5ff4d5e4e 100644 --- a/packages/core/tests/schema-node.test.ts +++ b/packages/core/tests/schema-node.test.ts @@ -43,7 +43,6 @@ describe("Python", () => { "endCharIdx": 1599, "excludedEmbedMetadataKeys": [], "excludedLlmMetadataKeys": [], - "hash": "RCOOuYzMV1p6fQdVbg1750LB9BbqECFI0IkEriAyZYc=", "id_": "e86be4a7-2ad0-4c3c-937b-3140f562e7a7", "metadata": {}, "metadataSeparator": " @@ -62,7 +61,6 @@ describe("Python", () => { "textTemplate": "{metadata_str} {content}", - "type": "1", } `); expect(node.id_).toBe("e86be4a7-2ad0-4c3c-937b-3140f562e7a7"); @@ -79,7 +77,7 @@ describe("Document", () => { test("should generate a hash", () => { expect(document.hash).toMatchInlineSnapshot( - `"1mkNkQC30mZlBBG48DNuG2WSKcTQ32DImC+4JUoVijg="`, + `"oznYDHYUGHArYnhRy9lj63IvEt/rNg1EH5EjwtPU/Pc="`, ); }); @@ -121,7 +119,6 @@ describe("TextNode", () => { "embedding": undefined, "excludedEmbedMetadataKeys": [], "excludedLlmMetadataKeys": [], - "hash": "Z6SWgFPlalaeblMGQGw0KS3qKgmZdEWXKfzEp/K+QN0=", "id_": Any, "metadata": { "something": 1, @@ -132,7 +129,6 @@ describe("TextNode", () => { "relationships": {}, "text": "Hello World", "textTemplate": "", - "type": "TEXT", } `, ); From 6f28735045c0da703833103a879800a0081da934 Mon Sep 17 00:00:00 2001 From: Alex Yang Date: Tue, 3 Sep 2024 11:00:09 -0700 Subject: [PATCH 08/13] fix: test --- packages/core/tests/decorator.test.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/core/tests/decorator.test.ts b/packages/core/tests/decorator.test.ts index 326810104d..a6b3086c19 100644 --- a/packages/core/tests/decorator.test.ts +++ b/packages/core/tests/decorator.test.ts @@ -12,14 +12,17 @@ describe("chunkSizeCheck", () => { env.ENABLE_CHUNK_SIZE_CHECK = "true"; let message = ""; - const consoleMock = vi - .spyOn(console, "warn") - .mockImplementation((msg) => (message += msg + "\n")); + vi.spyOn(console, "warn").mockImplementation( + (msg) => (message += msg + "\n"), + ); Settings.chunkSize = 0; + const node = new TextNode(); expect(message).toEqual(""); node.setContent("a".repeat(1024)); + expect(message).toBe(""); + node.getContent(); expect(message).toContain("is larger than chunk size"); }); }); From eb2dc73ef3e0c3ad07d33bfafa6ce1f5015b2236 Mon Sep 17 00:00:00 2001 From: Alex Yang Date: Tue, 3 Sep 2024 11:53:47 -0700 Subject: [PATCH 09/13] fix: add type back --- packages/core/src/schema/node.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/core/src/schema/node.ts b/packages/core/src/schema/node.ts index 2e45877964..245e1162a5 100644 --- a/packages/core/src/schema/node.ts +++ b/packages/core/src/schema/node.ts @@ -179,7 +179,8 @@ export abstract class BaseNode { toJSON(): Record { return { ...this, - // no `type` and `hash` here to align with Python side + type: this.type, + // no `hash` here to align with Python side }; } From 51e0435dfe8c08e16310f159a1c175e501aee4f2 Mon Sep 17 00:00:00 2001 From: Alex Yang Date: Tue, 3 Sep 2024 12:08:29 -0700 Subject: [PATCH 10/13] test: update --- packages/core/tests/schema-node.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/core/tests/schema-node.test.ts b/packages/core/tests/schema-node.test.ts index c5ff4d5e4e..628a1f42d7 100644 --- a/packages/core/tests/schema-node.test.ts +++ b/packages/core/tests/schema-node.test.ts @@ -61,6 +61,7 @@ describe("Python", () => { "textTemplate": "{metadata_str} {content}", + "type": "1", } `); expect(node.id_).toBe("e86be4a7-2ad0-4c3c-937b-3140f562e7a7"); From 732be4f2fea7360d5a870de400200c2ecd46a2d7 Mon Sep 17 00:00:00 2001 From: Alex Yang Date: Tue, 3 Sep 2024 13:49:08 -0700 Subject: [PATCH 11/13] test: update --- packages/core/tests/schema-node.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/core/tests/schema-node.test.ts b/packages/core/tests/schema-node.test.ts index 628a1f42d7..074b80ce75 100644 --- a/packages/core/tests/schema-node.test.ts +++ b/packages/core/tests/schema-node.test.ts @@ -130,6 +130,7 @@ describe("TextNode", () => { "relationships": {}, "text": "Hello World", "textTemplate": "", + "type": "1", } `, ); From 68d0f71269e37beda9290939d801bd547b5cecb8 Mon Sep 17 00:00:00 2001 From: Alex Yang Date: Wed, 4 Sep 2024 14:33:00 -0700 Subject: [PATCH 12/13] fix: hash check --- packages/core/src/schema/node.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/core/src/schema/node.ts b/packages/core/src/schema/node.ts index 245e1162a5..3d6725f8a4 100644 --- a/packages/core/src/schema/node.ts +++ b/packages/core/src/schema/node.ts @@ -76,7 +76,6 @@ export abstract class BaseNode { excludedEmbedMetadataKeys, excludedLlmMetadataKeys, relationships, - hash, embedding, } = init || {}; this.id_ = id_ ?? randomUUID(); @@ -85,9 +84,6 @@ export abstract class BaseNode { this.excludedLlmMetadataKeys = excludedLlmMetadataKeys ?? []; this.relationships = relationships ?? {}; this.embedding = embedding; - if (hash !== undefined) { - this.hash = hash; - } } abstract get type(): ObjectType; From dc77d9747f4fd1ccee569ef245408f045e91df15 Mon Sep 17 00:00:00 2001 From: Alex Yang Date: Wed, 4 Sep 2024 14:40:44 -0700 Subject: [PATCH 13/13] fix: legacy js object map --- packages/core/src/schema/index.ts | 2 +- packages/core/src/schema/python.ts | 32 +++++++++---------- .../src/ingestion/IngestionCache.ts | 14 +++++--- .../src/storage/docStore/KVDocumentStore.ts | 13 +++++--- .../llamaindex/src/storage/docStore/utils.ts | 4 +-- 5 files changed, 35 insertions(+), 30 deletions(-) diff --git a/packages/core/src/schema/index.ts b/packages/core/src/schema/index.ts index e99243f7c7..6abcaac5fd 100644 --- a/packages/core/src/schema/index.ts +++ b/packages/core/src/schema/index.ts @@ -1,5 +1,5 @@ export * from "./node"; -export { DATA_KEY, TYPE_KEY, fromPythonDocStore, type DocJson } from "./python"; +export { DATA_KEY, TYPE_KEY, fromDocStore, type DocJson } from "./python"; export { FileReader, TransformComponent, type BaseReader } from "./type"; export { EngineResponse } from "./type/engine–response"; export * from "./zod"; diff --git a/packages/core/src/schema/python.ts b/packages/core/src/schema/python.ts index 8808183bea..d789cc2e75 100644 --- a/packages/core/src/schema/python.ts +++ b/packages/core/src/schema/python.ts @@ -28,12 +28,20 @@ const PYTHON_TO_JS_TYPE_MAP = { "4": ObjectType.DOCUMENT, }; +const LEGACY_JS_MAP = { + TEXT: ObjectType.TEXT, + IMAGE: ObjectType.IMAGE, + INDEX: ObjectType.INDEX, + DOCUMENT: ObjectType.DOCUMENT, + IMAGE_DOCUMENT: ObjectType.DOCUMENT, +}; + export type DocJson = { [TYPE_KEY]: string; [DATA_KEY]: string; }; -async function fromPythonImpl(data: Record) { +async function fromImpl(data: Record) { const convertedJson = await camelCaseJson(data); if (convertedJson.relationships) { for (const [key, value] of Object.entries(convertedJson.relationships)) { @@ -49,26 +57,16 @@ async function fromPythonImpl(data: Record) { return convertedJson; } -export async function fromPythonDocStore({ +export async function fromDocStore({ [TYPE_KEY]: type, [DATA_KEY]: data, }: DocJson) { - if (!(type in PYTHON_TO_JS_TYPE_MAP)) { - throw new Error(""); - } - const objectType = - PYTHON_TO_JS_TYPE_MAP[type as keyof typeof PYTHON_TO_JS_TYPE_MAP]; - const convertedJson = await fromPythonImpl(JSON.parse(data)); - return jsonToNode(convertedJson, objectType); -} - -export async function fromPythonNode(json: Record) { - const convertedJson = await fromPythonImpl(json); - const type = convertedJson["metadata"]?.["_node_type"]; - if (!(type in PYTHON_TO_JS_TYPE_MAP)) { - throw new Error(""); + if (!(type in PYTHON_TO_JS_TYPE_MAP) && !(type in LEGACY_JS_MAP)) { + throw new Error("Invalid type"); } const objectType = - PYTHON_TO_JS_TYPE_MAP[type as keyof typeof PYTHON_TO_JS_TYPE_MAP]; + PYTHON_TO_JS_TYPE_MAP[type as keyof typeof PYTHON_TO_JS_TYPE_MAP] || + LEGACY_JS_MAP[type as keyof typeof LEGACY_JS_MAP]; + const convertedJson = await fromImpl(JSON.parse(data)); return jsonToNode(convertedJson, objectType); } diff --git a/packages/llamaindex/src/ingestion/IngestionCache.ts b/packages/llamaindex/src/ingestion/IngestionCache.ts index 44bfd69a25..796711801e 100644 --- a/packages/llamaindex/src/ingestion/IngestionCache.ts +++ b/packages/llamaindex/src/ingestion/IngestionCache.ts @@ -1,7 +1,11 @@ -import type { BaseNode, TransformComponent } from "@llamaindex/core/schema"; -import { MetadataMode } from "@llamaindex/core/schema"; +import { + type BaseNode, + fromDocStore, + MetadataMode, + type TransformComponent, +} from "@llamaindex/core/schema"; import { createSHA256 } from "@llamaindex/env"; -import { docToJson, jsonToDoc } from "../storage/docStore/utils.js"; +import { docToJson } from "../storage/docStore/utils.js"; import { SimpleKVStore } from "../storage/kvStore/SimpleKVStore.js"; import type { BaseKVStore } from "../storage/kvStore/types.js"; @@ -63,6 +67,8 @@ export class IngestionCache { if (!json || !json[this.nodesKey] || !Array.isArray(json[this.nodesKey])) { return undefined; } - return Promise.all(json[this.nodesKey].map((doc: any) => jsonToDoc(doc))); + return Promise.all( + json[this.nodesKey].map((doc: any) => fromDocStore(doc)), + ); } } diff --git a/packages/llamaindex/src/storage/docStore/KVDocumentStore.ts b/packages/llamaindex/src/storage/docStore/KVDocumentStore.ts index b563197d76..94a4bfe355 100644 --- a/packages/llamaindex/src/storage/docStore/KVDocumentStore.ts +++ b/packages/llamaindex/src/storage/docStore/KVDocumentStore.ts @@ -1,11 +1,14 @@ -import type { BaseNode } from "@llamaindex/core/schema"; -import { ObjectType } from "@llamaindex/core/schema"; +import { + type BaseNode, + fromDocStore, + ObjectType, +} from "@llamaindex/core/schema"; import _ from "lodash"; import { DEFAULT_NAMESPACE } from "../constants.js"; import type { BaseKVStore } from "../kvStore/types.js"; import type { RefDocInfo } from "./types.js"; import { BaseDocumentStore } from "./types.js"; -import { docToJson, isValidDocJson, jsonToDoc } from "./utils.js"; +import { docToJson, isValidDocJson } from "./utils.js"; type DocMetaData = { docHash: string; refDocId?: string }; @@ -29,7 +32,7 @@ export class KVDocumentStore extends BaseDocumentStore { for (const key in jsonDict) { const value = jsonDict[key]; if (isValidDocJson(value)) { - docs[key] = await jsonToDoc(value); + docs[key] = await fromDocStore(value); } else { console.warn(`Invalid JSON for docId ${key}`); } @@ -94,7 +97,7 @@ export class KVDocumentStore extends BaseDocumentStore { if (!isValidDocJson(json)) { throw new Error(`Invalid JSON for docId ${docId}`); } - return jsonToDoc(json); + return fromDocStore(json); } async getRefDocInfo(refDocId: string): Promise { diff --git a/packages/llamaindex/src/storage/docStore/utils.ts b/packages/llamaindex/src/storage/docStore/utils.ts index d79426c48b..b2c68a5516 100644 --- a/packages/llamaindex/src/storage/docStore/utils.ts +++ b/packages/llamaindex/src/storage/docStore/utils.ts @@ -1,5 +1,5 @@ import type { BaseNode } from "@llamaindex/core/schema"; -import { ObjectType, fromPythonDocStore } from "@llamaindex/core/schema"; +import { ObjectType } from "@llamaindex/core/schema"; const TYPE_KEY = "__type__"; const DATA_KEY = "__data__"; @@ -24,5 +24,3 @@ export function docToJson(doc: BaseNode): DocJson { [TYPE_KEY]: doc.type, }; } - -export const jsonToDoc = fromPythonDocStore;