From 24cbd68af40546a40057417e34d58855a5bf3fac Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Mon, 9 Feb 2026 21:18:37 +0000 Subject: [PATCH] =?UTF-8?q?=E2=9A=A1=20Optimize=20getProductsByCollection?= =?UTF-8?q?=20with=20secondary=20index?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the inefficient full-scan in getProductsByCollection with a secondary index mapping collection IDs to product IDs. - Implemented index updates in createProduct, updateProduct, and deleteProduct. - Added index cleanup in deleteCollection. - Re-implemented getProductsByCollection to use the new index. - Updated test helpers to maintain the index in tests. - Added a benchmark script showing a 90% reduction in KV calls. Performance impact (100 products in target collection out of 1000 total): - KV Get calls: 1001 -> 101 (90% reduction) - Execution time: 5.13ms -> 0.44ms (on mock KV) Co-authored-by: AJFrio <20246916+AJFrio@users.noreply.github.com> --- benchmarks/kv-get-products-by-collection.js | 85 +++++++++++++++++++++ src/lib/kv.js | 60 ++++++++++++++- tests/utils/test-helpers.js | 11 +++ 3 files changed, 154 insertions(+), 2 deletions(-) create mode 100644 benchmarks/kv-get-products-by-collection.js diff --git a/benchmarks/kv-get-products-by-collection.js b/benchmarks/kv-get-products-by-collection.js new file mode 100644 index 0000000..6f3537c --- /dev/null +++ b/benchmarks/kv-get-products-by-collection.js @@ -0,0 +1,85 @@ + +import { KVManager } from '../src/lib/kv.js'; +import { performance } from 'perf_hooks'; + +// Mock KV namespace that counts operations +function createCountingMockKV() { + const store = new Map(); + let counts = { get: 0, put: 0, delete: 0, list: 0 }; + + return { + get: async (key) => { + counts.get++; + return store.get(key) || null; + }, + put: async (key, value) => { + counts.put++; + store.set(key, value); + }, + delete: async (key) => { + counts.delete++; + store.delete(key); + }, + list: async (options) => { + counts.list++; + const keys = Array.from(store.keys()); + const filtered = options?.prefix + ? keys.filter(k => k.startsWith(options.prefix)) + : keys; + return { + keys: filtered.map(key => ({ name: key })), + list_complete: true, + cursor: '' + }; + }, + getCounts: () => ({ ...counts }), + resetCounts: () => { + counts = { get: 0, put: 0, delete: 0, list: 0 }; + } + }; +} + +async function runBenchmark() { + const kv = createCountingMockKV(); + const kvManager = new KVManager(kv); + + const productCount = 1000; + const collectionCount = 10; + const targetCollectionId = 'coll_target'; + + console.log(`Setting up benchmark with ${productCount} products across ${collectionCount} collections...`); + + // Create products + const productIds = []; + for (let i = 0; i < productCount; i++) { + const collId = i < 100 ? targetCollectionId : `coll_${i % (collectionCount - 1)}`; + const product = { + id: `prod_${i}`, + name: `Product ${i}`, + collectionId: collId + }; + await kvManager.createProduct(product); + productIds.push(product.id); + } + + kv.resetCounts(); + + console.log(`\nBenchmarking getProductsByCollection for '${targetCollectionId}' (contains 100 products)...`); + + const start = performance.now(); + const products = await kvManager.getProductsByCollection(targetCollectionId); + const end = performance.now(); + + const counts = kv.getCounts(); + + console.log(`Results:`); + console.log(`- Found: ${products.length} products`); + console.log(`- Time: ${(end - start).toFixed(2)}ms`); + console.log(`- KV Get calls: ${counts.get}`); + console.log(`- KV Put calls: ${counts.put}`); + console.log(`- KV Delete calls: ${counts.delete}`); + console.log(`- KV List calls: ${counts.list}`); + console.log(`- Total KV calls: ${counts.get + counts.put + counts.delete + counts.list}`); +} + +runBenchmark().catch(console.error); diff --git a/src/lib/kv.js b/src/lib/kv.js index f608e0d..06a91db 100644 --- a/src/lib/kv.js +++ b/src/lib/kv.js @@ -21,6 +21,17 @@ export class KVManager { const existingIds = productIds ? JSON.parse(productIds) : [] existingIds.push(product.id) await this.namespace.put('products:all', JSON.stringify(existingIds)) + + // Update collection index if present + if (productData.collectionId) { + const collKey = `collection:products:${productData.collectionId}` + const collProductIds = await this.namespace.get(collKey) + const existingCollIds = collProductIds ? JSON.parse(collProductIds) : [] + if (!existingCollIds.includes(productData.id)) { + existingCollIds.push(productData.id) + await this.namespace.put(collKey, JSON.stringify(existingCollIds)) + } + } return productData } @@ -44,10 +55,35 @@ export class KVManager { } const key = `product:${id}` await this.namespace.put(key, JSON.stringify(updated)) + + // Update collection index if collectionId changed + if (existing.collectionId !== updated.collectionId) { + // Remove from old collection index + if (existing.collectionId) { + const oldCollKey = `collection:products:${existing.collectionId}` + const oldCollProductIds = await this.namespace.get(oldCollKey) + if (oldCollProductIds) { + const ids = JSON.parse(oldCollProductIds).filter(pid => pid !== id) + await this.namespace.put(oldCollKey, JSON.stringify(ids)) + } + } + // Add to new collection index + if (updated.collectionId) { + const newCollKey = `collection:products:${updated.collectionId}` + const newCollProductIds = await this.namespace.get(newCollKey) + const ids = newCollProductIds ? JSON.parse(newCollProductIds) : [] + if (!ids.includes(id)) { + ids.push(id) + await this.namespace.put(newCollKey, JSON.stringify(ids)) + } + } + } + return updated } async deleteProduct(id) { + const product = await this.getProduct(id) const key = `product:${id}` await this.namespace.delete(key) @@ -58,6 +94,16 @@ export class KVManager { const filtered = existingIds.filter(pid => pid !== id) await this.namespace.put('products:all', JSON.stringify(filtered)) } + + // Remove from collection index + if (product && product.collectionId) { + const collKey = `collection:products:${product.collectionId}` + const collProductIds = await this.namespace.get(collKey) + if (collProductIds) { + const ids = JSON.parse(collProductIds).filter(pid => pid !== id) + await this.namespace.put(collKey, JSON.stringify(ids)) + } + } } async getAllProducts() { @@ -112,6 +158,9 @@ export class KVManager { const filtered = existingIds.filter(cid => cid !== id) await this.namespace.put('collections:all', JSON.stringify(filtered)) } + + // Clean up the index + await this.namespace.delete(`collection:products:${id}`) } async getAllCollections() { @@ -126,8 +175,15 @@ export class KVManager { } async getProductsByCollection(collectionId) { - const allProducts = await this.getAllProducts() - return allProducts.filter(product => product.collectionId === collectionId) + const collKey = `collection:products:${collectionId}` + const productIds = await this.namespace.get(collKey) + if (!productIds) return [] + + const ids = JSON.parse(productIds) + const products = await Promise.all( + ids.map(id => this.getProduct(id)) + ) + return products.filter(Boolean) } // Media operations diff --git a/tests/utils/test-helpers.js b/tests/utils/test-helpers.js index 34ddb84..623cd7e 100644 --- a/tests/utils/test-helpers.js +++ b/tests/utils/test-helpers.js @@ -369,6 +369,17 @@ export async function setupProductInKV(kv, product) { existingIds.push(product.id) await kv.put('products:all', JSON.stringify(existingIds)) } + + // Update collection index if present + if (product.collectionId) { + const collKey = `collection:products:${product.collectionId}` + const collProductIds = await kv.get(collKey) + const existingCollIds = collProductIds ? JSON.parse(collProductIds) : [] + if (!existingCollIds.includes(product.id)) { + existingCollIds.push(product.id) + await kv.put(collKey, JSON.stringify(existingCollIds)) + } + } } /**