⚡ Optimize getProductsByCollection with secondary index

benchmarks/kv-get-products-by-collection.js

@@ -0,0 +1,85 @@
+
+import { KVManager } from '../src/lib/kv.js';
+import { performance } from 'perf_hooks';
+
+// Mock KV namespace that counts operations
+function createCountingMockKV() {
+  const store = new Map();
+  let counts = { get: 0, put: 0, delete: 0, list: 0 };
+
+  return {
+    get: async (key) => {
+      counts.get++;
+      return store.get(key) || null;
+    },
+    put: async (key, value) => {
+      counts.put++;
+      store.set(key, value);
+    },
+    delete: async (key) => {
+      counts.delete++;
+      store.delete(key);
+    },
+    list: async (options) => {
+      counts.list++;
+      const keys = Array.from(store.keys());
+      const filtered = options?.prefix
+        ? keys.filter(k => k.startsWith(options.prefix))
+        : keys;
+      return {
+        keys: filtered.map(key => ({ name: key })),
+        list_complete: true,
+        cursor: ''
+      };
+    },
+    getCounts: () => ({ ...counts }),
+    resetCounts: () => {
+      counts = { get: 0, put: 0, delete: 0, list: 0 };
+    }
+  };
+}
+
+async function runBenchmark() {
+  const kv = createCountingMockKV();
+  const kvManager = new KVManager(kv);
+
+  const productCount = 1000;
+  const collectionCount = 10;
+  const targetCollectionId = 'coll_target';
+
+  console.log(`Setting up benchmark with ${productCount} products across ${collectionCount} collections...`);
+
+  // Create products
+  const productIds = [];
+  for (let i = 0; i < productCount; i++) {
+    const collId = i < 100 ? targetCollectionId : `coll_${i % (collectionCount - 1)}`;
+    const product = {
+      id: `prod_${i}`,
+      name: `Product ${i}`,
+      collectionId: collId
+    };
+    await kvManager.createProduct(product);
+    productIds.push(product.id);
+  }
+
+  kv.resetCounts();
+
+  console.log(`\nBenchmarking getProductsByCollection for '${targetCollectionId}' (contains 100 products)...`);
+
+  const start = performance.now();
+  const products = await kvManager.getProductsByCollection(targetCollectionId);
+  const end = performance.now();
+
+  const counts = kv.getCounts();
+
+  console.log(`Results:`);
+  console.log(`- Found: ${products.length} products`);
+  console.log(`- Time: ${(end - start).toFixed(2)}ms`);
+  console.log(`- KV Get calls: ${counts.get}`);
+  console.log(`- KV Put calls: ${counts.put}`);
+  console.log(`- KV Delete calls: ${counts.delete}`);
+  console.log(`- KV List calls: ${counts.list}`);
+  console.log(`- Total KV calls: ${counts.get + counts.put + counts.delete + counts.list}`);
+}
+
+runBenchmark().catch(console.error);

src/lib/kv.js

@@ -21,6 +21,17 @@ export class KVManager {
     const existingIds = productIds ? JSON.parse(productIds) : []
     existingIds.push(product.id)
     await this.namespace.put('products:all', JSON.stringify(existingIds))
+
+    // Update collection index if present
+    if (productData.collectionId) {
+      const collKey = `collection:products:${productData.collectionId}`
+      const collProductIds = await this.namespace.get(collKey)
+      const existingCollIds = collProductIds ? JSON.parse(collProductIds) : []
+      if (!existingCollIds.includes(productData.id)) {
+        existingCollIds.push(productData.id)
+        await this.namespace.put(collKey, JSON.stringify(existingCollIds))
+      }
+    }
 
     return productData
   }
@@ -44,10 +55,35 @@ export class KVManager {
     }
     const key = `product:${id}`
     await this.namespace.put(key, JSON.stringify(updated))
+
+    // Update collection index if collectionId changed
+    if (existing.collectionId !== updated.collectionId) {
+      // Remove from old collection index
+      if (existing.collectionId) {
+        const oldCollKey = `collection:products:${existing.collectionId}`
+        const oldCollProductIds = await this.namespace.get(oldCollKey)
+        if (oldCollProductIds) {
+          const ids = JSON.parse(oldCollProductIds).filter(pid => pid !== id)
+          await this.namespace.put(oldCollKey, JSON.stringify(ids))
+        }
+      }
+      // Add to new collection index
+      if (updated.collectionId) {
+        const newCollKey = `collection:products:${updated.collectionId}`
+        const newCollProductIds = await this.namespace.get(newCollKey)
+        const ids = newCollProductIds ? JSON.parse(newCollProductIds) : []
+        if (!ids.includes(id)) {
+          ids.push(id)
+          await this.namespace.put(newCollKey, JSON.stringify(ids))
+        }
+      }
+    }
+
     return updated
   }
 
   async deleteProduct(id) {
+    const product = await this.getProduct(id)
     const key = `product:${id}`
     await this.namespace.delete(key)
 
@@ -58,6 +94,16 @@ export class KVManager {
       const filtered = existingIds.filter(pid => pid !== id)
       await this.namespace.put('products:all', JSON.stringify(filtered))
     }
+
+    // Remove from collection index
+    if (product && product.collectionId) {
+      const collKey = `collection:products:${product.collectionId}`
+      const collProductIds = await this.namespace.get(collKey)
+      if (collProductIds) {
+        const ids = JSON.parse(collProductIds).filter(pid => pid !== id)
+        await this.namespace.put(collKey, JSON.stringify(ids))
+      }
+    }
   }
 
   async getAllProducts() {
@@ -112,6 +158,9 @@ export class KVManager {
       const filtered = existingIds.filter(cid => cid !== id)
       await this.namespace.put('collections:all', JSON.stringify(filtered))
     }
+
+    // Clean up the index
+    await this.namespace.delete(`collection:products:${id}`)
   }
 
   async getAllCollections() {
@@ -126,8 +175,15 @@ export class KVManager {
   }
 
   async getProductsByCollection(collectionId) {
-    const allProducts = await this.getAllProducts()
-    return allProducts.filter(product => product.collectionId === collectionId)
+    const collKey = `collection:products:${collectionId}`
+    const productIds = await this.namespace.get(collKey)
+    if (!productIds) return []
+
+    const ids = JSON.parse(productIds)
+    const products = await Promise.all(
+      ids.map(id => this.getProduct(id))
+    )
+    return products.filter(Boolean)
   }
 
   // Media operations

tests/utils/test-helpers.js

@@ -369,6 +369,17 @@ export async function setupProductInKV(kv, product) {
     existingIds.push(product.id)
     await kv.put('products:all', JSON.stringify(existingIds))
   }
+
+  // Update collection index if present
+  if (product.collectionId) {
+    const collKey = `collection:products:${product.collectionId}`
+    const collProductIds = await kv.get(collKey)
+    const existingCollIds = collProductIds ? JSON.parse(collProductIds) : []
+    if (!existingCollIds.includes(product.id)) {
+      existingCollIds.push(product.id)
+      await kv.put(collKey, JSON.stringify(existingCollIds))
+    }
+  }
 }
 
 /**