From 32bca7a729f578d4f3f241ed75241411cd6c2e69 Mon Sep 17 00:00:00 2001 From: MaghirSokker Date: Tue, 23 Jan 2024 19:46:12 +0100 Subject: [PATCH 01/11] Adding solana API support --- WalletLabels | 1 + lib/assembleTypese.ts | 2 +- pages/api/solana/label.ts | 111 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 113 insertions(+), 1 deletion(-) create mode 160000 WalletLabels create mode 100644 pages/api/solana/label.ts diff --git a/WalletLabels b/WalletLabels new file mode 160000 index 00000000..f0beb94a --- /dev/null +++ b/WalletLabels @@ -0,0 +1 @@ +Subproject commit f0beb94ac38acd9043871c5af798c40f3b4ca881 diff --git a/lib/assembleTypese.ts b/lib/assembleTypese.ts index 679cf564..ef20657c 100644 --- a/lib/assembleTypese.ts +++ b/lib/assembleTypese.ts @@ -1,5 +1,5 @@ // ignore ts error all file -// @ts-nocheck +//@ts-nocheck import TypesenseInstantSearchAdapter from "typesense-instantsearch-adapter"; diff --git a/pages/api/solana/label.ts b/pages/api/solana/label.ts new file mode 100644 index 00000000..e27c90f6 --- /dev/null +++ b/pages/api/solana/label.ts @@ -0,0 +1,111 @@ +import middlewares, { middlewares_special } from "@/lib/rateLimits" +import { connectToDatabase } from "../../../lib/mongodb" + +export default async function handler(req, res) { + // Database connection setup + let db; + let address, limit, label; + try { + const database = await connectToDatabase(); + db = database.db; + } catch (error) { + console.log(error); + throw new Error("Unable to connect to database"); + } + + // Validate 'address' query parameter + if (req.query.address === undefined && req.query.label === undefined) { + return res.status(400).json({ message: "Bad request: 'address'/'label' parameter missing" }); + } + + if (req.query.address === "" || req.query.address === undefined) { + address = ""; + } else { + address = req.query.address; + } + if (req.query.label === "" || req.query.label === undefined) { + label = ""; + } else { + label = req.query.label; + } + + if (req.query.limit === "" || req.query.limit === undefined) { + limit = 20; + } else { + limit = Number(req.query.limit); + } + + if (limit > 100) { + limit = 100; + } + + if (req.method !== "GET") { + return res.status(405).json({ message: "Method not allowed" }); + } + + const clc_name = process.env.CLC_NAME_WLBLS_SOLANA; + let labels = null; + + try { + if (address === "" && label === "") { + labels = await db.collection(clc_name).find().limit(limit).toArray(); + } else if (!!address) { + // Adjust the MongoDB query to search by 'address' + const queryAtlas = { + address: address, + }; + + const projection = { + address_name: 1, + label_type: 1, + label_subtype: 1, + address: 1, + label: 1, + }; + + const cursor = await db.collection(clc_name).find(queryAtlas, { projection }).collation( + { locale: 'en', strength: 1 } + ) + .limit(limit); + labels = await cursor.toArray(); + } else if (!!label) { + // Adjust the MongoDB query to search by 'label' + const queryAtlas = { + label: label, + }; + + const projection = { + address_name: 1, + label_type: 1, + label_subtype: 1, + address: 1, + label: 1, + }; + + const cursor = await db.collection(clc_name).find(queryAtlas, { projection }).collation( + { locale: 'en', strength: 1 } + ) + .limit(limit); + labels = await cursor.toArray(); + } + + labels = labels.map((lbl) => ({ + address: lbl.address, + address_name: lbl.address_name, + label_type: lbl.label_type, + label_subtype: lbl.label_subtype, + label: lbl.label, + })); + + } catch (error) { + console.log(error); + res.status(500).json({ message: "Internal server error" }); + return; + } + + const response = { + data: labels, + }; + + res.status(200).json(response); +} From 57828109b1172831b8216c015017d9018a26308e Mon Sep 17 00:00:00 2001 From: MaghirSokker Date: Tue, 23 Jan 2024 19:59:05 +0100 Subject: [PATCH 02/11] Adding solana API --- WalletLabels | 1 - 1 file changed, 1 deletion(-) delete mode 160000 WalletLabels diff --git a/WalletLabels b/WalletLabels deleted file mode 160000 index f0beb94a..00000000 --- a/WalletLabels +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f0beb94ac38acd9043871c5af798c40f3b4ca881 From 385611ffba82596b85d679a39f7cd1934aa4e848 Mon Sep 17 00:00:00 2001 From: MaghirSokker Date: Thu, 25 Jan 2024 06:10:30 +0100 Subject: [PATCH 03/11] latenight incomplete search --- pages/api/solana/search.ts | 128 +++++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 pages/api/solana/search.ts diff --git a/pages/api/solana/search.ts b/pages/api/solana/search.ts new file mode 100644 index 00000000..4db8d6eb --- /dev/null +++ b/pages/api/solana/search.ts @@ -0,0 +1,128 @@ +import { Collection, Db, Document } from 'mongodb' +import { connectToDatabase } from "../../../lib/mongodb" +import type { NextApiRequest, NextApiResponse } from 'next' + + +let db: Db; + +let solana_wallets: Collection + + +export default async function handler( + req: NextApiRequest, + res: NextApiResponse +) { + //only allows GET requests to API + if (req.method !== "GET") { + return res.status(405).json({ message: "Method not allowed" }); + } + let labels: Label[] | null = null; + const skip = + typeof req.query.skip === 'string' ? Number(req.query.skip) : 0; + const limit = + typeof req.query.limit === 'string' ? Math.min(Number(req.query.limit), 100) : 20; + + const search = + typeof req.query.search === 'string' ? req.query.search : undefined; + + const { solana_wallets } = await getSolana({ skip, limit, query: search }) + labels = solana_wallets.map((lbl) => ({ + address: lbl.address, + address_name: lbl.address_name, + label_type: lbl.label_type, + label_subtype: lbl.label_subtype, + label: lbl.label, + })); + + const response = { + data: labels, + }; + + res.status(200).json(response); +} +//decoupled async search query +const getSolana = async ({ + query, + skip, + limit +}: { + query?: string + skip: number + limit: number +}) => { + try { + if (!solana_wallets) await init() + + const pipeline: PipelineStage[] = [{ $skip: skip }, { $limit: limit }] + + if (query) { + pipeline.unshift({ + $search: { + index: 'search', + text: { + query, + fuzzy: { + maxEdits: 1, + prefixLength: 3, + maxExpansions: 50 + }, + path: { + wildcard: '*' + } + } + } + }) + } + + const result = await solana_wallets.aggregate(pipeline).toArray() + + return { solana_wallets: result } + } catch (error) { + return { error } + } +} +//connect to db +async function init() { + try { + const database = await connectToDatabase(); + db = database.db; + + } catch (error) { + console.log(error); + throw new Error("Unable to connect to database"); + } +} + + +//mongosh query interface +type PipelineStage = + | { + $search: { + index: string + text: { + query: string + fuzzy: {} + path: { + wildcard: string + } + } + } + } + | { + $skip: number + } + | { + $limit: number + } +//response interface ? +type ResponseData = { + [key: string]: string | Label[]; + +} +type Label = { + address: string, + address_name: string, + label_type: string, + label_subtype: string, + label: string, +} \ No newline at end of file From 3e09f81d3bfe29ad644d23c933b2db69431acee7 Mon Sep 17 00:00:00 2001 From: MaghirSokker Date: Fri, 26 Jan 2024 18:27:50 +0100 Subject: [PATCH 04/11] search implemented --- pages/api/solana/search.ts | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/pages/api/solana/search.ts b/pages/api/solana/search.ts index 4db8d6eb..4b67bf0a 100644 --- a/pages/api/solana/search.ts +++ b/pages/api/solana/search.ts @@ -1,12 +1,9 @@ -import { Collection, Db, Document } from 'mongodb' +import { Db } from 'mongodb' import { connectToDatabase } from "../../../lib/mongodb" import type { NextApiRequest, NextApiResponse } from 'next' - let db: Db; - -let solana_wallets: Collection - +const clc_name = process.env.CLC_NAME_WLBLS_SOLANA; export default async function handler( req: NextApiRequest, @@ -20,12 +17,13 @@ export default async function handler( const skip = typeof req.query.skip === 'string' ? Number(req.query.skip) : 0; const limit = - typeof req.query.limit === 'string' ? Math.min(Number(req.query.limit), 100) : 20; + typeof req.query.limit === 'string' ? Math.max(Math.min(Number(req.query.limit), 100), 20) : 20; const search = typeof req.query.search === 'string' ? req.query.search : undefined; const { solana_wallets } = await getSolana({ skip, limit, query: search }) + console.log(solana_wallets); labels = solana_wallets.map((lbl) => ({ address: lbl.address, address_name: lbl.address_name, @@ -34,6 +32,7 @@ export default async function handler( label: lbl.label, })); + const response = { data: labels, }; @@ -51,14 +50,13 @@ const getSolana = async ({ limit: number }) => { try { - if (!solana_wallets) await init() + if (!db) await init() const pipeline: PipelineStage[] = [{ $skip: skip }, { $limit: limit }] - if (query) { pipeline.unshift({ $search: { - index: 'search', + index: 'dynamic', text: { query, fuzzy: { @@ -74,8 +72,8 @@ const getSolana = async ({ }) } - const result = await solana_wallets.aggregate(pipeline).toArray() - + const result = await db.collection(clc_name).aggregate(pipeline).toArray() + console.log(result) return { solana_wallets: result } } catch (error) { return { error } @@ -91,6 +89,7 @@ async function init() { console.log(error); throw new Error("Unable to connect to database"); } + return db; } From a036e6ab41fb82877aac20389e4091322deb4025 Mon Sep 17 00:00:00 2001 From: MaghirSokker Date: Fri, 26 Jan 2024 22:01:41 +0100 Subject: [PATCH 05/11] add error catch --- pages/api/solana/search.ts | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/pages/api/solana/search.ts b/pages/api/solana/search.ts index 4b67bf0a..23b9372c 100644 --- a/pages/api/solana/search.ts +++ b/pages/api/solana/search.ts @@ -23,14 +23,19 @@ export default async function handler( typeof req.query.search === 'string' ? req.query.search : undefined; const { solana_wallets } = await getSolana({ skip, limit, query: search }) - console.log(solana_wallets); - labels = solana_wallets.map((lbl) => ({ - address: lbl.address, - address_name: lbl.address_name, - label_type: lbl.label_type, - label_subtype: lbl.label_subtype, - label: lbl.label, - })); + try { + labels = solana_wallets.map((lbl) => ({ + address: lbl.address, + address_name: lbl.address_name, + label_type: lbl.label_type, + label_subtype: lbl.label_subtype, + label: lbl.label, + })); + } catch (error) { + console.log(error); + res.status(405).json({ message: "error in loading labels. Please try again later " }); + } + const response = { From 302cb87f39acab48c9cf5fbb2a4f19c377791532 Mon Sep 17 00:00:00 2001 From: MaghirSokker Date: Fri, 26 Jan 2024 22:48:43 +0100 Subject: [PATCH 06/11] Fixed a stealthy error with new db --- pages/api/solana/label.ts | 35 ++++++++++++++++++----------------- pages/api/solana/search.ts | 29 +++++++++++++++-------------- 2 files changed, 33 insertions(+), 31 deletions(-) diff --git a/pages/api/solana/label.ts b/pages/api/solana/label.ts index e27c90f6..ff14b4a0 100644 --- a/pages/api/solana/label.ts +++ b/pages/api/solana/label.ts @@ -52,15 +52,15 @@ export default async function handler(req, res) { } else if (!!address) { // Adjust the MongoDB query to search by 'address' const queryAtlas = { - address: address, + ADDRESS: address, }; const projection = { - address_name: 1, - label_type: 1, - label_subtype: 1, - address: 1, - label: 1, + ADDRESS_NAME: 1, + LABEL_TYPE: 1, + LABEL_SUBTYPE: 1, + ADDRESS: 1, + LABEL: 1, }; const cursor = await db.collection(clc_name).find(queryAtlas, { projection }).collation( @@ -71,15 +71,15 @@ export default async function handler(req, res) { } else if (!!label) { // Adjust the MongoDB query to search by 'label' const queryAtlas = { - label: label, + LABEL: label, }; const projection = { - address_name: 1, - label_type: 1, - label_subtype: 1, - address: 1, - label: 1, + ADDRESS_NAME: 1, + LABEL_TYPE: 1, + LABEL_SUBTYPE: 1, + ADDRESS: 1, + LABEL: 1, }; const cursor = await db.collection(clc_name).find(queryAtlas, { projection }).collation( @@ -87,14 +87,15 @@ export default async function handler(req, res) { ) .limit(limit); labels = await cursor.toArray(); + console.log(labels); } labels = labels.map((lbl) => ({ - address: lbl.address, - address_name: lbl.address_name, - label_type: lbl.label_type, - label_subtype: lbl.label_subtype, - label: lbl.label, + ADDRESS: lbl.ADDRESS, + ADDRESS_NAME: lbl.ADDRESS_NAME, + LABEL_TYPE: lbl.LABEL_TYPE, + LABEL_SUBTYPE: lbl.LABEL_SUBTYPE, + LABEL: lbl.LABEL, })); } catch (error) { diff --git a/pages/api/solana/search.ts b/pages/api/solana/search.ts index 23b9372c..9a1962fc 100644 --- a/pages/api/solana/search.ts +++ b/pages/api/solana/search.ts @@ -22,27 +22,28 @@ export default async function handler( const search = typeof req.query.search === 'string' ? req.query.search : undefined; - const { solana_wallets } = await getSolana({ skip, limit, query: search }) + const { solana_wallets } = await getSolana({ skip, limit, query: search }); try { labels = solana_wallets.map((lbl) => ({ - address: lbl.address, - address_name: lbl.address_name, - label_type: lbl.label_type, - label_subtype: lbl.label_subtype, - label: lbl.label, + address: lbl.ADDRESS, + address_name: lbl.ADDRESS_NAME, + label_type: lbl.LABEL_TYPE, + label_subtype: lbl.LABEL_SUBTYPE, + label: lbl.LABEL, })); - } catch (error) { - console.log(error); - res.status(405).json({ message: "error in loading labels. Please try again later " }); - } + const response = { + data: labels, + }; + + res.status(200).json(response); + } catch (error) { console.log(error); res.status(500); throw new Error("Unable to connect to database"); }; + + + - const response = { - data: labels, - }; - res.status(200).json(response); } //decoupled async search query const getSolana = async ({ From 1e2a6353f4ed393e0ced4d2ac3745d2ae0651a3f Mon Sep 17 00:00:00 2001 From: MaghirSokker Date: Fri, 26 Jan 2024 22:55:25 +0100 Subject: [PATCH 07/11] added correct logging --- pages/api/solana/label.ts | 5 ++++- pages/api/solana/search.ts | 3 +-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pages/api/solana/label.ts b/pages/api/solana/label.ts index ff14b4a0..67cda67b 100644 --- a/pages/api/solana/label.ts +++ b/pages/api/solana/label.ts @@ -68,6 +68,8 @@ export default async function handler(req, res) { ) .limit(limit); labels = await cursor.toArray(); + + console.log("solana label address :" + address.toString); } else if (!!label) { // Adjust the MongoDB query to search by 'label' const queryAtlas = { @@ -87,7 +89,8 @@ export default async function handler(req, res) { ) .limit(limit); labels = await cursor.toArray(); - console.log(labels); + + console.log("solana label query :" + label.toString); } labels = labels.map((lbl) => ({ diff --git a/pages/api/solana/search.ts b/pages/api/solana/search.ts index 9a1962fc..fb8d7927 100644 --- a/pages/api/solana/search.ts +++ b/pages/api/solana/search.ts @@ -77,9 +77,8 @@ const getSolana = async ({ } }) } - + console.log("solana search query :" + query.toString) const result = await db.collection(clc_name).aggregate(pipeline).toArray() - console.log(result) return { solana_wallets: result } } catch (error) { return { error } From e8cd0a150e1b9d1331a7f446202fe7357bdc1f2e Mon Sep 17 00:00:00 2001 From: MaghirSokker Date: Mon, 29 Jan 2024 22:59:02 +0100 Subject: [PATCH 08/11] Aiden optimization for find query --- pages/api/solana/label.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pages/api/solana/label.ts b/pages/api/solana/label.ts index 67cda67b..0145750f 100644 --- a/pages/api/solana/label.ts +++ b/pages/api/solana/label.ts @@ -63,9 +63,9 @@ export default async function handler(req, res) { LABEL: 1, }; - const cursor = await db.collection(clc_name).find(queryAtlas, { projection }).collation( - { locale: 'en', strength: 1 } - ) + const cursor = await db.collection(clc_name).find(queryAtlas, { projection },).collation( + { locale: 'en', strength: 2, maxVariable: "punct" } + ).hint("ADDRESS_1") .limit(limit); labels = await cursor.toArray(); @@ -85,8 +85,8 @@ export default async function handler(req, res) { }; const cursor = await db.collection(clc_name).find(queryAtlas, { projection }).collation( - { locale: 'en', strength: 1 } - ) + { locale: 'en', strength: 2 } + ).hint("LABEL_1") .limit(limit); labels = await cursor.toArray(); From 533ab336630e839a5fcf20734a1ea18236b45e27 Mon Sep 17 00:00:00 2001 From: MaghirSokker Date: Tue, 30 Jan 2024 04:04:57 +0100 Subject: [PATCH 09/11] limit testing --- pages/api/arbitrum/label.ts | 5 +++-- pages/api/contractlabel.ts | 5 +++-- pages/api/label.ts | 5 +++-- pages/api/optimism/label.ts | 5 +++-- pages/api/query.ts | 2 +- pages/api/query_q.ts | 2 +- pages/api/query_socials.ts | 2 +- pages/api/query_socials_a.ts | 2 +- pages/api/sc_names.ts | 2 +- pages/api/sc_names_sc.ts | 2 +- pages/api/search.ts | 6 +++--- pages/api/solana/label.ts | 39 ++++++------------------------------ pages/api/solana/search.ts | 12 +++++------ 13 files changed, 33 insertions(+), 56 deletions(-) diff --git a/pages/api/arbitrum/label.ts b/pages/api/arbitrum/label.ts index f378a30e..83ca0803 100644 --- a/pages/api/arbitrum/label.ts +++ b/pages/api/arbitrum/label.ts @@ -23,7 +23,7 @@ export default async function handler(req, res) { address = req.query.address; } - if (req.query.limit === "" || req.query.limit === undefined) { + if (req.query.limit === "" || req.query.limit === undefined || Number.isNaN(req.query.limit)) { limit = 20; } else { limit = Number(req.query.limit); @@ -42,7 +42,8 @@ export default async function handler(req, res) { try { if (address === "") { - labels = await db.collection(clc_name).find().limit(limit).toArray(); + res.status(200).json({ data: [] }); + return; } else { // Adjust the MongoDB query to search by 'address' const queryAtlas = { diff --git a/pages/api/contractlabel.ts b/pages/api/contractlabel.ts index 293b7a9f..1cf142fb 100644 --- a/pages/api/contractlabel.ts +++ b/pages/api/contractlabel.ts @@ -26,7 +26,7 @@ export default async function handler(req, res) { address = req.query.address; } - if (req.query.limit === "" || req.query.limit === undefined) { + if (req.query.limit === "" || req.query.limit === undefined || Number.isNaN(req.query.limit)) { limit = 20; } else { limit = Number(req.query.limit); @@ -45,7 +45,8 @@ export default async function handler(req, res) { try { if (address === "") { - labels = await db.collection(clc_name).find().limit(limit).toArray(); + res.status(200).json({ data: [] }); + return; } else { // Adjust the MongoDB query to search by 'address' const queryAtlas = { diff --git a/pages/api/label.ts b/pages/api/label.ts index 67161304..fa5aa818 100644 --- a/pages/api/label.ts +++ b/pages/api/label.ts @@ -26,7 +26,7 @@ export default async function handler(req, res) { address = req.query.address; } - if (req.query.limit === "" || req.query.limit === undefined) { + if (req.query.limit === "" || req.query.limit === undefined || Number.isNaN(req.query.limit)) { limit = 20; } else { limit = Number(req.query.limit); @@ -45,7 +45,8 @@ export default async function handler(req, res) { try { if (address === "") { - labels = await db.collection(clc_name).find().limit(limit).toArray(); + res.status(200).json({ data: [] }); + return; } else { // Adjust the MongoDB query to search by 'address' const queryAtlas = { diff --git a/pages/api/optimism/label.ts b/pages/api/optimism/label.ts index 8a410a7e..a2ddfa0d 100644 --- a/pages/api/optimism/label.ts +++ b/pages/api/optimism/label.ts @@ -24,7 +24,7 @@ export default async function handler(req, res) { address = req.query.address; } - if (req.query.limit === "" || req.query.limit === undefined) { + if (req.query.limit === "" || req.query.limit === undefined || Number.isNaN(req.query.limit)) { limit = 20; } else { limit = Number(req.query.limit); @@ -43,7 +43,8 @@ export default async function handler(req, res) { try { if (address === "") { - labels = await db.collection(clc_name).find().limit(limit).toArray(); + res.status(200).json({ data: [] }); + return; } else { // Adjust the MongoDB query to search by 'address' const queryAtlas = { diff --git a/pages/api/query.ts b/pages/api/query.ts index 13e97e52..3129ec2b 100644 --- a/pages/api/query.ts +++ b/pages/api/query.ts @@ -61,7 +61,7 @@ export default async function handler(req, res) { query = req.query.query } - if (req.query.limit === "" || req.query.limit === undefined) { + if (req.query.limit === "" || req.query.limit === undefined || Number.isNaN(req.query.limit)) { limit = 20 } else { limit = Number(req.query.limit) diff --git a/pages/api/query_q.ts b/pages/api/query_q.ts index ee9b3e62..b01b5df8 100644 --- a/pages/api/query_q.ts +++ b/pages/api/query_q.ts @@ -30,7 +30,7 @@ export default async function handler(req, res) { query = req.query.query } - if (req.query.limit === "" || req.query.limit === undefined) { + if (req.query.limit === "" || req.query.limit === undefined || Number.isNaN(req.query.limit)) { limit = 20 } else { limit = Number(req.query.limit) diff --git a/pages/api/query_socials.ts b/pages/api/query_socials.ts index ff5e0a82..da3932a8 100644 --- a/pages/api/query_socials.ts +++ b/pages/api/query_socials.ts @@ -55,7 +55,7 @@ export default async function handler( query = req.query.query as string } - if (req.query.limit === "" || req.query.limit === undefined) { + if (req.query.limit === "" || req.query.limit === undefined || Number.isNaN(req.query.limit)) { limit = 40 } else { limit = Number(req.query.limit) diff --git a/pages/api/query_socials_a.ts b/pages/api/query_socials_a.ts index 09073738..bd8769e2 100644 --- a/pages/api/query_socials_a.ts +++ b/pages/api/query_socials_a.ts @@ -55,7 +55,7 @@ export default async function handler( query = req.query.query as string } - if (req.query.limit === "" || req.query.limit === undefined) { + if (req.query.limit === "" || req.query.limit === undefined || Number.isNaN(req.query.limit)) { limit = 40 } else { limit = Number(req.query.limit) diff --git a/pages/api/sc_names.ts b/pages/api/sc_names.ts index c6002851..d87b48f0 100644 --- a/pages/api/sc_names.ts +++ b/pages/api/sc_names.ts @@ -60,7 +60,7 @@ export default async function handler(req, res) { query = req.query.query } - if (req.query.limit === "" || req.query.limit === undefined) { + if (req.query.limit === "" || req.query.limit === undefined || Number.isNaN(req.query.limit)) { limit = 20 } else { limit = Number(req.query.limit) diff --git a/pages/api/sc_names_sc.ts b/pages/api/sc_names_sc.ts index c9b65a1d..44c99f43 100644 --- a/pages/api/sc_names_sc.ts +++ b/pages/api/sc_names_sc.ts @@ -30,7 +30,7 @@ export default async function handler(req, res) { query = req.query.query } - if (req.query.limit === "" || req.query.limit === undefined) { + if (req.query.limit === "" || req.query.limit === undefined || Number.isNaN(req.query.limit)) { limit = 20 } else { limit = Number(req.query.limit) diff --git a/pages/api/search.ts b/pages/api/search.ts index dc007b94..66e79bd1 100644 --- a/pages/api/search.ts +++ b/pages/api/search.ts @@ -28,15 +28,15 @@ export default async function handler(req, res) { } - if (req.query.limit === "" || req.query.limit === undefined) { + if (req.query.limit === "" || req.query.limit === undefined || Number.isNaN(req.query.limit)) { limit = 20 } else { limit = Number(req.query.limit) } // max limit is 100 - if (limit > 1000) { - limit = 1000 + if (limit > 100) { + limit = 100 } // limit to only GET method or throw error diff --git a/pages/api/solana/label.ts b/pages/api/solana/label.ts index 0145750f..972b4d97 100644 --- a/pages/api/solana/label.ts +++ b/pages/api/solana/label.ts @@ -4,7 +4,7 @@ import { connectToDatabase } from "../../../lib/mongodb" export default async function handler(req, res) { // Database connection setup let db; - let address, limit, label; + let address, limit; try { const database = await connectToDatabase(); db = database.db; @@ -15,7 +15,7 @@ export default async function handler(req, res) { // Validate 'address' query parameter if (req.query.address === undefined && req.query.label === undefined) { - return res.status(400).json({ message: "Bad request: 'address'/'label' parameter missing" }); + return res.status(400).json({ message: "Bad request: 'address' parameter missing" }); } if (req.query.address === "" || req.query.address === undefined) { @@ -23,13 +23,8 @@ export default async function handler(req, res) { } else { address = req.query.address; } - if (req.query.label === "" || req.query.label === undefined) { - label = ""; - } else { - label = req.query.label; - } - if (req.query.limit === "" || req.query.limit === undefined) { + if (req.query.limit === "" || req.query.limit === undefined || Number.isNaN(req.query.limit)) { limit = 20; } else { limit = Number(req.query.limit); @@ -47,8 +42,9 @@ export default async function handler(req, res) { let labels = null; try { - if (address === "" && label === "") { - labels = await db.collection(clc_name).find().limit(limit).toArray(); + if (address === "") { + res.status(200).json({ data: [] }); + return; } else if (!!address) { // Adjust the MongoDB query to search by 'address' const queryAtlas = { @@ -68,29 +64,6 @@ export default async function handler(req, res) { ).hint("ADDRESS_1") .limit(limit); labels = await cursor.toArray(); - - console.log("solana label address :" + address.toString); - } else if (!!label) { - // Adjust the MongoDB query to search by 'label' - const queryAtlas = { - LABEL: label, - }; - - const projection = { - ADDRESS_NAME: 1, - LABEL_TYPE: 1, - LABEL_SUBTYPE: 1, - ADDRESS: 1, - LABEL: 1, - }; - - const cursor = await db.collection(clc_name).find(queryAtlas, { projection }).collation( - { locale: 'en', strength: 2 } - ).hint("LABEL_1") - .limit(limit); - labels = await cursor.toArray(); - - console.log("solana label query :" + label.toString); } labels = labels.map((lbl) => ({ diff --git a/pages/api/solana/search.ts b/pages/api/solana/search.ts index fb8d7927..fa6c10e2 100644 --- a/pages/api/solana/search.ts +++ b/pages/api/solana/search.ts @@ -17,12 +17,13 @@ export default async function handler( const skip = typeof req.query.skip === 'string' ? Number(req.query.skip) : 0; const limit = - typeof req.query.limit === 'string' ? Math.max(Math.min(Number(req.query.limit), 100), 20) : 20; + typeof req.query.limit === 'string' && !Number.isNaN(req.query.limit) ? Math.max(Math.min(Number(req.query.limit), 100), 1) : 20; - const search = - typeof req.query.search === 'string' ? req.query.search : undefined; + const searchtext = + typeof req.query.searchtext === 'string' ? req.query.searchtext : undefined; - const { solana_wallets } = await getSolana({ skip, limit, query: search }); + const { solana_wallets } = await getSolana({ skip, limit, query: searchtext }); + console.log("Searched element : " + searchtext); try { labels = solana_wallets.map((lbl) => ({ address: lbl.ADDRESS, @@ -36,7 +37,7 @@ export default async function handler( }; res.status(200).json(response); - } catch (error) { console.log(error); res.status(500); throw new Error("Unable to connect to database"); }; + } catch (error) { console.log(error); res.status(500); throw new Error("Bad request: 'searchtext' parameter missing"); }; @@ -77,7 +78,6 @@ const getSolana = async ({ } }) } - console.log("solana search query :" + query.toString) const result = await db.collection(clc_name).aggregate(pipeline).toArray() return { solana_wallets: result } } catch (error) { From 82b1684538d482b1a08cfee7bf96bcffcb7a8a6a Mon Sep 17 00:00:00 2001 From: MaghirSokker Date: Sun, 4 Feb 2024 01:21:43 +0100 Subject: [PATCH 10/11] batch label --- pages/api/batch_label.ts | 172 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 pages/api/batch_label.ts diff --git a/pages/api/batch_label.ts b/pages/api/batch_label.ts new file mode 100644 index 00000000..e87e9ec0 --- /dev/null +++ b/pages/api/batch_label.ts @@ -0,0 +1,172 @@ +import { Db } from 'mongodb' +import { connectToDatabase } from "../../lib/mongodb" +import type { NextApiRequest, NextApiResponse } from 'next' + +let db: Db; +const clc_sol = process.env.CLC_NAME_WLBLS_SOLANA; +const clc_eth = process.env.CLC_NAME_WLBLS; +export default async function handler( + req: NextApiRequest, + res: NextApiResponse +) { + + //only allows GET requests to API + if (req.method !== "GET") { + return res.status(405).json({ message: "Method not allowed" }); + } + let labels: Label[] | null = null; + try { + if (isSanitized(req)) { + // sets the collection name depending on the blockchain + const clc_name = isEth(req) ? clc_eth : clc_sol; + // extracts the data from the query + const adr_list: string[] = isEth(req) + ? extractAddress(req.query.eth) + : extractAddress(req.query.sol); + if (countLabels(adr_list) > 100) return res.status(403).json({ message: "Maximum address limit exceeded" }); + // validate formatting + adr_list.forEach((address) => { + if (!isAddressFormatted(address, isEth(req))) { + return res.status(403).json({ message: "Invalid address" }); + } + }) + // db call + const wallets = await getWallet({ limit: countLabels(adr_list), query: adr_list, clc_name: clc_name, is_eth: isEth(req) }); + // decoupling responses + + const response = { + "data": wallets, + }; + + res.status(200).json(response); + + } else return res.status(403).json({ message: "Pass eth/sol argument" }) + } catch (error) { console.log(error); throw new Error("error") } + + + + + +} +//decoupled async search query +const getWallet = async ({ + query, + limit, + is_eth, + clc_name, +}: { + query: string[], + limit: number, + is_eth: Boolean, + clc_name: string, +}) => { + try { + if (!db) await init() + const queryAtlas = + is_eth ? { "address": { "$in": query } } : { "ADDRESS": { "$in": query } }; + const projection = + is_eth ? { + address_name: 1, + label_type: 1, + label_subtype: 1, + address: 1, + label: 1, + } : { + ADDRESS_NAME: 1, + LABEL_TYPE: 1, + LABEL_SUBTYPE: 1, + ADDRESS: 1, + LABEL: 1, + }; + const collation = is_eth ? { locale: 'en', strength: 1 } : { locale: 'en', strength: 2, maxVariable: "punct" } + const index = "address_1"; + const cursor = await db.collection(clc_name) + .find(queryAtlas, { projection }) + .collation(collation) + .hint(index) + .limit(limit); + const labels = await cursor.toArray(); + const result = is_eth ? labels.map((label) => ({ + address: label.address, + address_name: label.address_name, + label_type: label.label_type, + label_subtype: label.label_subtype, + label: label.label, + score: label.score, + })) : labels.map((label) => ({ + address: label.ADDRESS, + address_name: label.ADDRESS_NAME, + label_type: label.LABEL_TYPE, + label_subtype: label.LABEL_SUBTYPE, + label: label.LABEL, + })); + return { batch_result: result } + } catch (error) { + return { error } + } +}; + +//connect to db +async function init() { + try { + const database = await connectToDatabase(); + db = database.db; + + } catch (error) { + console.log(error); + throw new Error("Unable to connect to database"); + } + return db; +}; + +//Type check on req (no empty address) +function isSanitized(res: NextApiRequest): Boolean { + try { + return ((typeof (res.query.sol) === "string" && res.query.eth === undefined && res.query.sol != "") || (res.query.sol === undefined && typeof (res.query.eth) === "string" && res.query.eth != "")); + + } catch (error) { + console.log(error); + throw new Error("Unable to assess query blockchain type") + } +}; + +//Determine if it's sol or eth +function isEth(res: NextApiRequest): Boolean { + return typeof (res.query.eth) === "string"; +}; + +//return a list from an array of addresses in string format +function extractAddress(addressArray: string | string[]): string[] { + + // Remove square brackets and split the string into an array + const adr_list = String(addressArray).split(','); + return adr_list; + +} + +//detects if addresses are valid +function isAddressFormatted(adr: string, isEth: Boolean): boolean { + if (isEth) { + return (adr.length === 42 && adr.startsWith("0x")); + } else { + return (adr.length <= 44 || adr.length >= 32); + } +}; + +//label counter +function countLabels(adr_list: string[]): number { + return adr_list.length; +} + +//response interface ? +type ResponseData = { + [key: string]: any; + +} +type Label = { + address: string, + address_name: string, + label_type: string, + label_subtype: string, + label: string, +} \ No newline at end of file From c1c964925d57aa0f8e4b35b92a567089a90ba56c Mon Sep 17 00:00:00 2001 From: MaghirSokker Date: Sun, 4 Feb 2024 19:06:13 +0100 Subject: [PATCH 11/11] Separated into solana & eth bathlabels --- pages/api/batch_label.ts | 68 +++++---------- pages/api/solana/batch_label.ts | 146 ++++++++++++++++++++++++++++++++ 2 files changed, 167 insertions(+), 47 deletions(-) create mode 100644 pages/api/solana/batch_label.ts diff --git a/pages/api/batch_label.ts b/pages/api/batch_label.ts index e87e9ec0..34ef8683 100644 --- a/pages/api/batch_label.ts +++ b/pages/api/batch_label.ts @@ -3,8 +3,7 @@ import { connectToDatabase } from "../../lib/mongodb" import type { NextApiRequest, NextApiResponse } from 'next' let db: Db; -const clc_sol = process.env.CLC_NAME_WLBLS_SOLANA; -const clc_eth = process.env.CLC_NAME_WLBLS; +const clc_name = process.env.CLC_NAME_WLBLS; export default async function handler( req: NextApiRequest, res: NextApiResponse @@ -16,22 +15,19 @@ export default async function handler( } let labels: Label[] | null = null; try { + // asserts if addresses is empty if (isSanitized(req)) { - // sets the collection name depending on the blockchain - const clc_name = isEth(req) ? clc_eth : clc_sol; // extracts the data from the query - const adr_list: string[] = isEth(req) - ? extractAddress(req.query.eth) - : extractAddress(req.query.sol); + const adr_list: string[] = extractAddress(req.query.addresses) if (countLabels(adr_list) > 100) return res.status(403).json({ message: "Maximum address limit exceeded" }); // validate formatting adr_list.forEach((address) => { - if (!isAddressFormatted(address, isEth(req))) { - return res.status(403).json({ message: "Invalid address" }); + if (!isAddressFormatted(address)) { + return res.status(403).json({ message: "Invalid address detected : " + address }); } }) // db call - const wallets = await getWallet({ limit: countLabels(adr_list), query: adr_list, clc_name: clc_name, is_eth: isEth(req) }); + const wallets = await getWallet({ limit: countLabels(adr_list), query: adr_list, clc_name: clc_name }); // decoupling responses const response = { @@ -52,33 +48,24 @@ export default async function handler( const getWallet = async ({ query, limit, - is_eth, clc_name, }: { query: string[], limit: number, - is_eth: Boolean, clc_name: string, }) => { try { if (!db) await init() - const queryAtlas = - is_eth ? { "address": { "$in": query } } : { "ADDRESS": { "$in": query } }; + const queryAtlas = { "address": { "$in": query } }; const projection = - is_eth ? { - address_name: 1, - label_type: 1, - label_subtype: 1, - address: 1, - label: 1, - } : { - ADDRESS_NAME: 1, - LABEL_TYPE: 1, - LABEL_SUBTYPE: 1, - ADDRESS: 1, - LABEL: 1, - }; - const collation = is_eth ? { locale: 'en', strength: 1 } : { locale: 'en', strength: 2, maxVariable: "punct" } + { + address_name: 1, + label_type: 1, + label_subtype: 1, + address: 1, + label: 1, + } + const collation = { locale: 'en', strength: 1 }; const index = "address_1"; const cursor = await db.collection(clc_name) .find(queryAtlas, { projection }) @@ -86,19 +73,13 @@ const getWallet = async ({ .hint(index) .limit(limit); const labels = await cursor.toArray(); - const result = is_eth ? labels.map((label) => ({ + const result = labels.map((label) => ({ address: label.address, address_name: label.address_name, label_type: label.label_type, label_subtype: label.label_subtype, label: label.label, score: label.score, - })) : labels.map((label) => ({ - address: label.ADDRESS, - address_name: label.ADDRESS_NAME, - label_type: label.LABEL_TYPE, - label_subtype: label.LABEL_SUBTYPE, - label: label.LABEL, })); return { batch_result: result } } catch (error) { @@ -122,18 +103,13 @@ async function init() { //Type check on req (no empty address) function isSanitized(res: NextApiRequest): Boolean { try { - return ((typeof (res.query.sol) === "string" && res.query.eth === undefined && res.query.sol != "") || (res.query.sol === undefined && typeof (res.query.eth) === "string" && res.query.eth != "")); - + return typeof (res.query.addresses) === "string" && res.query.addresses != "" } catch (error) { console.log(error); throw new Error("Unable to assess query blockchain type") } }; -//Determine if it's sol or eth -function isEth(res: NextApiRequest): Boolean { - return typeof (res.query.eth) === "string"; -}; //return a list from an array of addresses in string format function extractAddress(addressArray: string | string[]): string[] { @@ -145,12 +121,10 @@ function extractAddress(addressArray: string | string[]): string[] { } //detects if addresses are valid -function isAddressFormatted(adr: string, isEth: Boolean): boolean { - if (isEth) { - return (adr.length === 42 && adr.startsWith("0x")); - } else { - return (adr.length <= 44 || adr.length >= 32); - } +function isAddressFormatted(adr: string): boolean { + + return (adr.length === 42 && adr.startsWith("0x")); + }; //label counter diff --git a/pages/api/solana/batch_label.ts b/pages/api/solana/batch_label.ts new file mode 100644 index 00000000..d6f7eee4 --- /dev/null +++ b/pages/api/solana/batch_label.ts @@ -0,0 +1,146 @@ +import { Db } from 'mongodb' +import { connectToDatabase } from "../../../lib/mongodb" +import type { NextApiRequest, NextApiResponse } from 'next' + +let db: Db; +const clc_name = process.env.CLC_NAME_WLBLS_SOLANA; +export default async function handler( + req: NextApiRequest, + res: NextApiResponse +) { + + //only allows GET requests to API + if (req.method !== "GET") { + return res.status(405).json({ message: "Method not allowed" }); + } + let labels: Label[] | null = null; + try { + + // asserts if addresses is empty + if (isSanitized(req)) { + // extracts the data from the query + const adr_list: string[] = extractAddress(req.query.addresses); + if (countLabels(adr_list) > 100) return res.status(403).json({ message: "Maximum address limit exceeded" }); + // validate formatting + adr_list.forEach((address) => { + if (!isAddressFormatted(address)) { + return res.status(403).json({ message: "Invalid address detected : " + address }); + } + }) + // db call + const wallets = await getWallet({ limit: countLabels(adr_list), query: adr_list, clc_name: clc_name }); + // decoupling responses + + const response = { + "data": wallets, + }; + + res.status(200).json(response); + + } else return res.status(403).json({ message: "Pass addresses argument" }) + } catch (error) { console.log(error); throw new Error("error") } + + + + + +} +//decoupled async search query +const getWallet = async ({ + query, + limit, + clc_name, +}: { + query: string[], + limit: number, + clc_name: string, +}) => { + try { + if (!db) await init() + const queryAtlas = { "ADDRESS": { "$in": query } }; + const projection = + { + ADDRESS_NAME: 1, + LABEL_TYPE: 1, + LABEL_SUBTYPE: 1, + ADDRESS: 1, + LABEL: 1, + }; + const collation = { locale: 'en', strength: 2, maxVariable: "punct" } + const index = "address_1"; + const cursor = await db.collection(clc_name) + .find(queryAtlas, { projection }) + .collation(collation) + .hint(index) + .limit(limit); + const labels = await cursor.toArray(); + const result = labels.map((label) => ({ + address: label.ADDRESS, + address_name: label.ADDRESS_NAME, + label_type: label.LABEL_TYPE, + label_subtype: label.LABEL_SUBTYPE, + label: label.LABEL, + })); + return { batch_result: result } + } catch (error) { + return { error } + } +}; + +//connect to db +async function init() { + try { + const database = await connectToDatabase(); + db = database.db; + + } catch (error) { + console.log(error); + throw new Error("Unable to connect to database"); + } + return db; +}; + +//Type check on req (no empty address) +function isSanitized(res: NextApiRequest): Boolean { + try { + return typeof (res.query.addresses) === "string" && res.query.addresses != "" + } catch (error) { + console.log(error); + throw new Error("addresses parameter required") + } +}; + + +//return a list from an array of addresses in string format +function extractAddress(addressArray: string | string[]): string[] { + + // Remove square brackets and split the string into an array + const adr_list = String(addressArray).split(','); + return adr_list; + +} + +//detects if addresses are valid +function isAddressFormatted(adr: string): boolean { + + return (adr.length <= 44 || adr.length >= 32); + +}; + +//label counter +function countLabels(adr_list: string[]): number { + return adr_list.length; +} + +//response interface ? +type ResponseData = { + [key: string]: any; + +} +type Label = { + address: string, + address_name: string, + label_type: string, + label_subtype: string, + label: string, +} \ No newline at end of file