From 120284abd303d8a1535083fb8af9fca8e39eb042 Mon Sep 17 00:00:00 2001 From: jdh7190 Date: Tue, 24 Aug 2021 11:14:09 -0400 Subject: [PATCH] WhatOnChain API support --- README.md | 3 +- package.json | 4 +- src/config.js | 2 + src/index.js | 4 +- src/whatsonchain.js | 164 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 174 insertions(+), 3 deletions(-) create mode 100644 src/whatsonchain.js diff --git a/README.md b/README.md index 46bb057..2b4efa0 100644 --- a/README.md +++ b/README.md @@ -54,9 +54,10 @@ Create a .env file or set the following environment variables before running to | Name | Description | Default | | ---- | ----------- | ------- | -| **API**| mattercloud, planaria, bitcoin-node, run, or none | mattercloud +| **API**| mattercloud, planaria, whatsonchain, bitcoin-node, run, or none | mattercloud | **MATTERCLOUD_KEY** | Mattercloud API key | undefined | **PLANARIA_TOKEN** | Planaria API key | undefined +| **WHATSONCHAIN_KEY** | WhatsOnChain API key | undefined | **ZMQ_URL** | Only for bitcoin-node. ZMQ tcp url | null | **RPC_URL** | Only for bitcoin-node. bitcoin RPC http url | null | **NETWORK** | Bitcoin network (main or test) | main diff --git a/package.json b/package.json index a591328..d1f87ba 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "axios": "^0.21.1", "better-sqlite3": "^7.4.1", "body-parser": "^1.19.0", + "centrifuge": "^2.7.7", "cors": "^2.8.5", "dotenv": "^8.2.0", "event-stream": "^4.0.1", @@ -42,7 +43,8 @@ "morgan": "^1.10.0", "node-fetch": "^2.6.1", "reconnecting-eventsource": "^1.1.0", - "run-sdk": "^0.6.18" + "run-sdk": "^0.6.18", + "ws": "^8.2.0" }, "optionalDependencies": { "zeromq": "^5.2.8" diff --git a/src/config.js b/src/config.js index 741a77c..5f69806 100644 --- a/src/config.js +++ b/src/config.js @@ -13,6 +13,7 @@ require('dotenv').config() const API = process.env.API || 'mattercloud' const MATTERCLOUD_KEY = process.env.MATTERCLOUD_KEY const PLANARIA_TOKEN = process.env.PLANARIA_TOKEN +const WHATSONCHAIN_KEY = process.env.WHATSONCHAIN_KEY const NETWORK = process.env.NETWORK || 'main' const DB = process.env.DB || 'run.db' const PORT = parseInt(process.env.PORT, 0) @@ -104,6 +105,7 @@ module.exports = { API, MATTERCLOUD_KEY, PLANARIA_TOKEN, + WHATSONCHAIN_KEY, NETWORK, DB, PORT, diff --git a/src/index.js b/src/index.js index 80419dc..6a8b8d1 100644 --- a/src/index.js +++ b/src/index.js @@ -7,11 +7,12 @@ const Indexer = require('./indexer') const Server = require('./server') const { - API, DB, NETWORK, PORT, FETCH_LIMIT, WORKERS, MATTERCLOUD_KEY, PLANARIA_TOKEN, START_HEIGHT, + API, DB, NETWORK, PORT, FETCH_LIMIT, WORKERS, MATTERCLOUD_KEY, PLANARIA_TOKEN, WHATSONCHAIN_KEY, START_HEIGHT, MEMPOOL_EXPIRATION, ZMQ_URL, RPC_URL, DEFAULT_TRUSTLIST, DEBUG, SERVE_ONLY } = require('./config') const MatterCloud = require('./mattercloud') const Planaria = require('./planaria') +const WhatsOnChain = require('./whatsonchain') const RunConnectFetcher = require('./run-connect') const BitcoinNodeConnection = require('./bitcoin-node-connection') const BitcoinRpc = require('./bitcoin-rpc') @@ -33,6 +34,7 @@ let api = null switch (API) { case 'mattercloud': api = new MatterCloud(MATTERCLOUD_KEY, logger); break case 'planaria': api = new Planaria(PLANARIA_TOKEN, logger); break + case 'whatsonchain': api = new WhatsOnChain(WHATSONCHAIN_KEY, logger); break; case 'bitcoin-node': if (ZMQ_URL === null) { throw new Error('please specify ZQM_URL when using bitcoin-node API') diff --git a/src/whatsonchain.js b/src/whatsonchain.js new file mode 100644 index 0000000..7214acf --- /dev/null +++ b/src/whatsonchain.js @@ -0,0 +1,164 @@ +/** + * api.js + * + * API used to get transaction data + */ + +// ------------------------------------------------------------------------------------------------ +// Api +// ------------------------------------------------------------------------------------------------ + +const axios = require('axios') +const Centrifuge = require('centrifuge') +const WebSocket = require('ws') + +// ------------------------------------------------------------------------------------------------ +// Globals +// ------------------------------------------------------------------------------------------------ + +const RUN_0_6_FILTER = '006a0372756e0105' + +class WhatsOnChain { + constructor(apikey, logger) { + this.logger = logger + this.config = { + headers: { + 'woc-api-key': apikey + }, + timeout: 60000 + } + } + // Connect to the API at a particular block height and network + async connect (height, network) { + if (network !== 'main') throw new Error(`Network not yet supported with WhatsOnChain: ${network}`) + } + + // Stop any connections + async disconnect () { + if (this.mempoolEvents) { + this.mempoolEvents.close() + this.mempoolEvents = null + } + } + + // Returns the rawtx of the txid, or throws an error + async fetch (txid) { + const response = await axios.get(`https://api.whatsonchain.com/v1/bsv/main/tx/${txid}/hex`, this.config) + const detail = await axios.get(`https://api.whatsonchain.com/v1/bsv/main/tx/hash/${txid}`, this.config) + const hex = response.data + const height = detail.data.blockheight === 0 ? -1 : detail.data.blockheight + const time = detail.data.blocktime === 0 ? null : detail.data.blocktime + return { hex, height, time } + } + + // Gets the next relevant block of transactions to add + // currHash may be null + // If there is a next block, return: { height, hash, txids, txhexs? } + // If there is no next block yet, return null + // If the current block passed was reorged, return { reorg: true } + async getNextBlock (currHeight, currHash) { + const height = currHeight + 1 + let res, txs = [] + try { + if (height) { + res = await axios.get(`https://api.whatsonchain.com/v1/bsv/main/block/height/${height}`, this.config) + } + const hash = res.data.hash + if (!hash) { return undefined } + const time = res.data.time + const prevHash = res.data.previousblockhash + if (currHash && prevHash !== currHash) return { reorg: true } + if (res.data.tx !== undefined || res.data.tx !== null) { + res.data.tx.forEach(tx => { + txs.push(tx) + }) + } + if (res.data.pages) { + for (let page of res.data.pages.uri) { + const nes = await axios.get(`https://api.whatsonchain.com/v1/bsv/main${page}`, this.config) + if (nes.data) { + nes.data.forEach(tx => { + txs.push(tx) + }) + } + } + } + let txids = [], transactions = [], x = 0 + const mod = txs.length % 20 + const looptimes = parseInt(txs.length / 20) + for (let i = 0; i < looptimes; i++) { + txids = [] + for (let j = 0; j < 20; j++) { + txids.push(txs[x]) + x++ + } + const h = await axios.post('https://api.whatsonchain.com/v1/bsv/main/txs/hex', { txids }, this.config) + if (h.data) { + h.data.forEach(t => { + if (t.hex.includes(RUN_0_6_FILTER)) { + transactions.push(t) + } + }) + } + } + txids = [] + for (let k = txs.length - 1; k > txs.length - mod; k--) { + txids.push(txs[k]) + } + if (txids.length) { + const h = await axios.post('https://api.whatsonchain.com/v1/bsv/main/txs/hex', { txids }, this.config) + if (h.data) { + h.data.forEach(t => { + if (t.hex.includes(RUN_0_6_FILTER)) { + transactions.push(t) + } + }) + } + } + txids = transactions.map(t => t.txid) + const txhexs = transactions.map(t => t.hex) + return { height, hash, time, txids, txhexs } + } catch (e) { + if (e.response && e.response.status === 404) return undefined + throw e + } + } + + // Begins listening for mempool transactions + // The callback should be called with txid and optionally rawtx when mempool tx is found + // The crawler will call this after the block syncing is up-to-date. + async listenForMempool (mempoolTxCallback) { + this.logger.info('Listening for mempool via WhatsOnChain') + + return new Promise((resolve, reject) => { + this.mempoolEvents = new Centrifuge('wss://socket.whatsonchain.com/mempool', { + websocket: WebSocket + }) + + this.mempoolEvents.on('connect', ctx => { + console.log('Connected with client ID ' + ctx.client + ' over ' + ctx.transport ) + resolve() + }) + + this.mempoolEvents.close = ctx => { + console.log('Disconnected.') + } + + this.mempoolEvents.on('error', ctx => { + reject(ctx) + }) + + this.mempoolEvents.on('publish', message => { + const hex = message.data.hex + if (hex.includes(RUN_0_6_FILTER)) { + mempoolTxCallback(message.data.hash, hex) + } + }) + this.mempoolEvents.connect() + }) + } + } + + // ------------------------------------------------------------------------------------------------ + + module.exports = WhatsOnChain \ No newline at end of file