From 8bf9256f02552e869c0a3bc991cfe70a9d200499 Mon Sep 17 00:00:00 2001 From: Sam Holmes Date: Thu, 30 Jan 2025 15:17:39 -0800 Subject: [PATCH 1/4] Fix typo in index --- src/initDbs.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/initDbs.ts b/src/initDbs.ts index 2ab009c4..75526f2f 100644 --- a/src/initDbs.ts +++ b/src/initDbs.ts @@ -14,7 +14,7 @@ const transactionIndexFields: string[][] = [ ['status', 'payoutAmount', 'depositAmount'], ['status', 'payoutCurrency', 'isoDate'], ['status', 'usdValue'], - ['status', 'usdvalue', 'timestamp'], + ['status', 'usdValue', 'timestamp'], ['usdValue'], ['timestamp'] ] From 19c84a96d2f1cfc22210b358d9adfea252b039da Mon Sep 17 00:00:00 2001 From: Sam Holmes Date: Thu, 30 Jan 2025 17:07:45 -0800 Subject: [PATCH 2/4] Use edge-server-tools utilities for db inits --- package.json | 1 + src/initDbs.ts | 198 ++++++++++++++++++++++--------------------------- yarn.lock | 197 +++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 285 insertions(+), 111 deletions(-) diff --git a/package.json b/package.json index 43d94f99..69289d57 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,7 @@ "csv-stringify": "^6.2.0", "date-fns": "^2.16.1", "date-fns-tz": "^1.1.4", + "edge-server-tools": "^0.2.19", "express": "^4.17.3", "nano": "^10.1.0", "node-fetch": "^2.6.7", diff --git a/src/initDbs.ts b/src/initDbs.ts index 75526f2f..797f4793 100644 --- a/src/initDbs.ts +++ b/src/initDbs.ts @@ -1,126 +1,104 @@ -import nano from 'nano' +import { + DatabaseSetup, + JsDesignDocument, + makeMangoIndex, + MangoDesignDocument, + setupDatabase +} from 'edge-server-tools' import { config } from './config' -import { datelog } from './util' -const nanoDb = nano(config.couchDbFullpath) - -const transactionIndexFields: string[][] = [ - ['isoDate'], - ['status'], - ['status', 'depositCurrency', 'isoDate'], - ['status', 'depositCurrency', 'payoutCurrency', 'isoDate'], - ['status', 'isoDate'], - ['status', 'payoutAmount', 'depositAmount'], - ['status', 'payoutCurrency', 'isoDate'], - ['status', 'usdValue'], - ['status', 'usdValue', 'timestamp'], - ['usdValue'], - ['timestamp'] -] - -const transactionIndexFieldsNoPartition: string[][] = [ - ['depositAddress'], - ['payoutAddress'] -] - -interface Index { - index: { fields: string[] } - ddoc: string - name: string - type: 'json' - partitioned: boolean +interface DesignDocumentMap { + [designDocName: string]: MangoDesignDocument | JsDesignDocument } -const transactionIndexes: Index[] = [] - -transactionIndexFields.forEach(index => { - const indexLower = index.map(i => i.toLowerCase()) - const out: Index = { - index: { fields: index }, - ddoc: indexLower.join('-'), - name: indexLower.join('-'), - type: 'json', +function fieldsToDesign( + fields: string[], + withPartitionVariant: boolean = true +): DesignDocumentMap { + const indexLower = fields.map(i => i.toLowerCase()) + const name = indexLower.join('-') + const out: DesignDocumentMap = {} + out[`_design/${name}`] = makeMangoIndex(name, fields, { partitioned: false + }) + if (withPartitionVariant) { + out[`_design/${name}-p`] = makeMangoIndex(`${name}-p`, fields, { + partitioned: true + }) } - transactionIndexes.push(out) - const out2 = { ...out } - out2.ddoc += '-p' - out2.name += '-p' - out2.partitioned = true - transactionIndexes.push(out2) -}) + return out +} -transactionIndexFieldsNoPartition.forEach(index => { - const indexLower = index.map(i => i.toLowerCase()) - const out: Index = { - index: { fields: index }, - ddoc: indexLower.join('-'), - name: indexLower.join('-'), - type: 'json', - partitioned: false - } - transactionIndexes.push(out) -}) +const transactionIndexes: DesignDocumentMap = { + ...fieldsToDesign(['isoDate']), + ...fieldsToDesign(['status']), + ...fieldsToDesign(['status', 'depositCurrency', 'isoDate']), + ...fieldsToDesign(['status', 'depositCurrency', 'payoutCurrency', 'isoDate']), + ...fieldsToDesign(['status', 'isoDate']), + ...fieldsToDesign(['status', 'payoutAmount', 'depositAmount']), + ...fieldsToDesign(['status', 'payoutCurrency', 'isoDate']), + ...fieldsToDesign(['status', 'usdValue']), + ...fieldsToDesign(['status', 'usdValue', 'timestamp']), + ...fieldsToDesign(['usdValue']), + ...fieldsToDesign(['timestamp']), + ...fieldsToDesign(['depositAddress'], false), + ...fieldsToDesign(['payoutAddress'], false) +} -const cacheIndexes: Index[] = [ - { - index: { fields: ['timestamp'] }, - ddoc: 'timestamp-p', - name: 'timestamp-p', - type: 'json' as 'json', +const cacheIndexes: DesignDocumentMap = { + '_design/timestamp-p': makeMangoIndex('timestamp-p', ['timestamp'], { partitioned: true - } -] - -const options = { partitioned: true } + }) +} -const DB_NAMES = [ - { name: 'reports_apps' }, - { name: 'reports_settings' }, - { - name: 'reports_transactions', - options, - indexes: transactionIndexes - }, - { name: 'reports_progresscache', options }, - { - name: 'reports_hour', - options, - indexes: cacheIndexes - }, - { - name: 'reports_day', - options, - indexes: cacheIndexes - }, - { - name: 'reports_month', - options, - indexes: cacheIndexes +const appsDatabaseSetup: DatabaseSetup = { + name: 'reports_apps' +} +const settingsDatabaseSetup: DatabaseSetup = { + name: 'reports_settings' +} +const transactionsDatabaseSetup: DatabaseSetup = { + name: 'reports_transactions', + options: { partitioned: true }, + documents: { + ...transactionIndexes } +} +const progressCacheDatabaseSetup: DatabaseSetup = { + name: 'reports_progresscache', + options: { partitioned: true } +} +const hourDatabaseSetup: DatabaseSetup = { + name: 'reports_hour', + options: { partitioned: true }, + documents: cacheIndexes +} +const dayDatabaseSetup: DatabaseSetup = { + name: 'reports_day', + options: { partitioned: true }, + documents: cacheIndexes +} +const monthDatabaseSetup: DatabaseSetup = { + name: 'reports_month', + options: { partitioned: true }, + documents: cacheIndexes +} + +const databases = [ + appsDatabaseSetup, + settingsDatabaseSetup, + transactionsDatabaseSetup, + progressCacheDatabaseSetup, + hourDatabaseSetup, + dayDatabaseSetup, + monthDatabaseSetup ] export async function initDbs(): Promise { - // get a list of all databases within couchdb - const result = await nanoDb.db.list() - datelog(result) - // if database does not exist, create it - for (const dbName of DB_NAMES) { - if (!result.includes(dbName.name)) { - await nanoDb.db.create(dbName.name, dbName.options) - } - if (dbName.indexes !== undefined) { - const currentDb = nanoDb.db.use(dbName.name) - for (const dbIndex of dbName.indexes) { - try { - await currentDb.get(`_design/${dbIndex.ddoc}`) - datelog(`${dbName.name} already has '${dbIndex.name}' index.`) - } catch { - await currentDb.createIndex(dbIndex) - datelog(`Created '${dbIndex.name}' index for ${dbName.name}.`) - } - } - } - } + await Promise.all( + databases.map( + async setup => await setupDatabase(config.couchDbFullpath, setup) + ) + ) } diff --git a/yarn.lock b/yarn.lock index 59bcae51..16e957a8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1504,6 +1504,15 @@ axios@^1.1.3: form-data "^4.0.0" proxy-from-env "^1.1.0" +axios@^1.7.4: + version "1.7.9" + resolved "https://registry.yarnpkg.com/axios/-/axios-1.7.9.tgz#d7d071380c132a24accda1b2cfc1535b79ec650a" + integrity sha512-LhLcE7Hbiryz8oMDdDptSrWowmB4Bl6RCt6sIJKpRB4XtVf0iEgewX3au/pJqm+Py1kCASkb/FFKjxQaLtxJvw== + dependencies: + follow-redirects "^1.15.6" + form-data "^4.0.0" + proxy-from-env "^1.1.0" + babel-runtime@^6.26.0: version "6.26.0" resolved "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.26.0.tgz" @@ -1786,6 +1795,14 @@ cacheable-request@^6.0.0: normalize-url "^4.1.0" responselike "^1.0.2" +call-bind-apply-helpers@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.1.tgz#32e5892e6361b29b0b545ba6f7763378daca2840" + integrity sha512-BhYE+WDaywFg2TBWYNXAE+8B1ATnThNBqXHP5nQu0jWJdVvY2hvkpyB3qOmtmDePiS5/BDQ8wASEWGMWRG148g== + dependencies: + es-errors "^1.3.0" + function-bind "^1.1.2" + call-bind@^1.0.0, call-bind@^1.0.2: version "1.0.2" resolved "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz" @@ -1794,6 +1811,14 @@ call-bind@^1.0.0, call-bind@^1.0.2: function-bind "^1.1.1" get-intrinsic "^1.0.2" +call-bound@^1.0.2: + version "1.0.3" + resolved "https://registry.yarnpkg.com/call-bound/-/call-bound-1.0.3.tgz#41cfd032b593e39176a71533ab4f384aa04fd681" + integrity sha512-YTd+6wGlNlPxSuri7Y6X8tY2dmm12UMH66RpKMhiX6rsk5wXXnYgbUcOt8kiS31/AjfoTOvCsE+w8nZQLQnzHA== + dependencies: + call-bind-apply-helpers "^1.0.1" + get-intrinsic "^1.2.6" + callsite@1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/callsite/-/callsite-1.0.0.tgz" @@ -1926,7 +1951,7 @@ cleaner-config@^0.1.10: minimist "^1.2.5" sucrase "^3.17.1" -cleaners@^0.3.17: +cleaners@^0.3.11, cleaners@^0.3.17: version "0.3.17" resolved "https://registry.yarnpkg.com/cleaners/-/cleaners-0.3.17.tgz#dae498f3d49b7e9364050402d2f4ad09abcd31ba" integrity sha512-X5acjsLwJK+JEK5hv0Rve7G78+E6iYh1TzJZ40z7Yjrba0WhW6spTq28WgG9w+AK+YQIOHtQTrzaiuntMBBIwQ== @@ -2594,6 +2619,15 @@ dotenv@^7.0.0: resolved "https://registry.npmjs.org/dotenv/-/dotenv-7.0.0.tgz" integrity sha512-M3NhsLbV1i6HuGzBUH8vXrtxOk+tWmzWKDMbAVSUp3Zsjm7ywFeuwrUXhmhQyRK1q5B5GGy7hcXPbj3bnfZg2g== +dunder-proto@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/dunder-proto/-/dunder-proto-1.0.1.tgz#d7ae667e1dc83482f8b70fd0f6eefc50da30f58a" + integrity sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A== + dependencies: + call-bind-apply-helpers "^1.0.1" + es-errors "^1.3.0" + gopd "^1.2.0" + duplexer3@^0.1.4: version "0.1.4" resolved "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz" @@ -2607,6 +2641,16 @@ ecc-jsbn@~0.1.1: jsbn "~0.1.0" safer-buffer "^2.1.0" +edge-server-tools@^0.2.19: + version "0.2.19" + resolved "https://registry.yarnpkg.com/edge-server-tools/-/edge-server-tools-0.2.19.tgz#495a7c691b32c9a2d9abee3115bca38ad72efd5e" + integrity sha512-iCHxop9064CwrJtpgmbXlMfx93FuNiwmq63c1q/i+mSk5Q0+QCRVP81GXmuxzm9ts0z2s4smhkbd715QTH2Ebw== + dependencies: + cleaners "^0.3.11" + nano "^10.1.3" + node-fetch "^2.6.1" + yavent "^0.1.3" + ee-first@1.1.1: version "1.1.1" resolved "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz" @@ -2789,6 +2833,16 @@ es-abstract@^1.22.1: unbox-primitive "^1.0.2" which-typed-array "^1.1.11" +es-define-property@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/es-define-property/-/es-define-property-1.0.1.tgz#983eb2f9a6724e9303f61addf011c72e09e0b0fa" + integrity sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g== + +es-errors@^1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/es-errors/-/es-errors-1.3.0.tgz#05f75a25dab98e4fb1dcd5e1472c0546d5057c8f" + integrity sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw== + es-iterator-helpers@^1.0.12: version "1.0.15" resolved "https://registry.yarnpkg.com/es-iterator-helpers/-/es-iterator-helpers-1.0.15.tgz#bd81d275ac766431d19305923707c3efd9f1ae40" @@ -2809,6 +2863,13 @@ es-iterator-helpers@^1.0.12: iterator.prototype "^1.1.2" safe-array-concat "^1.0.1" +es-object-atoms@^1.0.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/es-object-atoms/-/es-object-atoms-1.1.1.tgz#1c4f2c4837327597ce69d2ca190a7fdd172338c1" + integrity sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA== + dependencies: + es-errors "^1.3.0" + es-set-tostringtag@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/es-set-tostringtag/-/es-set-tostringtag-2.0.1.tgz#338d502f6f674301d710b80c8592de8a15f09cd8" @@ -3444,6 +3505,11 @@ follow-redirects@^1.14.0, follow-redirects@^1.15.0: resolved "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz" integrity sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA== +follow-redirects@^1.15.6: + version "1.15.9" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.9.tgz#a604fa10e443bf98ca94228d9eebcc2e8a2c8ee1" + integrity sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ== + for-each@^0.3.3: version "0.3.3" resolved "https://registry.yarnpkg.com/for-each/-/for-each-0.3.3.tgz#69b447e88a0a5d32c3e7084f3f1710034b21376e" @@ -3529,6 +3595,11 @@ function-bind@^1.1.1: resolved "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz" integrity sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A== +function-bind@^1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.2.tgz#2c02d864d97f3ea6c8830c464cbd11ab6eab7a1c" + integrity sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA== + function.prototype.name@^1.1.5, function.prototype.name@^1.1.6: version "1.1.6" resolved "https://registry.yarnpkg.com/function.prototype.name/-/function.prototype.name-1.1.6.tgz#cdf315b7d90ee77a4c6ee216c3c3362da07533fd" @@ -3573,6 +3644,22 @@ get-intrinsic@^1.1.3, get-intrinsic@^1.2.0, get-intrinsic@^1.2.1: has-proto "^1.0.1" has-symbols "^1.0.3" +get-intrinsic@^1.2.5, get-intrinsic@^1.2.6: + version "1.2.7" + resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.2.7.tgz#dcfcb33d3272e15f445d15124bc0a216189b9044" + integrity sha512-VW6Pxhsrk0KAOqs3WEd0klDiF/+V7gQOpAvY1jVU/LHmaD/kQO4523aiJuikX/QAKYiW6x8Jh+RJej1almdtCA== + dependencies: + call-bind-apply-helpers "^1.0.1" + es-define-property "^1.0.1" + es-errors "^1.3.0" + es-object-atoms "^1.0.0" + function-bind "^1.1.2" + get-proto "^1.0.0" + gopd "^1.2.0" + has-symbols "^1.1.0" + hasown "^2.0.2" + math-intrinsics "^1.1.0" + get-own-enumerable-property-symbols@^3.0.0: version "3.0.2" resolved "https://registry.npmjs.org/get-own-enumerable-property-symbols/-/get-own-enumerable-property-symbols-3.0.2.tgz" @@ -3583,6 +3670,14 @@ get-port@^4.2.0: resolved "https://registry.npmjs.org/get-port/-/get-port-4.2.0.tgz" integrity sha512-/b3jarXkH8KJoOMQc3uVGHASwGLPq3gSFJ7tgJm2diza+bydJPTGOibin2steecKeOylE8oY2JERlVWkAJO6yw== +get-proto@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/get-proto/-/get-proto-1.0.1.tgz#150b3f2743869ef3e851ec0c49d15b1d14d00ee1" + integrity sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g== + dependencies: + dunder-proto "^1.0.1" + es-object-atoms "^1.0.0" + get-stream@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz" @@ -3703,6 +3798,11 @@ gopd@^1.0.1: dependencies: get-intrinsic "^1.1.3" +gopd@^1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/gopd/-/gopd-1.2.0.tgz#89f56b8217bdbc8802bd299df6d7f1081d7e51a1" + integrity sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg== + got@9.6.0: version "9.6.0" resolved "https://registry.npmjs.org/got/-/got-9.6.0.tgz" @@ -3832,6 +3932,11 @@ has-symbols@^1.0.2, has-symbols@^1.0.3: resolved "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz" integrity sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A== +has-symbols@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.1.0.tgz#fc9c6a783a084951d0b971fe1018de813707a338" + integrity sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ== + has-to-string-tag-x@^1.2.0: version "1.4.1" resolved "https://registry.npmjs.org/has-to-string-tag-x/-/has-to-string-tag-x-1.4.1.tgz" @@ -3870,6 +3975,13 @@ hash.js@^1.0.0, hash.js@^1.0.3, hash.js@^1.1.7: inherits "^2.0.3" minimalistic-assert "^1.0.1" +hasown@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/hasown/-/hasown-2.0.2.tgz#003eaf91be7adc372e84ec59dc37252cedb80003" + integrity sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ== + dependencies: + function-bind "^1.1.2" + he@1.2.0: version "1.2.0" resolved "https://registry.npmjs.org/he/-/he-1.2.0.tgz" @@ -4798,6 +4910,11 @@ math-expression-evaluator@^1.2.14: resolved "https://registry.npmjs.org/math-expression-evaluator/-/math-expression-evaluator-1.2.22.tgz" integrity sha512-L0j0tFVZBQQLeEjmWOvDLoRciIY8gQGWahvkztXUal8jH8R5Rlqo9GCvgqvXcy9LQhEWdQCVvzqAbxgYNt4blQ== +math-intrinsics@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/math-intrinsics/-/math-intrinsics-1.1.0.tgz#a0dd74be81e2aa5c2f27e65ce283605ee4e2b7f9" + integrity sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g== + md5.js@^1.3.4: version "1.3.5" resolved "https://registry.npmjs.org/md5.js/-/md5.js-1.3.5.tgz" @@ -5129,6 +5246,15 @@ nano@^10.1.0: qs "^6.11.0" tough-cookie "^4.1.2" +nano@^10.1.3: + version "10.1.4" + resolved "https://registry.yarnpkg.com/nano/-/nano-10.1.4.tgz#cb4cabd677733ddb81c88c1b8635101e2d84e926" + integrity sha512-bJOFIPLExIbF6mljnfExXX9Cub4W0puhDjVMp+qV40xl/DBvgKao7St4+6/GB6EoHZap7eFnrnx4mnp5KYgwJA== + dependencies: + axios "^1.7.4" + node-abort-controller "^3.1.1" + qs "^6.13.0" + nanoid@3.1.23: version "3.1.23" resolved "https://registry.npmjs.org/nanoid/-/nanoid-3.1.23.tgz" @@ -5164,6 +5290,11 @@ node-abort-controller@^3.0.1: resolved "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.0.1.tgz" integrity sha512-/ujIVxthRs+7q6hsdjHMaj8hRG9NuWmwrz+JdRwZ14jdFoKSkm+vDsCbF9PLpnSqjaWQJuTmVtcWHNLr+vrOFw== +node-abort-controller@^3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/node-abort-controller/-/node-abort-controller-3.1.1.tgz#a94377e964a9a37ac3976d848cb5c765833b8548" + integrity sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ== + node-addon-api@^2.0.0: version "2.0.2" resolved "https://registry.npmjs.org/node-addon-api/-/node-addon-api-2.0.2.tgz" @@ -5179,6 +5310,13 @@ node-addon-api@^4.3.0: resolved "https://registry.npmjs.org/node-addon-api/-/node-addon-api-4.3.0.tgz" integrity sha512-73sE9+3UaLYYFmDsFZnqCInzPyh3MqIwZO9cw58yIqAZhONrrabrYyYe3TuIqtIiOuTXVhsGau8hcrhhwSsDIQ== +node-fetch@^2.6.1: + version "2.7.0" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" + integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== + dependencies: + whatwg-url "^5.0.0" + node-fetch@^2.6.7: version "2.6.7" resolved "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz" @@ -5293,6 +5431,11 @@ object-inspect@^1.12.3: resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.13.0.tgz#42695d3879e1cd5bda6df5062164d80c996e23e2" integrity sha512-HQ4J+ic8hKrgIt3mqk6cVOVrW2ozL4KdvHlqpBv9vDYWx9ysAgENAdvy4FoGF+KFdhR7nQTNm5J0ctAeOwn+3g== +object-inspect@^1.13.3: + version "1.13.3" + resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.13.3.tgz#f14c183de51130243d6d18ae149375ff50ea488a" + integrity sha512-kDCGIbxkDSXE3euJZZXzc6to7fCrKHNI/hSRQnRuQ+BWjFNzZwiFF8fj/6o2t2G9/jTj8PSIYTfCLelLZEeRpA== + object-inspect@^1.7.0: version "1.8.0" resolved "https://registry.npmjs.org/object-inspect/-/object-inspect-1.8.0.tgz" @@ -5875,6 +6018,13 @@ qs@6.11.0, qs@^6.11.0: dependencies: side-channel "^1.0.4" +qs@^6.13.0: + version "6.14.0" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.14.0.tgz#c63fa40680d2c5c941412a0e899c89af60c0a930" + integrity sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w== + dependencies: + side-channel "^1.1.0" + qs@~6.5.2: version "6.5.2" resolved "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz" @@ -6522,6 +6672,35 @@ shell-quote@^1.6.1: resolved "https://registry.npmjs.org/shell-quote/-/shell-quote-1.7.2.tgz" integrity sha512-mRz/m/JVscCrkMyPqHc/bczi3OQHkLTqXHEFu0zDhK/qfv3UcOA4SVmRCLmos4bhjr9ekVQubj/R7waKapmiQg== +side-channel-list@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/side-channel-list/-/side-channel-list-1.0.0.tgz#10cb5984263115d3b7a0e336591e290a830af8ad" + integrity sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA== + dependencies: + es-errors "^1.3.0" + object-inspect "^1.13.3" + +side-channel-map@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/side-channel-map/-/side-channel-map-1.0.1.tgz#d6bb6b37902c6fef5174e5f533fab4c732a26f42" + integrity sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA== + dependencies: + call-bound "^1.0.2" + es-errors "^1.3.0" + get-intrinsic "^1.2.5" + object-inspect "^1.13.3" + +side-channel-weakmap@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz#11dda19d5368e40ce9ec2bdc1fb0ecbc0790ecea" + integrity sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A== + dependencies: + call-bound "^1.0.2" + es-errors "^1.3.0" + get-intrinsic "^1.2.5" + object-inspect "^1.13.3" + side-channel-map "^1.0.1" + side-channel@^1.0.4: version "1.0.4" resolved "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz" @@ -6531,6 +6710,17 @@ side-channel@^1.0.4: get-intrinsic "^1.0.2" object-inspect "^1.9.0" +side-channel@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.1.0.tgz#c3fcff9c4da932784873335ec9765fa94ff66bc9" + integrity sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw== + dependencies: + es-errors "^1.3.0" + object-inspect "^1.13.3" + side-channel-list "^1.0.0" + side-channel-map "^1.0.1" + side-channel-weakmap "^1.0.2" + signal-exit@^3.0.2: version "3.0.3" resolved "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.3.tgz" @@ -7908,6 +8098,11 @@ yargs@16.2.0: y18n "^5.0.5" yargs-parser "^20.2.2" +yavent@^0.1.3: + version "0.1.4" + resolved "https://registry.yarnpkg.com/yavent/-/yavent-0.1.4.tgz#b5ddbd01ff7351a8dee51025609e3bd924090733" + integrity sha512-8N18drD5IWdqwFkvDmYmJdRFj9LCuoTMqKMCBa/mYAnDOZNDrW4itJwHGXQMY7TgeK8SmR75Hg5dds1/hRf9iQ== + yeast@0.1.2: version "0.1.2" resolved "https://registry.npmjs.org/yeast/-/yeast-0.1.2.tgz" From b76bfa07f9d1df8ec7aa45f2c8868721267f9e4c Mon Sep 17 00:00:00 2001 From: Sam Holmes Date: Wed, 4 Jun 2025 16:34:48 -0700 Subject: [PATCH 3/4] Add migrate CLI --- package.json | 2 + src/bin/migrate.ts | 295 +++++++++++++++++++++++++++++++++++++++++++++ src/initDbs.ts | 1 + src/types.ts | 48 +++++++- 4 files changed, 342 insertions(+), 4 deletions(-) create mode 100644 src/bin/migrate.ts diff --git a/package.json b/package.json index 69289d57..d0b9e854 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,8 @@ "start:rates": "node -r sucrase/register src/indexRates.ts", "start:api": "node -r sucrase/register src/indexApi.ts", "start:destroyPartition": "node -r sucrase/register src/bin/destroyPartition.ts", + "start:migrate": "node -r sucrase/register src/bin/migrate.ts", + "start:migrate:inspect": "node --inspect-brk -r sucrase/register src/bin/migrate.ts", "stats": "node -r sucrase/register src/bin/partitionStats.ts", "test": "mocha -r sucrase/register 'test/**/*.test.ts'", "demo": "parcel serve src/demo/index.html", diff --git a/src/bin/migrate.ts b/src/bin/migrate.ts new file mode 100644 index 00000000..70b92c5f --- /dev/null +++ b/src/bin/migrate.ts @@ -0,0 +1,295 @@ +import { + asDate, + asJSON, + asObject, + asOptional, + asString, + uncleaner +} from 'cleaners' +import fs from 'fs' +import nano from 'nano' +import path from 'path' + +import { config } from '../config' +import { processBanxaTx } from '../partners/banxa' +import { processBitaccessTx } from '../partners/bitaccess' +import { processBitrefillTx } from '../partners/bitrefill' +import { processBitsOfGoldTx } from '../partners/bitsofgold' +import { processBityTx } from '../partners/bity' +import { processChangeHeroTx } from '../partners/changehero' +import { processChangellyTx } from '../partners/changelly' +import { processChangeNowTx } from '../partners/changenow' +import { processCoinSwitchTx } from '../partners/coinswitch' +import { processExolixTx } from '../partners/exolix' +import { processFaastTx } from '../partners/faast' +import { processFoxExchangeTx } from '../partners/foxExchange' +import { processGodexTx } from '../partners/godex' +import { processIoniaGiftCardsTx } from '../partners/ioniagiftcard' +import { processIoniaVisaRewardsTx } from '../partners/ioniavisarewards' +import { processKadoTx } from '../partners/kado' +import { processLetsExchangeTx } from '../partners/letsexchange' +import { processLibertyxTx } from '../partners/libertyx' +import { processLifiTx } from '../partners/lifi' +import { processMoonpayTx } from '../partners/moonpay' +import { processPaybisTx } from '../partners/paybis' +import { processPaytrieTx } from '../partners/paytrie' +import { processSafelloTx } from '../partners/safello' +import { processShapeshiftTx } from '../partners/shapeshift' +import { processSideshiftTx } from '../partners/sideshift' +import { processSimplexTx } from '../partners/simplex' +import { processSwapuzTx } from '../partners/swapuz' +import { processSwitchainTx } from '../partners/switchain' +import { processTransakTx } from '../partners/transak' +import { processWyreTx } from '../partners/wyre' +import { processXanpoolTx } from '../partners/xanpool' +import { DbTx, StandardTx, wasDbTx } from '../types' +import { datelog } from '../util' + +const processors: { + [partnerId: string]: undefined | ((rawTx: unknown) => StandardTx) +} = { + banxa: processBanxaTx, + bitaccess: processBitaccessTx, + bitrefill: processBitrefillTx, + bitsofgold: processBitsOfGoldTx, + bity: processBityTx, + changehero: processChangeHeroTx, + changelly: processChangellyTx, + changenow: processChangeNowTx, + coinswitch: processCoinSwitchTx, + exolix: processExolixTx, + faast: processFaastTx, + foxExchange: processFoxExchangeTx, + gebo: undefined, + godex: processGodexTx, + ioniagiftcards: processIoniaGiftCardsTx, + ioniavisarewards: processIoniaVisaRewardsTx, + kado: processKadoTx, + letsexchange: processLetsExchangeTx, + libertyx: processLibertyxTx, + lifi: processLifiTx, + moonpay: processMoonpayTx, + paybis: processPaybisTx, + paytrie: processPaytrieTx, + safello: processSafelloTx, + shapeshift: processShapeshiftTx, + sideshift: processSideshiftTx, + simplex: processSimplexTx, + swapuz: processSwapuzTx, + switchain: processSwitchainTx, + // thorchain: processThorchainTx, + totle: undefined, + transak: processTransakTx, + wyre: processWyreTx, + xanpool: processXanpoolTx +} + +type MigrationState = ReturnType +const asMigrationState = asJSON( + asObject({ + bookmark: asOptional(asString) + }) +) +const wasMigrationState = uncleaner(asMigrationState) + +const MIGRATION_STATE_FILE = './cache/migrationState.json' +const PAGE_SIZE = 1000 + +const nanoDb = nano(config.couchDbFullpath) + +// Ensure migration state file directory exists. +fs.mkdirSync(path.dirname(MIGRATION_STATE_FILE), { recursive: true }) + +migration().catch(e => { + datelog(e) +}) + +async function migration(): Promise { + // Args: + const shouldInitialize = process.argv.includes('--init') + const shouldStartOver = process.argv.includes('--start-over') + + const reportsTransactions = nanoDb.use('reports_transactions') + + // Initialize migration state file: + if (!fs.existsSync(MIGRATION_STATE_FILE)) { + saveMigrationState({ + bookmark: undefined + }) + } + const migrationState = readMigrationState() + + if (shouldStartOver) { + migrationState.bookmark = undefined + saveMigrationState(migrationState) + } + + // Migrate all transactions that do not have a createTime field. + if (shouldInitialize) { + console.log('Initializing documents...') + while (true) { + const response = await reportsTransactions.find({ + selector: { + status: { $eq: 'complete' }, + createTime: { $exists: false } + }, + use_index: 'status-createtime', + limit: PAGE_SIZE + }) + if (response.docs.length === 0) { + break + } + const newDocs = await initializeDocument(response.docs) + console.log( + `Initialized ${newDocs.length} documents after ${response.docs[0]._id}` + ) + await reportsTransactions.bulk({ docs: newDocs }) + } + console.log('Initial migration complete.') + } + + while (true) { + const response = await reportsTransactions.find({ + selector: { + status: { $eq: 'complete' } + }, + sort: [{ status: 'asc' }, { createTime: 'asc' }], + use_index: 'status-createtime', + limit: PAGE_SIZE, + bookmark: migrationState.bookmark + }) + if (response.docs.length === 0) { + break + } + const newDocs = await updateDocument(response.docs) + console.log( + `Updated ${newDocs.length} documents after ${response.docs[0]._id}` + ) + await reportsTransactions.bulk({ docs: newDocs }) + // Update migration state: + migrationState.bookmark = response.bookmark + saveMigrationState(migrationState) + } + console.log('Migration complete.') + + migrationState.bookmark = undefined + saveMigrationState(migrationState) +} + +function readMigrationState(): MigrationState { + const migrationStateFileContent = fs.readFileSync(MIGRATION_STATE_FILE, { + encoding: 'utf8' + }) + return asMigrationState(migrationStateFileContent) +} + +function saveMigrationState(migrationState: MigrationState): void { + fs.writeFileSync(MIGRATION_STATE_FILE, wasMigrationState(migrationState), { + encoding: 'utf8' + }) +} + +function getDocumentIdentifiers( + documentId: string +): { appId: string; partnerId: string } { + const parts = documentId.split(':')[0].split('_') + if (parts.length === 0) { + throw new Error(`Invalid documentId ${documentId}`) + } + const partnerId = parts.pop() as string + const appId = parts.join('_') + return { appId, partnerId } +} + +/** + * Adds the createTime field to documents that are missing it. + * This is a required field for migration processing. + */ +async function initializeDocument( + docs: Array +): Promise { + const newDocs: DbTx[] = [] + for (const doc of docs) { + const { partnerId } = getDocumentIdentifiers(doc._id) + const processor = processors[partnerId] + + if (processor == null) { + // Add createTime to document for minimal requirement for + // migration/processing. + newDocs.push({ ...doc, createTime: new Date() }) + datelog(`Not found ${partnerId} for document ${doc._id}`) + continue + } + + if (doc.rawTx == null) { + // Add createTime to document for minimal requirement for + // migration/processing. + newDocs.push({ ...doc, createTime: new Date() }) + datelog(`Missing rawTx for document ${doc._id}`) + continue + } + + let standardTx + try { + standardTx = processor(doc.rawTx) + } catch (error) { + // Add createTime to document for minimal requirement for + // migration/processing. + newDocs.push({ ...doc, createTime: new Date() }) + datelog(`Error processing ${doc._id}`, error) + continue + } + + newDocs.push( + wasDbTx({ + _id: doc._id, + _rev: doc._rev, + ...standardTx + }) + ) + } + + return newDocs +} + +/** + * Updates the documents with any new fields using its processor function. + */ +async function updateDocument( + docs: Array +): Promise { + const newDocs: DbTx[] = [] + for (const doc of docs) { + const { partnerId } = getDocumentIdentifiers(doc._id) + const processor = processors[partnerId] + + if (processor == null) { + datelog(`Not found ${partnerId} for document ${doc._id}`) + continue + } + + if (doc.rawTx == null) { + datelog(`Missing rawTx for document ${doc._id}`) + continue + } + + let standardTx + try { + standardTx = processor(doc.rawTx) + } catch (error) { + datelog(`Error processing ${doc._id}`, error) + continue + } + + newDocs.push( + wasDbTx({ + _id: doc._id, + _rev: doc._rev, + createTime: asDate(doc.createTime), + ...standardTx + }) + ) + } + + return newDocs +} diff --git a/src/initDbs.ts b/src/initDbs.ts index 797f4793..286a1495 100644 --- a/src/initDbs.ts +++ b/src/initDbs.ts @@ -38,6 +38,7 @@ const transactionIndexes: DesignDocumentMap = { ...fieldsToDesign(['status', 'isoDate']), ...fieldsToDesign(['status', 'payoutAmount', 'depositAmount']), ...fieldsToDesign(['status', 'payoutCurrency', 'isoDate']), + ...fieldsToDesign(['status', 'createTime']), ...fieldsToDesign(['status', 'usdValue']), ...fieldsToDesign(['status', 'usdValue', 'timestamp']), ...fieldsToDesign(['usdValue']), diff --git a/src/types.ts b/src/types.ts index a51b5cd6..55d7bd77 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,5 +1,7 @@ import { asArray, + asCodec, + asDate, asEither, asMap, asNull, @@ -8,7 +10,8 @@ import { asOptional, asString, asUnknown, - asValue + asValue, + uncleaner } from 'cleaners' /** Earliest date that transactions may show in Edge */ @@ -41,6 +44,7 @@ const asStatus = asValue( 'other' ) +type SafeNumber = ReturnType const asSafeNumber = (raw: any): number => { if (isNaN(raw) || raw === null) { return 0 @@ -49,6 +53,7 @@ const asSafeNumber = (raw: any): number => { } /** A null direction is for swap exchange types. */ +type Direction = ReturnType const asDirection = asEither(asValue('buy', 'sell'), asNull) /** @@ -106,9 +111,39 @@ const asFiatPaymentType = asValue( export type FiatPaymentType = ReturnType /** The type of exchange that the partner is. A 'fiat' type means on/off ramp. */ +export type ExchangeType = ReturnType const asExchangeType = asValue('fiat', 'swap') -export const asStandardTx = asObject({ +export interface StandardTx { + orderId: string + countryCode: string | null + depositTxid?: string + depositAddress?: string + depositCurrency: string + depositAmount: SafeNumber + direction: Direction + exchangeType: ExchangeType + paymentType: FiatPaymentType | null + payoutTxid?: string + payoutAddress?: string + payoutCurrency: string + payoutAmount: SafeNumber + status: Status + usdValue: number + + /** When the transaction occurred (ISO date). */ + isoDate: string + /** When the transaction occurred (unix timestamp). */ + timestamp: number + + /** When the document was created. */ + createTime?: Date + + /** The raw transaction data from the partner API. */ + rawTx?: unknown +} + +export const asStandardTx = asObject({ orderId: asString, countryCode: asEither(asString, asNull), depositTxid: asOptional(asString), @@ -123,9 +158,14 @@ export const asStandardTx = asObject({ payoutCurrency: asString, payoutAmount: asSafeNumber, status: asStatus, + usdValue: asNumber, isoDate: asString, timestamp: asNumber, - usdValue: asNumber, + createTime: asCodec( + asOptional(asDate), + // Default to now + uncleaner(asOptional(asDate, (): Date | undefined => new Date())) + ), rawTx: asUnknown }) @@ -134,6 +174,7 @@ export const asDbTx = asObject({ _id: asOptional(asString), _rev: asOptional(asString) }) +export const wasDbTx = uncleaner(asDbTx) export const asProgressSettings = asObject({ _id: asOptional(asString), @@ -209,6 +250,5 @@ export type AnalyticsResult = ReturnType export type CurrencyCodeMappings = ReturnType export type DbCurrencyCodeMappings = ReturnType export type DbTx = ReturnType -export type StandardTx = ReturnType export type PluginParams = ReturnType export type Status = ReturnType From c1bb57a45bbf76a7fb9ddddb38ac6fb966bc752c Mon Sep 17 00:00:00 2001 From: Sam Holmes Date: Wed, 4 Jun 2025 16:29:35 -0700 Subject: [PATCH 4/4] Add clickhouseEngine --- .vscode/launch.json | 70 ++++------ .vscode/settings.json | 1 + package.json | 5 +- src/clickhouseEngine.ts | 231 +++++++++++++++++++++++++++++++ src/config.ts | 11 ++ src/indexClickhouse.ts | 3 + src/initDbs.ts | 1 + src/partners/banxa.ts | 1 + src/partners/bitaccess.ts | 1 + src/partners/bitrefill.ts | 1 + src/partners/bitsofgold.ts | 1 + src/partners/bity.ts | 1 + src/partners/changehero.ts | 1 + src/partners/changelly.ts | 1 + src/partners/changenow.ts | 1 + src/partners/coinswitch.ts | 1 + src/partners/exolix.ts | 1 + src/partners/faast.ts | 1 + src/partners/foxExchange.ts | 10 +- src/partners/godex.ts | 1 + src/partners/ioniagiftcard.ts | 1 + src/partners/ioniavisarewards.ts | 1 + src/partners/kado.ts | 3 +- src/partners/letsexchange.ts | 1 + src/partners/libertyx.ts | 3 +- src/partners/lifi.ts | 1 + src/partners/moonpay.ts | 2 + src/partners/paybis.ts | 1 + src/partners/paytrie.ts | 1 + src/partners/safello.ts | 1 + src/partners/shapeshift.ts | 1 + src/partners/sideshift.ts | 1 + src/partners/simplex.ts | 2 +- src/partners/swapuz.ts | 2 +- src/partners/switchain.ts | 1 + src/partners/thorchain.ts | 1 + src/partners/totle.ts | 1 + src/partners/transak.ts | 1 + src/partners/wyre.ts | 1 + src/partners/xanpool.ts | 2 + src/queryEngine.ts | 15 +- src/types.ts | 3 + yarn.lock | 26 +++- 43 files changed, 358 insertions(+), 58 deletions(-) create mode 100644 src/clickhouseEngine.ts create mode 100644 src/indexClickhouse.ts diff --git a/.vscode/launch.json b/.vscode/launch.json index f1e7434c..08bfc632 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -4,6 +4,15 @@ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ + { + "type": "node", + "request": "attach", + "name": "Attach to Node.js", + "address": "127.0.0.1", + "port": 9229, + "restart": true, + "timeout": 30000 + }, { "type": "node", "request": "launch", @@ -11,67 +20,51 @@ "args": [ "-r", "sucrase/register", - "${workspaceFolder}/src/indexQuery.ts", - + "${workspaceFolder}/src/indexQuery.ts" ], "internalConsoleOptions": "openOnSessionStart", - "skipFiles": [ - "/**" - ] + "skipFiles": ["/**"] }, { "type": "node", "request": "launch", "name": "indexRates", - "args":[ + "args": [ "-r", "sucrase/register", - "${workspaceFolder}/src/indexRates.ts", + "${workspaceFolder}/src/indexRates.ts" ], - "skipFiles": [ - "/**" - ] + "skipFiles": ["/**"] }, { "type": "node", "request": "launch", "name": "indexApi", - "args": [ - "-r", - "sucrase/register", - "${workspaceFolder}/src/indexApi.ts", - - ], + "args": ["-r", "sucrase/register", "${workspaceFolder}/src/indexApi.ts"], "internalConsoleOptions": "openOnSessionStart", - "skipFiles": [ - "/**" - ] + "skipFiles": ["/**"] }, { "type": "node", "request": "launch", "name": "indexCache", - "args":[ + "args": [ "-r", "sucrase/register", - "${workspaceFolder}/src/indexCache.ts", + "${workspaceFolder}/src/indexCache.ts" ], - "skipFiles": [ - "/**" - ] + "skipFiles": ["/**"] }, { "type": "node", "request": "launch", "name": "lifiReporter", - "args":[ + "args": [ "-r", "sucrase/register", - "${workspaceFolder}/src/bin/lifiReporter.ts", + "${workspaceFolder}/src/bin/lifiReporter.ts" ], - "skipFiles": [ - "/**" - ] + "skipFiles": ["/**"] }, { "type": "node", @@ -80,13 +73,10 @@ "args": [ "-r", "sucrase/register", - "${workspaceFolder}/src/bin/testpartner.ts", - + "${workspaceFolder}/src/bin/testpartner.ts" ], "internalConsoleOptions": "openOnSessionStart", - "skipFiles": [ - "/**" - ] + "skipFiles": ["/**"] }, { "type": "node", @@ -95,13 +85,10 @@ "args": [ "-r", "sucrase/register", - "${workspaceFolder}/src/bin/partnerTotals.ts", - + "${workspaceFolder}/src/bin/partnerTotals.ts" ], "internalConsoleOptions": "openOnSessionStart", - "skipFiles": [ - "/**" - ] + "skipFiles": ["/**"] }, { "type": "node", @@ -115,10 +102,7 @@ "edge_letsexchange" ], "internalConsoleOptions": "openOnSessionStart", - "skipFiles": [ - "/**" - ] + "skipFiles": ["/**"] } - ] } diff --git a/.vscode/settings.json b/.vscode/settings.json index 5d28cf45..91ee12f7 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -4,6 +4,7 @@ "Bitrefill", "Bity", "Changelly", + "Clickhouse", "Faast", "godex", "Kado", diff --git a/package.json b/package.json index d0b9e854..bcfe0d85 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,8 @@ "prepare": "./scripts/prepare.sh && npm-run-all clean configure -p build.*", "start": "node -r sucrase/register src/indexQuery.ts", "start:cache": "node -r sucrase/register src/indexCache.ts", + "start:clickhouse": "node -r sucrase/register src/indexClickhouse.ts", + "start:clickhouse:inspect": "node --inspect-brk -r sucrase/register src/indexClickhouse.ts", "start:rates": "node -r sucrase/register src/indexRates.ts", "start:api": "node -r sucrase/register src/indexApi.ts", "start:destroyPartition": "node -r sucrase/register src/bin/destroyPartition.ts", @@ -39,7 +41,8 @@ "*.{js,jsx,ts,tsx}": "eslint" }, "dependencies": { - "@types/node": "^14.0.22", + "@clickhouse/client": "^1.10.1", + "@types/node": "^20.17.17", "api-changelly": "git://github.com/changelly/api-changelly.git#8e350f3", "axios": "^0.21.2", "biggystring": "^4.1.3", diff --git a/src/clickhouseEngine.ts b/src/clickhouseEngine.ts new file mode 100644 index 00000000..29517248 --- /dev/null +++ b/src/clickhouseEngine.ts @@ -0,0 +1,231 @@ +import { createClient } from '@clickhouse/client' +import { asDate, asObject, asOptional, asString, uncleaner } from 'cleaners' +import nano from 'nano' + +import { config } from './config' +import { initDbs } from './initDbs' +import { asStandardTx, DbTx, StandardTx, wasDbTx } from './types' +import { datelog, snooze } from './util' + +// Clickhouse recommends large batch inserts. We consider the couchdb +// query size as well. +const PAGE_SIZE = 10_000 + +const nanoDb = nano(config.couchDbFullpath) +const clickhouseDb = createClient(config.clickhouseConnection) + +const progressDocName = 'clickhouse:clickhouseEngine' +const asClickhouseProgress = asObject({ + _id: asOptional(asString, progressDocName), + _rev: asOptional(asString), + afterTime: asOptional(asDate, new Date(0)) +}) +const wasClickhouseProgress = uncleaner(asClickhouseProgress) + +export async function clickhouseEngine(): Promise { + await initDbs() + await initClickhouseDb() + + const dbTransactions = nanoDb.db.use('reports_transactions') + + const dbProgress = nanoDb.db.use('reports_progresscache') + const out = await dbProgress.get(progressDocName).catch(error => { + if (error.error != null && error.error === 'not_found') { + datelog(`Previous Progress Record Not Found ${progressDocName}`) + return {} + } + throw error + }) + const progressDoc = asClickhouseProgress(out) + + let bookmark: string | undefined + + while (true) { + const response = await dbTransactions.find({ + selector: { + status: { $eq: 'complete' }, + updateTime: { $gt: progressDoc.afterTime.toISOString() } + }, + sort: [{ updateTime: 'asc' }], + use_index: 'status-updatetime', + limit: PAGE_SIZE, + bookmark + }) + + bookmark = response.bookmark + + const startDocId = response.docs[0]?._id + const endDocId = response.docs[response.docs.length - 1]?._id + + if (response.docs.length > 0) { + datelog( + `Processing ${response.docs.length} rows from ${startDocId} to ${endDocId}.` + ) + } else { + datelog( + `Queried for new documents after ${progressDoc.afterTime.toISOString()}.` + ) + } + + const newDocs: DbTx[] = [] + const newRows: any[][] = [] + let lastDocUpdateTime: string | undefined + + for (const doc of response.docs) { + const { appId, partnerId } = getDocumentIdentifiers(doc._id) + + const standardTx = asStandardTx(doc) + + newRows.push([ + appId, + partnerId, + standardTx.orderId, + standardTx.countryCode, + standardTx.depositTxid, + standardTx.depositAddress, + standardTx.depositCurrency, + standardTx.depositAmount, + standardTx.direction, + standardTx.exchangeType, + standardTx.paymentType, + standardTx.payoutTxid, + standardTx.payoutAddress, + standardTx.payoutCurrency, + standardTx.payoutAmount, + standardTx.status, + Math.round(standardTx.timestamp), + standardTx.usdValue, + config.clickhouseIndexVersion + ]) + + newDocs.push( + wasDbTx({ + _id: doc._id, + _rev: doc._rev, + ...standardTx + }) + ) + + const txUpdateTime = standardTx.updateTime.toISOString() + if (lastDocUpdateTime == null || lastDocUpdateTime < txUpdateTime) { + lastDocUpdateTime = txUpdateTime + } + } + + // Add the standardTx to the clickhouse database + await clickhouseDb.insert({ + table: 'reports_transactions', + columns: [ + 'appId', + 'partnerId', + 'orderId', + 'countryCode', + 'depositTxid', + 'depositAddress', + 'depositCurrency', + 'depositAmount', + 'direction', + 'exchangeType', + 'paymentType', + 'payoutTxid', + 'payoutAddress', + 'payoutCurrency', + 'payoutAmount', + 'status', + 'timestamp', + 'usdValue', + 'indexVersion' + ], + values: newRows + }) + // Update all documents processed + await dbTransactions.bulk({ docs: newDocs }) + + // We've reached the end of the view index, so we'll continue but with a + // delay so as not to thrash the couchdb unnecessarily. + if (response.docs.length !== PAGE_SIZE) { + bookmark = undefined + if (lastDocUpdateTime != null) { + progressDoc.afterTime = new Date(lastDocUpdateTime) + await dbProgress.insert(wasClickhouseProgress(progressDoc)) + } + await snooze(5000) + } + } +} + +function getDocumentIdentifiers( + documentId: string +): { appId: string; partnerId: string } { + const parts = documentId.split(':')[0].split('_') + if (parts.length === 0) { + throw new Error(`Invalid documentId ${documentId}`) + } + const partnerId = parts.pop() as string + const appId = parts.join('_') + return { appId, partnerId } +} + +const REPORTS_TRANSACTIONS_SCHEMA = `\ +CREATE TABLE default.reports_transactions +( + \`partnerId\` String, + \`appId\` String, + \`orderId\` String, + \`countryCode\` String, + \`depositTxid\` String, + \`depositAddress\` String, + \`depositCurrency\` String, + \`depositAmount\` Float64, + \`direction\` String, + \`exchangeType\` String, + \`paymentType\` String, + \`payoutTxid\` String, + \`payoutAddress\` String, + \`payoutCurrency\` String, + \`payoutAmount\` Float64, + \`status\` String, + \`timestamp\` DateTime DEFAULT now(), + \`usdValue\` Float64, + \`indexVersion\` UInt16 +) +ENGINE = ReplacingMergeTree +PRIMARY KEY (appId, partnerId, orderId) +ORDER BY (appId, partnerId, orderId, timestamp) +SETTINGS index_granularity = 8192` + +async function initClickhouseDb(): Promise { + // Check if the table exists + const tableExists = await clickhouseDb.query({ + query: ` + SELECT 1 + FROM system.tables + WHERE database = 'default' AND name = 'reports_transactions' + LIMIT 1; + `, + format: 'JSONEachRow' + }) + + const result = await tableExists.json() + if (result.length > 0) { + const response = await clickhouseDb.query({ + query: 'SHOW CREATE reports_transactions' + }) + const result = await response.json() + const tableSchema = (result.data[0] as any).statement + + if (tableSchema !== REPORTS_TRANSACTIONS_SCHEMA) { + console.log(tableSchema) + throw new Error('Table "reports_transactions" schema does not match.') + } + + datelog('Table "reports_transactions" exists.') + return + } + + // Create the table + await clickhouseDb.command({ + query: REPORTS_TRANSACTIONS_SCHEMA + }) + datelog('Table "reports_transactions" has been created.') +} diff --git a/src/config.ts b/src/config.ts index 92b904f4..8f914fa8 100644 --- a/src/config.ts +++ b/src/config.ts @@ -6,6 +6,17 @@ export const asConfig = asObject({ asString, 'http://username:password@localhost:5984' ), + clickhouseConnection: asOptional( + asObject({ + url: asString, + password: asString + }), + { + url: 'http://localhost:8123', + password: '' + } + ), + clickhouseIndexVersion: asOptional(asNumber, 1), httpPort: asOptional(asNumber, 8008), bog: asOptional(asObject({ apiKey: asString }), { apiKey: '' }), diff --git a/src/indexClickhouse.ts b/src/indexClickhouse.ts new file mode 100644 index 00000000..53dddbcf --- /dev/null +++ b/src/indexClickhouse.ts @@ -0,0 +1,3 @@ +import { clickhouseEngine } from './clickhouseEngine' + +clickhouseEngine().catch(e => console.log(e)) diff --git a/src/initDbs.ts b/src/initDbs.ts index 286a1495..6f748aa1 100644 --- a/src/initDbs.ts +++ b/src/initDbs.ts @@ -39,6 +39,7 @@ const transactionIndexes: DesignDocumentMap = { ...fieldsToDesign(['status', 'payoutAmount', 'depositAmount']), ...fieldsToDesign(['status', 'payoutCurrency', 'isoDate']), ...fieldsToDesign(['status', 'createTime']), + ...fieldsToDesign(['status', 'updateTime']), ...fieldsToDesign(['status', 'usdValue']), ...fieldsToDesign(['status', 'usdValue', 'timestamp']), ...fieldsToDesign(['usdValue']), diff --git a/src/partners/banxa.ts b/src/partners/banxa.ts index 29012417..2659212b 100644 --- a/src/partners/banxa.ts +++ b/src/partners/banxa.ts @@ -285,6 +285,7 @@ export function processBanxaTx(rawTx: unknown): StandardTx { payoutCurrency: outputCurrency, payoutAmount: outputAmount, timestamp, + updateTime: new Date(), isoDate, usdValue: -1, rawTx diff --git a/src/partners/bitaccess.ts b/src/partners/bitaccess.ts index 65a61593..fc132cd3 100644 --- a/src/partners/bitaccess.ts +++ b/src/partners/bitaccess.ts @@ -153,6 +153,7 @@ export function processBitaccessTx(rawTx: unknown): StandardTx { payoutCurrency: tx.withdrawal_currency.toUpperCase(), payoutAmount: tx.withdrawal_amount, timestamp, + updateTime: new Date(), isoDate: tx.updated_at, usdValue: -1, rawTx diff --git a/src/partners/bitrefill.ts b/src/partners/bitrefill.ts index 850dbd28..4ef2aff9 100644 --- a/src/partners/bitrefill.ts +++ b/src/partners/bitrefill.ts @@ -160,6 +160,7 @@ export function processBitrefillTx(rawTx: unknown): StandardTx { payoutCurrency: tx.currency, payoutAmount: parseInt(tx.value), timestamp, + updateTime: new Date(), isoDate: new Date(tx.invoiceTime).toISOString(), usdValue: tx.usdPrice, rawTx diff --git a/src/partners/bitsofgold.ts b/src/partners/bitsofgold.ts index 6cf6b7b4..5a49ceec 100644 --- a/src/partners/bitsofgold.ts +++ b/src/partners/bitsofgold.ts @@ -136,6 +136,7 @@ export function processBitsOfGoldTx(rawTx: unknown): StandardTx { payoutCurrency, payoutAmount, timestamp: timestamp / 1000, + updateTime: new Date(), isoDate: date.toISOString(), usdValue: -1, rawTx diff --git a/src/partners/bity.ts b/src/partners/bity.ts index 7fc4612d..dab8ad55 100644 --- a/src/partners/bity.ts +++ b/src/partners/bity.ts @@ -186,6 +186,7 @@ export function processBityTx(rawTx: unknown): StandardTx { payoutCurrency: tx.output.currency.toUpperCase(), payoutAmount: safeParseFloat(tx.output.amount), timestamp: Date.parse(tx.timestamp_created.concat('Z')) / 1000, + updateTime: new Date(), isoDate: new Date(tx.timestamp_created.concat('Z')).toISOString(), usdValue: -1, rawTx diff --git a/src/partners/changehero.ts b/src/partners/changehero.ts index 7ca5ccd9..2021631a 100644 --- a/src/partners/changehero.ts +++ b/src/partners/changehero.ts @@ -177,6 +177,7 @@ export function processChangeHeroTx(rawTx: unknown): StandardTx { payoutCurrency: tx.currencyTo.toUpperCase(), payoutAmount: safeParseFloat(tx.amountTo), timestamp: tx.createdAt, + updateTime: new Date(), isoDate: smartIsoDateFromTimestamp(tx.createdAt).isoDate, usdValue: -1, rawTx diff --git a/src/partners/changelly.ts b/src/partners/changelly.ts index a961ae2f..50e0a4b9 100644 --- a/src/partners/changelly.ts +++ b/src/partners/changelly.ts @@ -189,6 +189,7 @@ export function processChangellyTx(rawTx: unknown): StandardTx { payoutCurrency: tx.currencyTo.toUpperCase(), payoutAmount: safeParseFloat(tx.amountTo), timestamp: tx.createdAt, + updateTime: new Date(), isoDate: new Date(tx.createdAt * 1000).toISOString(), usdValue: -1, rawTx diff --git a/src/partners/changenow.ts b/src/partners/changenow.ts index 29d22107..91289db2 100644 --- a/src/partners/changenow.ts +++ b/src/partners/changenow.ts @@ -160,6 +160,7 @@ export function processChangeNowTx(rawTx: unknown): StandardTx { payoutCurrency: tx.payout.currency.toUpperCase(), payoutAmount: tx.payout.amount ?? tx.payout.expectedAmount ?? 0, timestamp, + updateTime: new Date(), isoDate: date.toISOString(), usdValue: -1, rawTx diff --git a/src/partners/coinswitch.ts b/src/partners/coinswitch.ts index 524d7d25..03c802b6 100644 --- a/src/partners/coinswitch.ts +++ b/src/partners/coinswitch.ts @@ -125,6 +125,7 @@ export function processCoinSwitchTx(rawTx: unknown): StandardTx { payoutCurrency: tx.destinationCoin.toUpperCase(), payoutAmount: tx.destinationCoinAmount, timestamp: tx.createdAt / 1000, + updateTime: new Date(), isoDate: new Date(tx.createdAt).toISOString(), usdValue: -1, rawTx: rawTx diff --git a/src/partners/exolix.ts b/src/partners/exolix.ts index 782ddef3..80a02a1d 100644 --- a/src/partners/exolix.ts +++ b/src/partners/exolix.ts @@ -184,6 +184,7 @@ export function processExolixTx(rawTx: unknown): StandardTx { payoutCurrency: tx.coinTo.coinCode, payoutAmount: tx.amountTo, timestamp, + updateTime: new Date(), isoDate, usdValue: -1, rawTx diff --git a/src/partners/faast.ts b/src/partners/faast.ts index d3badcc6..917efc84 100644 --- a/src/partners/faast.ts +++ b/src/partners/faast.ts @@ -125,6 +125,7 @@ export function processFaastTx(rawTx: unknown): StandardTx { payoutCurrency: tx.withdrawal_currency.toUpperCase(), payoutAmount: tx.amount_withdrawn, timestamp, + updateTime: new Date(), isoDate: tx.created_at, usdValue: -1, rawTx diff --git a/src/partners/foxExchange.ts b/src/partners/foxExchange.ts index 98bb453a..d04043e7 100644 --- a/src/partners/foxExchange.ts +++ b/src/partners/foxExchange.ts @@ -12,6 +12,7 @@ const asFoxExchangeTx = asObject({ destinationCoin: asString, destinationCoinAmount: asNumber, destinationAddress: asObject({ address: asString }), + outputTransactionHash: asString, createdAt: asNumber }) @@ -118,13 +119,7 @@ export const foxExchange: PartnerPlugin = { } export function processFoxExchangeTx(rawTx: unknown): StandardTx { - let tx - try { - tx = asFoxExchangeTx(rawTx) - } catch (e) { - datelog(e) - throw e - } + const tx = asFoxExchangeTx(rawTx) const standardTx: StandardTx = { status: 'complete', orderId: tx.orderId, @@ -141,6 +136,7 @@ export function processFoxExchangeTx(rawTx: unknown): StandardTx { payoutCurrency: tx.destinationCoin.toUpperCase(), payoutAmount: tx.destinationCoinAmount, timestamp: tx.createdAt / 1000, + updateTime: new Date(), isoDate: new Date(tx.createdAt).toISOString(), usdValue: -1, rawTx diff --git a/src/partners/godex.ts b/src/partners/godex.ts index c4ed225c..ff1126bf 100644 --- a/src/partners/godex.ts +++ b/src/partners/godex.ts @@ -168,6 +168,7 @@ export function processGodexTx(rawTx: unknown): StandardTx { payoutCurrency: tx.coin_to.toUpperCase(), payoutAmount: safeParseFloat(tx.withdrawal_amount), timestamp, + updateTime: new Date(), isoDate, usdValue: -1, rawTx: rawTx diff --git a/src/partners/ioniagiftcard.ts b/src/partners/ioniagiftcard.ts index 5304cab5..86a0de7a 100644 --- a/src/partners/ioniagiftcard.ts +++ b/src/partners/ioniagiftcard.ts @@ -154,6 +154,7 @@ export function processIoniaGiftCardsTx(rawTx: unknown): StandardTx { payoutCurrency: 'USD', payoutAmount: tx.GiftCardFaceValue, timestamp, + updateTime: new Date(), isoDate, usdValue: tx.GiftCardFaceValue, rawTx diff --git a/src/partners/ioniavisarewards.ts b/src/partners/ioniavisarewards.ts index ef222165..3a01431f 100644 --- a/src/partners/ioniavisarewards.ts +++ b/src/partners/ioniavisarewards.ts @@ -154,6 +154,7 @@ export function processIoniaVisaRewardsTx(rawTx: unknown): StandardTx { payoutCurrency: 'USD', payoutAmount: tx.GiftCardFaceValue, timestamp, + updateTime: new Date(), isoDate, usdValue: tx.GiftCardFaceValue, rawTx diff --git a/src/partners/kado.ts b/src/partners/kado.ts index ffe30ae4..6cd5f5ee 100644 --- a/src/partners/kado.ts +++ b/src/partners/kado.ts @@ -3,7 +3,6 @@ import { asBoolean, asDate, asEither, - asNull, asNumber, asObject, asString, @@ -147,6 +146,7 @@ export function processKadoTx(rawTx: unknown): StandardTx { payoutCurrency: tx.cryptoCurrency, payoutAmount: tx.receiveUnitCount, timestamp, + updateTime: new Date(), isoDate, usdValue: tx.paidAmountUsd, rawTx @@ -168,6 +168,7 @@ export function processKadoTx(rawTx: unknown): StandardTx { payoutCurrency: 'USD', payoutAmount: tx.receiveUsd, timestamp, + updateTime: new Date(), isoDate, usdValue: tx.receiveUsd, rawTx diff --git a/src/partners/letsexchange.ts b/src/partners/letsexchange.ts index fce878c6..ab7d3e0c 100644 --- a/src/partners/letsexchange.ts +++ b/src/partners/letsexchange.ts @@ -178,6 +178,7 @@ export function processLetsExchangeTx(rawTx: unknown): StandardTx { payoutCurrency: tx.coin_to.toUpperCase(), payoutAmount: safeParseFloat(tx.withdrawal_amount), timestamp, + updateTime: new Date(), isoDate, usdValue: -1, rawTx diff --git a/src/partners/libertyx.ts b/src/partners/libertyx.ts index 97b55a16..3f495f64 100644 --- a/src/partners/libertyx.ts +++ b/src/partners/libertyx.ts @@ -98,7 +98,8 @@ export function processLibertyxTx(rawTx: unknown): StandardTx { payoutAddress: undefined, payoutCurrency: 'BTC', payoutAmount: 0, - timestamp: timestamp, + timestamp, + updateTime: new Date(), isoDate: date.toISOString(), usdValue: tx.all_transactions_usd_sum, rawTx diff --git a/src/partners/lifi.ts b/src/partners/lifi.ts index fd3ab40d..004989fa 100644 --- a/src/partners/lifi.ts +++ b/src/partners/lifi.ts @@ -192,6 +192,7 @@ export function processLifiTx(rawTx: unknown): StandardTx { payoutCurrency: payoutToken.symbol, payoutAmount, timestamp, + updateTime: new Date(), isoDate, usdValue: Number(tx.receiving.amountUSD ?? tx.sending.amountUSD ?? '-1'), rawTx diff --git a/src/partners/moonpay.ts b/src/partners/moonpay.ts index 854ea085..0735b0d2 100644 --- a/src/partners/moonpay.ts +++ b/src/partners/moonpay.ts @@ -222,6 +222,7 @@ export function processMoonpayTx(rawTx: unknown): StandardTx { payoutCurrency: tx.currency.code.toUpperCase(), payoutAmount: tx.quoteCurrencyAmount, timestamp: timestamp / 1000, + updateTime: new Date(), isoDate, usdValue: -1, rawTx @@ -250,6 +251,7 @@ export function processMoonpaySellTx(rawTx: unknown): StandardTx { payoutCurrency: tx.quoteCurrency.code.toUpperCase(), payoutAmount: tx.quoteCurrencyAmount, timestamp: timestamp / 1000, + updateTime: new Date(), isoDate, usdValue: -1, rawTx: rawTx diff --git a/src/partners/paybis.ts b/src/partners/paybis.ts index 96dcac7c..918b5608 100644 --- a/src/partners/paybis.ts +++ b/src/partners/paybis.ts @@ -292,6 +292,7 @@ export function processPaybisTx(rawTx: unknown): StandardTx { payoutCurrency: receivedOriginal.currency, payoutAmount, timestamp, + updateTime: new Date(), isoDate, usdValue: -1, rawTx diff --git a/src/partners/paytrie.ts b/src/partners/paytrie.ts index 2cb16416..57d54c66 100644 --- a/src/partners/paytrie.ts +++ b/src/partners/paytrie.ts @@ -95,6 +95,7 @@ export function processPaytrieTx(rawTx: unknown): StandardTx { payoutCurrency: order.outputCurrency, payoutAmount: order.outputAmount, timestamp: new Date(order.timestamp).getTime() / 1000, + updateTime: new Date(), isoDate: order.timestamp, usdValue: -1, rawTx diff --git a/src/partners/safello.ts b/src/partners/safello.ts index 02924bbd..4cdc08c0 100644 --- a/src/partners/safello.ts +++ b/src/partners/safello.ts @@ -100,6 +100,7 @@ export function processSafelloTx(rawTx: unknown): StandardTx { payoutCurrency: tx.cryptoCurrency, payoutAmount: 0, timestamp: timestamp / 1000, + updateTime: new Date(), isoDate: date.toISOString(), usdValue: -1, rawTx diff --git a/src/partners/shapeshift.ts b/src/partners/shapeshift.ts index af05144b..31efeb1e 100644 --- a/src/partners/shapeshift.ts +++ b/src/partners/shapeshift.ts @@ -101,6 +101,7 @@ export function processShapeshiftTx(rawTx: unknown): StandardTx { payoutCurrency: tx.outputCurrency, payoutAmount: safeParseFloat(tx.outputAmount), timestamp: tx.timestamp, + updateTime: new Date(), isoDate: new Date(tx.timestamp * 1000).toISOString(), usdValue: -1, rawTx diff --git a/src/partners/sideshift.ts b/src/partners/sideshift.ts index e6b07efe..c6f56ca0 100644 --- a/src/partners/sideshift.ts +++ b/src/partners/sideshift.ts @@ -192,6 +192,7 @@ export function processSideshiftTx(rawTx: unknown): StandardTx { payoutCurrency: tx.settleAsset, payoutAmount: Number(tx.settleAmount), timestamp, + updateTime: new Date(), isoDate, usdValue: -1, rawTx diff --git a/src/partners/simplex.ts b/src/partners/simplex.ts index ebc8de61..5199a700 100644 --- a/src/partners/simplex.ts +++ b/src/partners/simplex.ts @@ -12,7 +12,6 @@ import { import { PartnerPlugin, PluginParams, PluginResult, StandardTx } from '../types' import { safeParseFloat } from '../util' -import { isFiatCurrency } from '../util/fiatCurrency' const asSimplexTx = asObject({ amount_usd: asString, @@ -159,6 +158,7 @@ export function processSimplexTx(rawTx: unknown): StandardTx { payoutCurrency: tx.crypto_currency, payoutAmount: safeParseFloat(tx.amount_crypto), timestamp: tx.created_at, + updateTime: new Date(), isoDate: new Date(tx.created_at * 1000).toISOString(), usdValue: safeParseFloat(tx.amount_usd), rawTx diff --git a/src/partners/swapuz.ts b/src/partners/swapuz.ts index e187c993..0593cbe4 100644 --- a/src/partners/swapuz.ts +++ b/src/partners/swapuz.ts @@ -16,7 +16,6 @@ import { Status } from '../types' import { datelog, retryFetch, smartIsoDateFromTimestamp } from '../util' -import { isFiatCurrency } from '../util/fiatCurrency' const asSwapuzLogin = asObject({ result: asObject({ @@ -186,6 +185,7 @@ export function processSwapuzTx(rawTx: unknown): StandardTx { payoutAddress: undefined, payoutAmount: tx.amountResult, timestamp, + updateTime: new Date(), isoDate, usdValue: -1, rawTx diff --git a/src/partners/switchain.ts b/src/partners/switchain.ts index 7118aaab..be1ecb56 100644 --- a/src/partners/switchain.ts +++ b/src/partners/switchain.ts @@ -128,6 +128,7 @@ export function processSwitchainTx(rawTx: unknown): StandardTx { payoutCurrency: pair[1].toUpperCase(), payoutAmount: safeParseFloat(tx.rate), timestamp: timestamp / 1000, + updateTime: new Date(), isoDate: tx.createdAt, usdValue: -1, rawTx diff --git a/src/partners/thorchain.ts b/src/partners/thorchain.ts index cb189dd9..b60df369 100644 --- a/src/partners/thorchain.ts +++ b/src/partners/thorchain.ts @@ -329,6 +329,7 @@ export function processThorchainTx( payoutCurrency, payoutAmount, timestamp, + updateTime: new Date(), isoDate, usdValue: -1, rawTx diff --git a/src/partners/totle.ts b/src/partners/totle.ts index fc2815b6..f04fabf7 100644 --- a/src/partners/totle.ts +++ b/src/partners/totle.ts @@ -417,6 +417,7 @@ export async function queryTotle( ) ), timestamp: timestamp, + updateTime: new Date(), isoDate: new Date(timestamp * 1000).toISOString(), usdValue: -1, rawTx: rawSwapEvent diff --git a/src/partners/transak.ts b/src/partners/transak.ts index efdb31eb..000e0a1b 100644 --- a/src/partners/transak.ts +++ b/src/partners/transak.ts @@ -138,6 +138,7 @@ export function processTransakTx(rawTx: unknown): StandardTx { payoutCurrency: tx.cryptoCurrency, payoutAmount: tx.cryptoAmount, timestamp: date.getTime() / 1000, + updateTime: new Date(), isoDate: date.toISOString(), usdValue: -1, rawTx diff --git a/src/partners/wyre.ts b/src/partners/wyre.ts index 2459b9fd..a6266c7f 100644 --- a/src/partners/wyre.ts +++ b/src/partners/wyre.ts @@ -130,6 +130,7 @@ export function processWyreTx(rawTx: unknown): StandardTx { payoutCurrency, payoutAmount: safeParseFloat(tx.destAmount), timestamp: dateMs / 1000, + updateTime: new Date(), isoDate: date.toISOString(), usdValue: safeParseFloat(tx.usdEquiv), rawTx diff --git a/src/partners/xanpool.ts b/src/partners/xanpool.ts index 4ca1288c..1d13b314 100644 --- a/src/partners/xanpool.ts +++ b/src/partners/xanpool.ts @@ -140,6 +140,7 @@ export function processXanpoolTx(rawTx: unknown): StandardTx { payoutAmount: tx.crypto, timestamp: smartIsoDateFromTimestamp(new Date(tx.createdAt).getTime()) .timestamp, + updateTime: new Date(), isoDate: tx.createdAt, usdValue: -1, rawTx @@ -162,6 +163,7 @@ export function processXanpoolTx(rawTx: unknown): StandardTx { payoutAmount: tx.fiat, timestamp: smartIsoDateFromTimestamp(new Date(tx.createdAt).getTime()) .timestamp, + updateTime: new Date(), isoDate: tx.createdAt, usdValue: -1, rawTx diff --git a/src/queryEngine.ts b/src/queryEngine.ts index 48fe930f..b9d5860b 100644 --- a/src/queryEngine.ts +++ b/src/queryEngine.ts @@ -33,7 +33,14 @@ import { maya, thorchain } from './partners/thorchain' import { transak } from './partners/transak' import { wyre } from './partners/wyre' import { xanpool } from './partners/xanpool' -import { asApp, asApps, asProgressSettings, DbTx, StandardTx } from './types' +import { + asApp, + asApps, + asProgressSettings, + DbTx, + StandardTx, + wasDbTx +} from './types' import { datelog, promiseTimeout, standardizeNames } from './util' const nanoDb = nano(config.couchDbFullpath) @@ -183,7 +190,11 @@ const filterAddNewTxs = async ( if (tx.status !== queryResult.doc?.status) { const oldStatus = queryResult.doc?.status const newStatus = tx.status - const newObj = { _id: docId, _rev: queryResult.doc?._rev, ...tx } + const newObj = wasDbTx({ + _id: docId, + _rev: queryResult.doc?._rev, + ...tx + }) newDocs.push(newObj) datelog(`updated doc id: ${newObj._id} ${oldStatus} -> ${newStatus}`) } diff --git a/src/types.ts b/src/types.ts index 55d7bd77..5efacf9d 100644 --- a/src/types.ts +++ b/src/types.ts @@ -138,6 +138,8 @@ export interface StandardTx { /** When the document was created. */ createTime?: Date + /** When the document was last updated. */ + updateTime: Date /** The raw transaction data from the partner API. */ rawTx?: unknown @@ -166,6 +168,7 @@ export const asStandardTx = asObject({ // Default to now uncleaner(asOptional(asDate, (): Date | undefined => new Date())) ), + updateTime: asDate, rawTx: asUnknown }) diff --git a/yarn.lock b/yarn.lock index 16e957a8..71586bdc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -42,6 +42,18 @@ dependencies: regenerator-runtime "^0.13.4" +"@clickhouse/client-common@1.11.1": + version "1.11.1" + resolved "https://registry.yarnpkg.com/@clickhouse/client-common/-/client-common-1.11.1.tgz#73a78165656f6e058e2d4eb09cce0f9cf6195255" + integrity sha512-bme0le2yhDSAh13d2fxhSW5ZrNoVqZ3LTyac8jK6hNH0qkksXnjYkLS6KQalPU6NMpffxHmpI4+/Gi2MnX0NCA== + +"@clickhouse/client@^1.10.1": + version "1.11.1" + resolved "https://registry.yarnpkg.com/@clickhouse/client/-/client-1.11.1.tgz#a8f7a7ff891bfc5181601d027c9b5230be5f0484" + integrity sha512-u9h++h72SmWystijNqfNvMkfA+5+Y1LNfmLL/odCL3VgI3oyAPP9ubSw/Yrt2zRZkLKehMMD1kuOej0QHbSoBA== + dependencies: + "@clickhouse/client-common" "1.11.1" + "@eslint-community/eslint-utils@^4.2.0": version "4.4.0" resolved "https://registry.yarnpkg.com/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz#a23514e8fb9af1269d5f7788aa556798d61c6b59" @@ -1007,7 +1019,7 @@ "@types/node" "*" form-data "^3.0.0" -"@types/node@*", "@types/node@^14.0.22": +"@types/node@*": version "14.6.0" resolved "https://registry.npmjs.org/@types/node/-/node-14.6.0.tgz" integrity sha512-mikldZQitV94akrc4sCcSjtJfsTKt4p+e/s0AGscVA6XArQ9kFclP+ZiYUMnq987rc6QlYxXv/EivqlfSLxpKA== @@ -1022,6 +1034,13 @@ resolved "https://registry.npmjs.org/@types/node/-/node-12.12.54.tgz" integrity sha512-ge4xZ3vSBornVYlDnk7yZ0gK6ChHf/CHB7Gl1I0Jhah8DDnEQqBzgohYG4FX4p81TNirSETOiSyn+y1r9/IR6w== +"@types/node@^20.17.17": + version "20.17.52" + resolved "https://registry.yarnpkg.com/@types/node/-/node-20.17.52.tgz#0cf582371f88529c61f384fcd57a21b797c56c6e" + integrity sha512-2aj++KfxubvW/Lc0YyXE3OEW7Es8TWn1MsRzYgcOGyTNQxi0L8rxQUCZ7ZbyOBWZQD5I63PV9egZWMsapVaklg== + dependencies: + undici-types "~6.19.2" + "@types/parse-json@^4.0.0": version "4.0.0" resolved "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.0.tgz" @@ -7418,6 +7437,11 @@ underscore@1.9.1: resolved "https://registry.npmjs.org/underscore/-/underscore-1.9.1.tgz" integrity sha512-5/4etnCkd9c8gwgowi5/om/mYO5ajCaOgdzj/oW+0eQV9WxKBDZw5+ycmKmeaTXjInS/W0BzpGLo2xR2aBwZdg== +undici-types@~6.19.2: + version "6.19.8" + resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-6.19.8.tgz#35111c9d1437ab83a7cdc0abae2f26d88eda0a02" + integrity sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw== + universal-cookie@^4.0.0: version "4.0.4" resolved "https://registry.npmjs.org/universal-cookie/-/universal-cookie-4.0.4.tgz"