From caff4a7124f1bf54ed9175adc882ca85dd6ea88d Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 26 Apr 2025 17:37:20 +0100 Subject: [PATCH 1/2] Add WWBOTA spots --- config_clean.js | 7 ++ hamutil.js | 3 + notify/email.js | 8 +++ notify/telnetconn.js | 3 + server.js | 9 +++ tools/updateWwbotaBunkers.js | 50 ++++++++++++++ wwbotaspots.js | 125 +++++++++++++++++++++++++++++++++++ 7 files changed, 205 insertions(+) create mode 100644 tools/updateWwbotaBunkers.js create mode 100644 wwbotaspots.js diff --git a/config_clean.js b/config_clean.js index 7fa561f..727b155 100644 --- a/config_clean.js +++ b/config_clean.js @@ -20,6 +20,13 @@ config.wwff = { listUrl: 'http://wwff.co/wwff-data/wwff_directory.csv' }; +config.wwbota = { + spotsUrl: 'https://api.wwbota.org/spots/', + listUrl: 'https://wwbota.org/wwbota-3', + refreshInterval: 60*1000, + spotMaxAge: 5*60*1000 +}; + config.mongodb = { url: 'mongodb://hamalert:@localhost:27017/hamalert', dbName: 'hamalert' diff --git a/hamutil.js b/hamutil.js index f0c551d..6724d32 100644 --- a/hamutil.js +++ b/hamutil.js @@ -57,6 +57,9 @@ exports.makeSpotParams = function(spot, comment, actions) { wwffRef: spot.wwffRef, wwffDivision: spot.wwffDivision, wwffName: spot.wwffName, + wwbotaRef: spot.wwbotaRef, + wwbotaScheme: spot.wwbotaScheme, + wwbotaName: spot.wwbotaName, iotaGroupRef: spot.iotaGroupRef, iotaGroupName: spot.iotaGroupName }; diff --git a/notify/email.js b/notify/email.js index 68fe82e..b46313b 100644 --- a/notify/email.js +++ b/notify/email.js @@ -125,6 +125,14 @@ class EmailNotifier extends Notifier { text += `Park name: ${spot.wwffName}\n`; } } + + if (spot.wwbotaRef) { + text += "\n"; + text += `Bunker ref: ${spot.wwbotaRef}\n`; + if (spot.wwbotaName) { + text += `Bunker name: ${spot.wwbotaName}\n`; + } + } if (spot.iotaGroupRef) { text += "\n"; diff --git a/notify/telnetconn.js b/notify/telnetconn.js index 494850e..6a4be8d 100644 --- a/notify/telnetconn.js +++ b/notify/telnetconn.js @@ -167,6 +167,9 @@ class TelnetConnection extends EventEmitter { if (spot.wwffRef) { commentElements.push(spot.wwffRef); } + if (spot.wwbotaRef) { + commentElements.push(spot.wwbotaRef); + } if (spot.iotaGroupRef) { commentElements.push(spot.iotaGroupRef); } diff --git a/server.js b/server.js index 94fc6f8..0fa247e 100644 --- a/server.js +++ b/server.js @@ -1,6 +1,7 @@ const util = require('util'); const SotaSpotReceiver = require('./sotaspots'); const PotaSpotReceiver = require('./potaspots'); +const WwbotaSpotReceiver = require('./wwbotaspots'); const RbnReceiver = require('./rbn'); const PskReporterReceiver = require('./pskreporter'); const ClusterReceiver = require('./cluster'); @@ -85,6 +86,10 @@ function startReceivers() { let potaSpotReceiver = new PotaSpotReceiver(db); potaSpotReceiver.on('spot', notifySpot); potaSpotReceiver.start(); + + let wwbotaSpotReceiver = new WwbotaSpotReceiver(db); + wwbotaSpotReceiver.on('spot', notifySpot); + wwbotaSpotReceiver.start(); config.rbn.forEach(rbnConfig => { let rbnReceiver = new RbnReceiver(rbnConfig); @@ -163,6 +168,10 @@ function runMatcher(spot) { if (spot.wwffDivision) { conditions.wwffDivision = [spot.wwffDivision, "*"]; } + + if (spot.wwbotaScheme) { + conditions.wwbotaScheme = [spot.wwbotaScheme, "*"]; + } if (spot.iotaGroupRef) { conditions.iotaGroupRef = [spot.iotaGroupRef, "*"]; diff --git a/tools/updateWwbotaBunkers.js b/tools/updateWwbotaBunkers.js new file mode 100644 index 0000000..e401961 --- /dev/null +++ b/tools/updateWwbotaBunkers.js @@ -0,0 +1,50 @@ +const request = require('request'); +const axios = require('axios'); +const MongoClient = require('mongodb').MongoClient; +const config = require('../config'); +const assert = require('assert'); +const parse = require('csv-parse'); +const fs = require('fs'); +const async = require('async'); + +let client = new MongoClient(config.mongodb.url) +client.connect((err) => { + assert.equal(null, err); + let db = client.db(config.mongodb.dbName); + + processBunkersList(db); +}); + +function processBunkersList(db) { + let schemes = new Map(); + + request.get(config.wwbota.listUrl, {headers: {'User-Agent': 'Mozilla'}}) + .on('error', (err) => { + callback(err); + return; + }) + .pipe(parse({columns: true, relax_column_count: true, relax: true}, (err, newBunkers) => { + assert.equal(err, null); + + if (newBunkers.length < 20000) { + callback(new Error("Bad number of WWBOTA bunkers, expecting more than 20000")); + return; + } + + for (let bunker of newBunkers) { + if (bunker.Scheme) { + let scheme = bunker.Scheme.trim().toUpperCase() + let schemeData = schemes.get(scheme); + if (!scheme) { + schemeData = {scheme: scheme, dxcc: bunker.DXCC, program: 'wwbota'}; + schemes.set(scheme, schemeData); + } + } + } + + let schemesCollection = db.collection('wwbotaSchemes'); + schemesCollection.deleteMany({}); + schemesCollection.insertMany([...schemes.values()]); + client.close(); + })); +} diff --git a/wwbotaspots.js b/wwbotaspots.js new file mode 100644 index 0000000..99d25d4 --- /dev/null +++ b/wwbotaspots.js @@ -0,0 +1,125 @@ +const request = require('request'); +const config = require('./config'); +const EventEmitter = require('events'); +const util = require('util'); +const crypto = require('crypto'); +const sprintf = require('sprintf'); + +class WwbotaSpotReceiver extends EventEmitter { + constructor(db) { + super(); + this.db = db; + this.lastProcessedTime = null; + } + + start() { + setInterval(() => { + this.refreshSpots(); + }, config.wwbota.refreshInterval); + this.refreshSpots(); + } + + refreshSpots() { + console.log("Refreshing WWBOTA JSON feed"); + + let req = request({ + url: config.wwbota.spotsUrl, + headers: { + 'User-Agent': 'HamAlert/1.0 (+https://hamalert.org)' + }, + json: true + }, (error, response, body) => { + if (error) { + console.error(`Loading WWBOTA feed failed: ${error}`); + return; + } + + if (response.statusCode !== 200) { + console.error(`Bad status code ${response.statusCode} from WWBOTA`); + return; + } + + if (!Array.isArray(body)) { + console.error(`Expected array from WWBOTA, but got something else`); + return; + } + + // reverse to process oldest to newest + body.reverse() + body.forEach(spot => { + this.processJsonSpot(spot) + }); + if (body.length > 0) { + this.lastProcessedTime = spot[body.length - 1].time; // Previously reversed, so newest last + } + }); + } + + processJsonSpot(jsonSpot) { + try { + jsonSpot.time = new Date(jsonSpot.time); + + // Check if spot newer that last batch processed + // edited spot have new timestamp as well + if (this.lastProcessedTime && this.lastProcessedTime >= jsonSpot.time) { + return; + } + + // Ignore old spots + if ((new Date() - jsonSpot.spotTime) > config.wwbota.spotMaxAge) { + return; + } + + // Ignore QRT/Test spots + if (jsonSpot.type !== "Live") { + return; + } + + jsonSpot.call = jsonSpot.call.toUpperCase().replace(/\s/g, ''); + + if (jsonSpot.comment === null) { + jsonSpot.comment = ""; + } + + // Clean up frequency + let frequency = sprintf("%.4f", jsonSpot.freq); + + let spot = { + source: 'wwbota', + time: jsonSpot.time.toISOString().substring(11, 16), + fullCallsign: jsonSpot.call, + wwbotaScheme: spot.reference[0].scheme, // Multiple, but lets just take the first. + wwbotaRef: spot.reference[0].reference, + wwbotaName: spot.reference[0].name, + frequency, + mode: jsonSpot.mode.toLowerCase().trim(), + comment: jsonSpot.comment.trim(), + spotter: jsonSpot.spotter.toUpperCase().trim().replace('-#', '') + }; + + spot.rawText = `${spot.time} ${spot.fullCallsign} in ${spot.wwbotaRef} (${spot.wwbotaName}) ${spot.frequency}`; + if (spot.mode) { + spot.rawText += ` ${spot.mode.toUpperCase()}`; + } + if (spot.comment) { + spot.rawText += `: ${spot.comment}`; + } + spot.title = `WWBOTA ${spot.fullCallsign} in ${spot.wwbotaRef} (${spot.frequency}`; + if (spot.mode) { + spot.title += " " + spot.mode.toUpperCase(); + } + spot.title += ")"; + + //console.log(spot); + this.emit("spot", spot); + + } catch (e) { + console.error("Exception while processing WWBOTA spot", e); + } + } +} + +let rec = new WwbotaSpotReceiver(); +rec.start(); + +module.exports = WwbotaSpotReceiver; From 01d99a2eb8efe6a688715a34a7fda96046c2e10c Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 31 Aug 2025 08:44:35 +0100 Subject: [PATCH 2/2] Change WWBOTA to use SSE --- config_clean.js | 4 +-- package-lock.json | 35 ++++++++++++++++++++++ package.json | 1 + wwbotaspots.js | 76 ++++++++++++----------------------------------- 4 files changed, 56 insertions(+), 60 deletions(-) diff --git a/config_clean.js b/config_clean.js index 727b155..cdbe1ba 100644 --- a/config_clean.js +++ b/config_clean.js @@ -21,10 +21,8 @@ config.wwff = { }; config.wwbota = { - spotsUrl: 'https://api.wwbota.org/spots/', + spotsUrl: 'https://api.wwbota.org/spots/?age=0', listUrl: 'https://wwbota.org/wwbota-3', - refreshInterval: 60*1000, - spotMaxAge: 5*60*1000 }; config.mongodb = { diff --git a/package-lock.json b/package-lock.json index 0d397df..e6517e0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,6 +19,7 @@ "carrier": "^0.3.0", "clone": "^2.1.2", "csv": "^5.3.2", + "eventsource": "^4.0.0", "expand-template": "^1.1.1", "express": "^4.17.1", "firebase-admin": "^13.4.0", @@ -3594,6 +3595,27 @@ "integrity": "sha512-5EM1GHXycJBS6mauYAbVKT1cVs7POKWb2NXD4Vyt8dDqeZa7LaDK1/sjtL+Zb0lzTpSNil4596Dyu97hz37QLg==", "license": "MIT" }, + "node_modules/eventsource": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-4.0.0.tgz", + "integrity": "sha512-fvIkb9qZzdMxgZrEQDyll+9oJsyaVvY92I2Re+qK0qEJ+w5s0X3dtz+M0VAPOjP1gtU3iqWyjQ0G3nvd5CLZ2g==", + "license": "MIT", + "dependencies": { + "eventsource-parser": "^3.0.1" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/eventsource-parser": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.0.6.tgz", + "integrity": "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/expand-template": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-1.1.1.tgz", @@ -10301,6 +10323,19 @@ "resolved": "https://registry.npmjs.org/eventemitter2/-/eventemitter2-5.0.1.tgz", "integrity": "sha512-5EM1GHXycJBS6mauYAbVKT1cVs7POKWb2NXD4Vyt8dDqeZa7LaDK1/sjtL+Zb0lzTpSNil4596Dyu97hz37QLg==" }, + "eventsource": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-4.0.0.tgz", + "integrity": "sha512-fvIkb9qZzdMxgZrEQDyll+9oJsyaVvY92I2Re+qK0qEJ+w5s0X3dtz+M0VAPOjP1gtU3iqWyjQ0G3nvd5CLZ2g==", + "requires": { + "eventsource-parser": "^3.0.1" + } + }, + "eventsource-parser": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.0.6.tgz", + "integrity": "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg==" + }, "expand-template": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-1.1.1.tgz", diff --git a/package.json b/package.json index cf0ca5c..407962e 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "carrier": "^0.3.0", "clone": "^2.1.2", "csv": "^5.3.2", + "eventsource": "^4.0.0", "expand-template": "^1.1.1", "express": "^4.17.1", "firebase-admin": "^13.4.0", diff --git a/wwbotaspots.js b/wwbotaspots.js index 99d25d4..df99902 100644 --- a/wwbotaspots.js +++ b/wwbotaspots.js @@ -1,8 +1,6 @@ -const request = require('request'); +const eventsource = require('eventsource'); const config = require('./config'); const EventEmitter = require('events'); -const util = require('util'); -const crypto = require('crypto'); const sprintf = require('sprintf'); class WwbotaSpotReceiver extends EventEmitter { @@ -13,62 +11,26 @@ class WwbotaSpotReceiver extends EventEmitter { } start() { - setInterval(() => { - this.refreshSpots(); - }, config.wwbota.refreshInterval); - this.refreshSpots(); - } - - refreshSpots() { - console.log("Refreshing WWBOTA JSON feed"); - - let req = request({ - url: config.wwbota.spotsUrl, - headers: { - 'User-Agent': 'HamAlert/1.0 (+https://hamalert.org)' - }, - json: true - }, (error, response, body) => { - if (error) { - console.error(`Loading WWBOTA feed failed: ${error}`); - return; - } - - if (response.statusCode !== 200) { - console.error(`Bad status code ${response.statusCode} from WWBOTA`); - return; - } - - if (!Array.isArray(body)) { - console.error(`Expected array from WWBOTA, but got something else`); - return; - } - - // reverse to process oldest to newest - body.reverse() - body.forEach(spot => { - this.processJsonSpot(spot) - }); - if (body.length > 0) { - this.lastProcessedTime = spot[body.length - 1].time; // Previously reversed, so newest last - } + let es = new eventsource.EventSource(config.wwbota.spotsUrl,{ + fetch: (input, init) => + fetch(input, { + ...init, + headers: { + ...init.headers, + 'User-Agent': 'HamAlert/1.0 (+https://hamalert.org)', + }, + }), }); + es.addEventListener('message', (event) => this.processSpotEvent(event)); + es.addEventListener('error', (error) => { + console.error(`Loading WWBOTA feed failed: ${error.responseCode}`) + }) } - processJsonSpot(jsonSpot) { + processSpotEvent(spotEvent) { try { + let jsonSpot = JSON.parse(spotEvent.data); jsonSpot.time = new Date(jsonSpot.time); - - // Check if spot newer that last batch processed - // edited spot have new timestamp as well - if (this.lastProcessedTime && this.lastProcessedTime >= jsonSpot.time) { - return; - } - - // Ignore old spots - if ((new Date() - jsonSpot.spotTime) > config.wwbota.spotMaxAge) { - return; - } // Ignore QRT/Test spots if (jsonSpot.type !== "Live") { @@ -88,9 +50,9 @@ class WwbotaSpotReceiver extends EventEmitter { source: 'wwbota', time: jsonSpot.time.toISOString().substring(11, 16), fullCallsign: jsonSpot.call, - wwbotaScheme: spot.reference[0].scheme, // Multiple, but lets just take the first. - wwbotaRef: spot.reference[0].reference, - wwbotaName: spot.reference[0].name, + wwbotaScheme: jsonSpot.references[0].scheme, // Multiple, but lets just take the first. + wwbotaRef: jsonSpot.references[0].reference, + wwbotaName: jsonSpot.references[0].name, frequency, mode: jsonSpot.mode.toLowerCase().trim(), comment: jsonSpot.comment.trim(),