From 7eb478fa3fca25739b3204b64085aeab754d9576 Mon Sep 17 00:00:00 2001 From: Alexander Ruliov Date: Sat, 14 Jan 2023 21:32:20 +0300 Subject: [PATCH 1/3] add in-memory dev option to not run mongo for simple fixes --- src/__tests__/helpers/create-context.ts | 1 + src/config.ts | 20 +++++++ src/context.ts | 32 ++++++++++- src/database/memory/database.ts | 73 +++++++++++++++++++++++++ src/index.ts | 18 ++++++ src/types/app-context.ts | 2 + src/types/database.ts | 5 +- 7 files changed, 148 insertions(+), 3 deletions(-) diff --git a/src/__tests__/helpers/create-context.ts b/src/__tests__/helpers/create-context.ts index 29635b0b..254f401e 100644 --- a/src/__tests__/helpers/create-context.ts +++ b/src/__tests__/helpers/create-context.ts @@ -36,6 +36,7 @@ export const createTestAppContext = ({ telegramApiRoot, telegramPollingInterval: 1, mongoUri: 'mongonotused', + database: 'memory', withPromo: false, logLevel: LogLevel.WARNING, l10nFilesPath: '', diff --git a/src/config.ts b/src/config.ts index d34771d6..6e97027b 100644 --- a/src/config.ts +++ b/src/config.ts @@ -24,6 +24,8 @@ export function getConfig(): Config { telegramAdminNickName: process.env.ADMIN_NICK, telegramPollingInterval: 30, mongoUri: ensureEnv('MONGO'), + database: getDatabaseType(), + memoryDatabaseDumpPath: process.env.MEMORY_DB_DUMP_PATH, withPromo: process.env.PROMO_DISABLED !== '1', l10nFilesPath: process.env.L10N_PATH || `${__dirname}/../l10n`, logLevel: logLevelNameToLevel(process.env.LOG_LEVEL), @@ -34,6 +36,10 @@ export function getConfig(): Config { throw firstError; } + if (config.workersCount !== 1 && config.database === 'memory') { + throw new Error('in-memory database supported only for single worker'); + } + return config; function ensureEnv(name: string): string { @@ -51,3 +57,17 @@ export function getConfig(): Config { return value; } } + +const getDatabaseType = (): Config['database'] => { + const type = process.env.DATABASE; + + switch (type) { + case 'mongo': + case 'memory': + return type; + case undefined: + return 'mongo'; + default: + throw new Error(`unknown db type: ${type}, supported: mongo, memory`); + } +}; diff --git a/src/context.ts b/src/context.ts index 30784b59..0b0013d2 100644 --- a/src/context.ts +++ b/src/context.ts @@ -14,6 +14,7 @@ import {createFsPoTranslationsLoader} from './i18n/translations-loader-fs-po'; import {getCommands} from './commands/all-commands'; import {ExplicitPartial} from './types/utility'; import {toDoValidateResponse} from './types/hacks/to-do-validate'; +import {MemoryDatabase} from './database/memory/database'; export type ContextOptions = Partial<{ isWorker: boolean; @@ -36,11 +37,16 @@ const defaultCreateTranslations = ({appContext}: {appContext: AppContext}) => logger: appContext.logger.fork('l10n'), }); +const defaultCreateDatabase = ({appContext}: {appContext: AppContext}) => + new MongoDatabase({appContext}); +const createMemoryDatabase = ({appContext}: {appContext: AppContext}) => + new MemoryDatabase({appContext}); + export function createContext({ isWorker = true, instanceId, config: customConfig, - createDatabase = ({appContext}) => new MongoDatabase({appContext}), + createDatabase: createDatabaseFn = defaultCreateDatabase, createTranslations = defaultCreateTranslations, getCurrentDate = () => new Date(), getLogger = ({name, config}) => new Logger(name, {logLevel: config.logLevel}), @@ -90,6 +96,24 @@ export function createContext({ // and undefined fields will be not used on initialization appContext = initialAppContext as AppContext; + const createDatabase = (() => { + if (createDatabaseFn !== defaultCreateDatabase) { + return createDatabaseFn; + } + + switch (config.database) { + case 'mongo': + return defaultCreateDatabase; + case 'memory': + return createMemoryDatabase; + default: { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const shouldBeNever: never = config.database; + throw new Error('impossible'); + } + } + })(); + appContext.database = createDatabase({appContext}); appContext.translations = createTranslations({appContext}); @@ -106,6 +130,8 @@ export function createContext({ }; appContext.stop = async () => { + logger.info('stop', {state: 'start'}); + await shutdownHandlers.reduceRight((acc, handler) => { return acc.finally(() => handler().catch((error) => { @@ -115,8 +141,12 @@ export function createContext({ ); }, Promise.resolve()); + await appContext.database.stop?.(); + await bot.stop(); await reportedShutdown(); + + logger.info('stop', {state: 'finish'}); }; appContext.run = (fun) => { diff --git a/src/database/memory/database.ts b/src/database/memory/database.ts index 17073303..b526b5eb 100644 --- a/src/database/memory/database.ts +++ b/src/database/memory/database.ts @@ -15,6 +15,7 @@ import {sleep} from '@root/util/async/sleep'; import {assertNonNullish} from '@root/util/assert/assert-non-nullish'; import {getOrCreateMap, getOrCreateSet} from '@root/util/map/get-or-create'; import {MessageId} from '@root/models/message'; +import {readFile, writeFile} from 'fs/promises'; /** * In-memory mock of MongoDatabase, used in tests. @@ -39,8 +40,80 @@ export class MemoryDatabase implements Database { // behavior like real db connector // If MemoryDatabase will be used for real, move those sleeps to a flag await sleep(1); + + await this.restoreDump(); }; + async stop() { + await this.makeDump(); + } + + private async restoreDump() { + const {memoryDatabaseDumpPath} = this.appContext.config; + + if (!memoryDatabaseDumpPath) { + return; + } + + const content = await (async () => { + try { + return await readFile(memoryDatabaseDumpPath, 'utf-8'); + } catch (error) { + if (error.code === 'ENOENT') { + return null; + } + + throw error; + } + })(); + + if (!content) { + return; + } + + // WARN: there should be some validation, describe data in `zod`, please + const dump = JSON.parse(content); + + [ + [this.chats, dump.chats], + [this.cappedKickedUsers, dump.cappedKickedUsers], + [this.entryMessages, dump.entryMessages], + [this.messagesToDelete, dump.messagesToDelete], + [this.cappedMessages, dump.cappedMessages], + ].forEach(([map, dump]) => { + dump.forEach(([key, value]) => { + map.set(key, value); + }); + }); + + dump.verifiedUsers.forEach((id) => { + this.verifiedUsers.add(id); + }); + } + + private async makeDump() { + const { + isWorker, + config: {memoryDatabaseDumpPath}, + } = this.appContext; + + // Only workers have some state here + if (!isWorker || !memoryDatabaseDumpPath) { + return; + } + + const dump = { + chats: Array.from(this.chats.entries()), + cappedKickedUsers: Array.from(this.cappedKickedUsers.entries()), + entryMessages: Array.from(this.entryMessages.entries()), + verifiedUsers: Array.from(this.verifiedUsers), + messagesToDelete: Array.from(this.messagesToDelete.entries()), + cappedMessages: Array.from(this.cappedMessages.entries()), + }; + + await writeFile(memoryDatabaseDumpPath, JSON.stringify(dump), 'utf-8'); + } + getChatById = async (chatId: ChatId): Promise => { await sleep(1); diff --git a/src/index.ts b/src/index.ts index b16b029d..beffa479 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,6 +6,24 @@ import {createContext} from './context'; const appContext = createContext(); +// in-memory database can work only with single worker and intended for dev-purposes only, +// and also SIGINT handling works bad under `concurrently`, looks like we should resend +// this signal from master to workers and vise-versa (or maybe investigate +// `concurrently` options to send SIGNINT for all child processes +if (appContext.config.database === 'memory') { + process.on('SIGINT', () => { + appContext.stop().then( + () => { + process.exit(0); + }, + (error) => { + appContext.logger.error('sigint', undefined, {error}); + process.exit(1); + }, + ); + }); +} + appContext.run(() => { if (isMaster) { appContext.isWorker = false; diff --git a/src/types/app-context.ts b/src/types/app-context.ts index eb9b67e9..b6f5b61a 100644 --- a/src/types/app-context.ts +++ b/src/types/app-context.ts @@ -17,6 +17,8 @@ export type Config = { telegramApiRoot: string; telegramPollingInterval: number; mongoUri: string; + database: 'mongo' | 'memory'; + memoryDatabaseDumpPath?: string; isNeedUpdateAutocomplete: boolean; /** @deprecated */ withPromo: boolean; diff --git a/src/types/database.ts b/src/types/database.ts index 5010a3e1..470ca730 100644 --- a/src/types/database.ts +++ b/src/types/database.ts @@ -22,8 +22,9 @@ export type SetChatPropertyOptions = Values; type FilteredKeys = {[P in keyof T]: T[P] extends U ? P : never}[keyof T]; export type BooleanChatPropertyKey = NonNullable>; -export interface Database { +export type Database = { init: () => Promise; + stop?: () => Promise; // TODO: change to `Chat | undefined` getChatById: (chatId: ChatId) => Promise; @@ -64,4 +65,4 @@ export interface Database { findCappedMessages: ( query: Partial, ) => Promise; -} +}; From 518254071d0bcadff66b65a6cd444d3fe59f1e6c Mon Sep 17 00:00:00 2001 From: Alexander Ruliov Date: Sat, 14 Jan 2023 21:35:07 +0300 Subject: [PATCH 2/3] Revert "remove GroupAnonymousBot" This reverts commit 6d6de93a4503d19b537152a4fd68b3ad0d59d922. --- src/middlewares/checkIfFromReplier.ts | 9 +++++++++ src/middlewares/checkLock.ts | 4 ++++ 2 files changed, 13 insertions(+) diff --git a/src/middlewares/checkIfFromReplier.ts b/src/middlewares/checkIfFromReplier.ts index 30666819..302cabd7 100644 --- a/src/middlewares/checkIfFromReplier.ts +++ b/src/middlewares/checkIfFromReplier.ts @@ -16,6 +16,15 @@ export const checkIfFromReplierMiddleware: BotMiddlewareFn = async ( ctx.callbackQuery.message.reply_to_message ) { const message = ctx.callbackQuery.message; + // Anonymous admin + if ( + message.reply_to_message && + message.reply_to_message.from && + message.reply_to_message.from.username && + message.reply_to_message.from.username === 'GroupAnonymousBot' + ) { + return BotMiddlewareNextStrategy.next; + } assertNonNullish(message.reply_to_message?.from); diff --git a/src/middlewares/checkLock.ts b/src/middlewares/checkLock.ts index 6777c0eb..59c14d5d 100644 --- a/src/middlewares/checkLock.ts +++ b/src/middlewares/checkLock.ts @@ -21,6 +21,10 @@ export const checkLockMiddleware: BotMiddlewareFn = async (ctx) => { if (ctx.from.id === ctx.appContext.config.telegramAdminId) { return BotMiddlewareNextStrategy.next; } + // If from the group anonymous bot, then continue + if (ctx.from?.username === 'GroupAnonymousBot') { + return BotMiddlewareNextStrategy.next; + } // If from admin, then continue if (ctx.isAdministrator) { return BotMiddlewareNextStrategy.next; From 44f6ffb36a70d97f66c77cdfa45dfcdfa2ed7ec5 Mon Sep 17 00:00:00 2001 From: Alexander Ruliov Date: Sat, 14 Jan 2023 21:38:57 +0300 Subject: [PATCH 3/3] increase node version --- .github/workflows/github-actions-pr.yml | 2 +- Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/github-actions-pr.yml b/.github/workflows/github-actions-pr.yml index 29e746ed..328ccd85 100644 --- a/.github/workflows/github-actions-pr.yml +++ b/.github/workflows/github-actions-pr.yml @@ -11,7 +11,7 @@ jobs: - name: Setup nodejs uses: actions/setup-node@v2 with: - node-version: '12' + node-version: '16' cache: 'yarn' - name: Install dependencies run: yarn install diff --git a/Dockerfile b/Dockerfile index 1027a64a..a0db8e71 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node:12-alpine +FROM node:16-alpine RUN mkdir -p /usr/src/app WORKDIR /usr/src/app