From be735fa63cbfa9b7d2388ac76b89169bb08884d0 Mon Sep 17 00:00:00 2001 From: Artem Niehrieiev Date: Mon, 12 Jan 2026 09:03:17 +0000 Subject: [PATCH] added columns_view field in get table rows response object --- .../dto/create-personal-table-settings.dto.ts | 40 +- .../personal-table-settings.controller.ts | 260 +++--- .../use-cases/get-table-rows.use.case.ts | 857 +++++++++--------- 3 files changed, 579 insertions(+), 578 deletions(-) diff --git a/backend/src/entities/table-settings/personal-table-settings/dto/create-personal-table-settings.dto.ts b/backend/src/entities/table-settings/personal-table-settings/dto/create-personal-table-settings.dto.ts index 83a4551fd..f2a962f40 100644 --- a/backend/src/entities/table-settings/personal-table-settings/dto/create-personal-table-settings.dto.ts +++ b/backend/src/entities/table-settings/personal-table-settings/dto/create-personal-table-settings.dto.ts @@ -3,29 +3,29 @@ import { QueryOrderingEnum } from '../../../../enums/query-ordering.enum.js'; import { IsBoolean, IsEnum, IsOptional } from 'class-validator'; export class CreatePersonalTableSettingsDto { - @ApiProperty({ enumName: 'QueryOrderingEnum', enum: QueryOrderingEnum, description: 'The ordering direction' }) - @IsOptional() - @IsEnum(QueryOrderingEnum) - ordering: QueryOrderingEnum; + @ApiProperty({ enumName: 'QueryOrderingEnum', enum: QueryOrderingEnum, description: 'The ordering direction' }) + @IsOptional() + @IsEnum(QueryOrderingEnum) + ordering: QueryOrderingEnum; - @ApiProperty({ type: String, description: 'The ordering field' }) - @IsOptional() - ordering_field: string; + @ApiProperty({ type: String, description: 'The ordering field' }) + @IsOptional() + ordering_field: string; - @ApiProperty({ type: Number, description: 'The number of items per page' }) - @IsOptional() - list_per_page: number; + @ApiProperty({ type: Number, description: 'The number of items per page' }) + @IsOptional() + list_per_page: number; - @ApiProperty({ isArray: true, type: String, description: 'The columns view' }) - @IsOptional() - columns_view: Array; + @ApiProperty({ isArray: true, type: String, description: 'The columns view' }) + @IsOptional() + columns_view: Array; - @ApiProperty({ type: [String], description: 'The order of columns' }) - @IsOptional() - list_fields: Array; + @ApiProperty({ type: [String], description: 'The order of columns' }) + @IsOptional() + list_fields: Array; - @ApiProperty({ type: Boolean, description: 'Whether to use original column names' }) - @IsOptional() - @IsBoolean() - original_names: boolean; + @ApiProperty({ type: Boolean, description: 'Whether to use original column names' }) + @IsOptional() + @IsBoolean() + original_names: boolean; } diff --git a/backend/src/entities/table-settings/personal-table-settings/personal-table-settings.controller.ts b/backend/src/entities/table-settings/personal-table-settings/personal-table-settings.controller.ts index 5d8884a45..6cce7e763 100644 --- a/backend/src/entities/table-settings/personal-table-settings/personal-table-settings.controller.ts +++ b/backend/src/entities/table-settings/personal-table-settings/personal-table-settings.controller.ts @@ -1,23 +1,23 @@ import { - Body, - Controller, - Delete, - Get, - HttpException, - HttpStatus, - Inject, - Injectable, - Put, - UseGuards, - UseInterceptors, + Body, + Controller, + Delete, + Get, + HttpException, + HttpStatus, + Inject, + Injectable, + Put, + UseGuards, + UseInterceptors, } from '@nestjs/common'; import { ApiBearerAuth, ApiBody, ApiOperation, ApiParam, ApiQuery, ApiResponse, ApiTags } from '@nestjs/swagger'; import { SentryInterceptor } from '../../../interceptors/sentry.interceptor.js'; import { UseCaseType } from '../../../common/data-injection.tokens.js'; import { - ICreateUpdatePersonalTableSettings, - IDeletePersonalTableSettings, - IFindPersonalTableSettings, + ICreateUpdatePersonalTableSettings, + IDeletePersonalTableSettings, + IFindPersonalTableSettings, } from './use-cases/personal-table-settings.use-cases.interface.js'; import { TableReadGuard } from '../../../guards/table-read.guard.js'; import { UserId } from '../../../decorators/user-id.decorator.js'; @@ -37,123 +37,123 @@ import { CreatePersonalTableSettingsDto } from './dto/create-personal-table-sett @ApiTags('Personal table settings') @Injectable() export class PersonalTableSettingsController { - constructor( - @Inject(UseCaseType.FIND_PERSONAL_TABLE_SETTINGS) - private readonly findPersonalTableSettingsUseCase: IFindPersonalTableSettings, - @Inject(UseCaseType.CREATE_UPDATE_PERSONAL_TABLE_SETTINGS) - private readonly createUpdatePersonalTableSettingsUseCase: ICreateUpdatePersonalTableSettings, - @Inject(UseCaseType.DELETE_PERSONAL_TABLE_SETTINGS) - private readonly deletePersonalTableSettingsUseCase: IDeletePersonalTableSettings, - ) {} + constructor( + @Inject(UseCaseType.FIND_PERSONAL_TABLE_SETTINGS) + private readonly findPersonalTableSettingsUseCase: IFindPersonalTableSettings, + @Inject(UseCaseType.CREATE_UPDATE_PERSONAL_TABLE_SETTINGS) + private readonly createUpdatePersonalTableSettingsUseCase: ICreateUpdatePersonalTableSettings, + @Inject(UseCaseType.DELETE_PERSONAL_TABLE_SETTINGS) + private readonly deletePersonalTableSettingsUseCase: IDeletePersonalTableSettings, + ) {} - @ApiOperation({ summary: 'Find user personal table settings' }) - @ApiResponse({ - status: 200, - description: 'Table settings found.', - type: FoundPersonalTableSettingsDto, - }) - @ApiParam({ name: 'connectionId', required: true }) - @ApiQuery({ name: 'tableName', required: true }) - @UseGuards(TableReadGuard) - @Get('/settings/personal/:connectionId') - async findAll( - @SlugUuid('connectionId') connectionId: string, - @QueryTableName() tableName: string, - @MasterPassword() masterPwd: string, - @UserId() userId: string, - ): Promise { - if (!connectionId) { - throw new HttpException( - { - message: Messages.CONNECTION_ID_MISSING, - }, - HttpStatus.BAD_REQUEST, - ); - } - const inputData: FindPersonalTableSettingsDs = { - connectionId, - tableName, - userId, - masterPassword: masterPwd, - }; - return await this.findPersonalTableSettingsUseCase.execute(inputData, InTransactionEnum.OFF); - } + @ApiOperation({ summary: 'Find user personal table settings' }) + @ApiResponse({ + status: 200, + description: 'Table settings found.', + type: FoundPersonalTableSettingsDto, + }) + @ApiParam({ name: 'connectionId', required: true }) + @ApiQuery({ name: 'tableName', required: true }) + @UseGuards(TableReadGuard) + @Get('/settings/personal/:connectionId') + async findAll( + @SlugUuid('connectionId') connectionId: string, + @QueryTableName() tableName: string, + @MasterPassword() masterPwd: string, + @UserId() userId: string, + ): Promise { + if (!connectionId) { + throw new HttpException( + { + message: Messages.CONNECTION_ID_MISSING, + }, + HttpStatus.BAD_REQUEST, + ); + } + const inputData: FindPersonalTableSettingsDs = { + connectionId, + tableName, + userId, + masterPassword: masterPwd, + }; + return await this.findPersonalTableSettingsUseCase.execute(inputData, InTransactionEnum.OFF); + } - @ApiOperation({ summary: 'Create or update user personal table settings' }) - @ApiResponse({ - status: 200, - description: 'Table settings crated/updated.', - type: FoundPersonalTableSettingsDto, - }) - @ApiBody({ type: CreatePersonalTableSettingsDto }) - @ApiParam({ name: 'connectionId', required: true }) - @ApiQuery({ name: 'tableName', required: true }) - @UseGuards(TableReadGuard) - @Put('/settings/personal/:connectionId') - async createOrUpdate( - @SlugUuid('connectionId') connectionId: string, - @QueryTableName() tableName: string, - @MasterPassword() masterPwd: string, - @UserId() userId: string, - @Body() personalSettingsData: CreatePersonalTableSettingsDto, - ): Promise { - if (!connectionId) { - throw new HttpException( - { - message: Messages.CONNECTION_ID_MISSING, - }, - HttpStatus.BAD_REQUEST, - ); - } - const inputData: CreatePersonalTableSettingsDs = { - table_settings_metadata: { - connection_id: connectionId, - table_name: tableName, - user_id: userId, - master_password: masterPwd, - }, - table_settings_data: { - columns_view: personalSettingsData.columns_view || null, - list_fields: personalSettingsData.list_fields || null, - list_per_page: personalSettingsData.list_per_page || null, - ordering: personalSettingsData.ordering || null, - ordering_field: personalSettingsData.ordering_field || null, - original_names: personalSettingsData.original_names || null, - }, - }; - return await this.createUpdatePersonalTableSettingsUseCase.execute(inputData, InTransactionEnum.OFF); - } + @ApiOperation({ summary: 'Create or update user personal table settings' }) + @ApiResponse({ + status: 200, + description: 'Table settings crated/updated.', + type: FoundPersonalTableSettingsDto, + }) + @ApiBody({ type: CreatePersonalTableSettingsDto }) + @ApiParam({ name: 'connectionId', required: true }) + @ApiQuery({ name: 'tableName', required: true }) + @UseGuards(TableReadGuard) + @Put('/settings/personal/:connectionId') + async createOrUpdate( + @SlugUuid('connectionId') connectionId: string, + @QueryTableName() tableName: string, + @MasterPassword() masterPwd: string, + @UserId() userId: string, + @Body() personalSettingsData: CreatePersonalTableSettingsDto, + ): Promise { + if (!connectionId) { + throw new HttpException( + { + message: Messages.CONNECTION_ID_MISSING, + }, + HttpStatus.BAD_REQUEST, + ); + } + const inputData: CreatePersonalTableSettingsDs = { + table_settings_metadata: { + connection_id: connectionId, + table_name: tableName, + user_id: userId, + master_password: masterPwd, + }, + table_settings_data: { + columns_view: personalSettingsData.columns_view || null, + list_fields: personalSettingsData.list_fields || null, + list_per_page: personalSettingsData.list_per_page || null, + ordering: personalSettingsData.ordering || null, + ordering_field: personalSettingsData.ordering_field || null, + original_names: personalSettingsData.original_names || null, + }, + }; + return await this.createUpdatePersonalTableSettingsUseCase.execute(inputData, InTransactionEnum.OFF); + } - @ApiOperation({ summary: 'Clear user personal table settings' }) - @ApiResponse({ - status: 200, - description: 'Table settings removed.', - type: FoundPersonalTableSettingsDto, - }) - @ApiParam({ name: 'connectionId', required: true }) - @ApiQuery({ name: 'tableName', required: true }) - @UseGuards(TableReadGuard) - @Delete('/settings/personal/:connectionId') - async clearTableSettings( - @SlugUuid('connectionId') connectionId: string, - @QueryTableName() tableName: string, - @MasterPassword() masterPwd: string, - @UserId() userId: string, - ): Promise { - if (!connectionId) { - throw new HttpException( - { - message: Messages.CONNECTION_ID_MISSING, - }, - HttpStatus.BAD_REQUEST, - ); - } - const inputData: FindPersonalTableSettingsDs = { - connectionId, - tableName, - userId, - masterPassword: masterPwd, - }; - return await this.deletePersonalTableSettingsUseCase.execute(inputData, InTransactionEnum.OFF); - } + @ApiOperation({ summary: 'Clear user personal table settings' }) + @ApiResponse({ + status: 200, + description: 'Table settings removed.', + type: FoundPersonalTableSettingsDto, + }) + @ApiParam({ name: 'connectionId', required: true }) + @ApiQuery({ name: 'tableName', required: true }) + @UseGuards(TableReadGuard) + @Delete('/settings/personal/:connectionId') + async clearTableSettings( + @SlugUuid('connectionId') connectionId: string, + @QueryTableName() tableName: string, + @MasterPassword() masterPwd: string, + @UserId() userId: string, + ): Promise { + if (!connectionId) { + throw new HttpException( + { + message: Messages.CONNECTION_ID_MISSING, + }, + HttpStatus.BAD_REQUEST, + ); + } + const inputData: FindPersonalTableSettingsDs = { + connectionId, + tableName, + userId, + masterPassword: masterPwd, + }; + return await this.deletePersonalTableSettingsUseCase.execute(inputData, InTransactionEnum.OFF); + } } diff --git a/backend/src/entities/table/use-cases/get-table-rows.use.case.ts b/backend/src/entities/table/use-cases/get-table-rows.use.case.ts index 1b2837389..b98e8db80 100644 --- a/backend/src/entities/table/use-cases/get-table-rows.use.case.ts +++ b/backend/src/entities/table/use-cases/get-table-rows.use.case.ts @@ -10,10 +10,10 @@ import AbstractUseCase from '../../../common/abstract-use.case.js'; import { IGlobalDatabaseContext } from '../../../common/application/global-database-context.interface.js'; import { BaseType } from '../../../common/data-injection.tokens.js'; import { - AmplitudeEventTypeEnum, - LogOperationTypeEnum, - OperationResultStatusEnum, - WidgetTypeEnum, + AmplitudeEventTypeEnum, + LogOperationTypeEnum, + OperationResultStatusEnum, + WidgetTypeEnum, } from '../../../enums/index.js'; import { ExceptionOperations } from '../../../exceptions/custom-exceptions/exception-operation.js'; import { NonAvailableInFreePlanException } from '../../../exceptions/custom-exceptions/non-available-in-free-plan-exception.js'; @@ -46,429 +46,430 @@ import { PersonalTableSettingsEntity } from '../../table-settings/personal-table @Injectable() export class GetTableRowsUseCase extends AbstractUseCase implements IGetTableRows { - constructor( - @Inject(BaseType.GLOBAL_DB_CONTEXT) - protected _dbContext: IGlobalDatabaseContext, - private amplitudeService: AmplitudeService, - private tableLogsService: TableLogsService, - ) { - super(); - } - - protected async implementation(inputData: GetTableRowsDs): Promise { - let operationResult = OperationResultStatusEnum.unknown; - - const { connectionId, masterPwd, page, perPage, query, tableName, userId, filters } = inputData; - let { searchingFieldValue } = inputData; - const connection = await this._dbContext.connectionRepository.findAndDecryptConnection(connectionId, masterPwd); - if (!connection) { - throw new HttpException( - { - message: Messages.CONNECTION_NOT_FOUND, - }, - HttpStatus.BAD_REQUEST, - ); - } - - if (connection.is_frozen) { - throw new NonAvailableInFreePlanException(Messages.CONNECTION_IS_FROZEN); - } - - try { - const dao = getDataAccessObject(connection); - const tablesInConnection = await dao.getTablesFromDB(); - const tableNames = tablesInConnection.map((table) => table.tableName); - - if (!tableNames.includes(tableName)) { - throw new BadRequestException(Messages.TABLE_NOT_FOUND); - } - - let userEmail: string; - if (isConnectionTypeAgent(connection.type)) { - userEmail = await this._dbContext.userRepository.getUserEmailOrReturnNull(userId); - } - - // eslint-disable-next-line prefer-const - let { tableSettings, tableCustomFields, tableWidgets } = - await this._dbContext.tableSettingsRepository.findTableCustoms(connectionId, tableName); - - /* eslint-disable */ - let [ - tablePrimaryColumns, - tableForeignKeys, - tableStructure, - userTablePermissions, - customActionEvents, - savedTableFilters, - personalTableSettings, - /* eslint-enable */ - ] = await Promise.all([ - dao.getTablePrimaryColumns(tableName, userEmail), - dao.getTableForeignKeys(tableName, userEmail), - dao.getTableStructure(tableName, userEmail), - this._dbContext.userAccessRepository.getUserTablePermissions(userId, connectionId, tableName, masterPwd), - this._dbContext.actionEventsRepository.findCustomEventsForTable(connectionId, tableName), - this._dbContext.tableFiltersRepository.findTableFiltersForTableInConnection(tableName, connectionId), - this._dbContext.personalTableSettingsRepository.findUserTableSettings(userId, connectionId, tableName), - ]); - const filteringFields: Array = isObjectEmpty(filters) - ? findFilteringFieldsUtil(query, tableStructure) - : parseFilteringFieldsFromBodyData(filters, tableStructure); - - const orderingField = findOrderingFieldUtil(query, tableStructure, tableSettings); - - const configured = !!tableSettings; - - const allowCsvExport = tableSettings?.allow_csv_export ?? true; - const allowCsvImport = tableSettings?.allow_csv_import ?? true; - const can_delete = tableSettings?.can_delete ?? true; - const can_update = tableSettings?.can_update ?? true; - const can_add = tableSettings?.can_add ?? true; - - //todo rework in daos - tableSettings = tableSettings ? tableSettings : ({} as TableSettingsEntity); - personalTableSettings = personalTableSettings ? personalTableSettings : ({} as PersonalTableSettingsEntity); - - const { autocomplete, referencedColumn } = query; - - const autocompleteFields = - autocomplete && referencedColumn - ? findAutocompleteFieldsUtil(query, tableStructure, tableSettings, referencedColumn) - : undefined; - - const builtDAOsTableSettings = buildDAOsTableSettingsDs(tableSettings, personalTableSettings); - if (orderingField) { - builtDAOsTableSettings.ordering_field = orderingField.field; - builtDAOsTableSettings.ordering = orderingField.value; - } - if ( - isHexString(searchingFieldValue) && - (tableStructure.some((field) => isBinary(field.data_type)) || - connection.type === ConnectionTypesEnum.mongodb || - connection.type === ConnectionTypesEnum.agent_mongodb) - ) { - searchingFieldValue = hexToBinary(searchingFieldValue) as any; - builtDAOsTableSettings.search_fields = tableStructure - .filter((field) => isBinary(field.data_type)) - .map((field) => field.column_name); - if (connection.type === 'mongodb' || connection.type === 'agent_mongodb') { - builtDAOsTableSettings.search_fields.push('_id'); - } - } - - let rows: FoundRowsDS; - try { - rows = await dao.getRowsFromTable( - tableName, - builtDAOsTableSettings, - page, - perPage, - searchingFieldValue, - filteringFields, - autocompleteFields, - tableStructure, - userEmail, - ); - } catch (e) { - Sentry.captureException(e); - throw new UnknownSQLException(e.message, ExceptionOperations.FAILED_TO_GET_ROWS_FROM_TABLE); - } - rows = processRowsUtil(rows, tableWidgets, tableStructure, tableCustomFields); - - const foreignKeysFromWidgets: Array = tableWidgets - .filter((widget) => widget.widget_type === WidgetTypeEnum.Foreign_key) - .reduce>((acc, widget) => { - if (widget.widget_params) { - try { - const widgetParams = JSON5.parse(widget.widget_params) as ForeignKeyDSInfo; - acc.push(widgetParams); - } catch (_e) { - // - } - } - return acc; - }, []); - - tableForeignKeys = [...tableForeignKeys, ...foreignKeysFromWidgets]; - - const canUserReadForeignTables = await Promise.all( - tableForeignKeys.map((foreignKey) => - this._dbContext.userAccessRepository - .improvedCheckTableRead(userId, connectionId, foreignKey.referenced_table_name, masterPwd) - .then((canRead) => ({ - tableName: foreignKey.referenced_table_name, - canRead, - })), - ), - ); - - const readableForeignTables = new Set( - canUserReadForeignTables.filter(({ canRead }) => canRead).map(({ tableName }) => tableName), - ); - - tableForeignKeys = tableForeignKeys.filter(({ referenced_table_name }) => - readableForeignTables.has(referenced_table_name), - ); - - if (tableForeignKeys && tableForeignKeys.length > 0) { - tableForeignKeys = await Promise.all( - tableForeignKeys.map((el) => this.attachForeignColumnNames(el, userId, connectionId, dao).catch(() => el)), - ); - } - - const formedTableStructure = formFullTableStructure(tableStructure, tableSettings); - - const largeDataset = rows.large_dataset || rows.pagination.total > Constants.LARGE_DATASET_ROW_LIMIT; - - const listFields = findAvailableFields(builtDAOsTableSettings, tableStructure); - const actionEventsDtos = customActionEvents.map((el) => buildActionEventDto(el)); - const savedFiltersRO = savedTableFilters.map((el) => buildCreatedTableFilterRO(el)); - - const rowsRO = { - rows: rows.data, - primaryColumns: tablePrimaryColumns, - pagination: rows.pagination, - sortable_by: builtDAOsTableSettings?.sortable_by?.length > 0 ? builtDAOsTableSettings.sortable_by : [], - ordering_field: personalTableSettings.ordering_field ? personalTableSettings.ordering_field : undefined, - ordering: personalTableSettings.ordering ? personalTableSettings.ordering : undefined, - columns_view: builtDAOsTableSettings.columns_view ? builtDAOsTableSettings.columns_view : undefined, - structure: formedTableStructure, - foreignKeys: tableForeignKeys, - configured: configured, - widgets: tableWidgets, - identity_column: builtDAOsTableSettings.identity_column ? builtDAOsTableSettings.identity_column : null, - table_permissions: userTablePermissions, - list_fields: listFields, - action_events: actionEventsDtos, - table_actions: actionEventsDtos, - large_dataset: largeDataset, - allow_csv_export: allowCsvExport, - allow_csv_import: allowCsvImport, - saved_filters: savedFiltersRO, - can_delete: can_delete, - can_update: can_update, - can_add: can_add, - table_settings: { - sortable_by: builtDAOsTableSettings?.sortable_by?.length > 0 ? builtDAOsTableSettings.sortable_by : [], - ordering: personalTableSettings.ordering ? personalTableSettings.ordering : undefined, - identity_column: builtDAOsTableSettings.identity_column ? builtDAOsTableSettings.identity_column : null, - list_fields: personalTableSettings?.list_fields?.length > 0 ? personalTableSettings.list_fields : [], - allow_csv_export: allowCsvExport, - allow_csv_import: allowCsvImport, - can_delete: can_delete, - can_update: can_update, - can_add: can_add, - }, - }; - - const identitiesMap = new Map(); - - if (tableForeignKeys?.length > 0) { - const uniqueReferencedTables = [...new Set(tableForeignKeys.map((fk) => fk.referenced_table_name))]; - const foreignTableSettingsPromises = uniqueReferencedTables.map((tableName) => - this._dbContext.tableSettingsRepository.findTableSettings(connectionId, tableName).then((settings) => ({ - tableName, - settings, - })), - ); - const foreignTableSettingsResults = await Promise.all(foreignTableSettingsPromises); - const foreignTableSettingsMap = new Map( - foreignTableSettingsResults.map((result) => [result.tableName, result.settings]), - ); - - const foreignKeyDataMap = new Map }>(); - - for (const foreignKey of tableForeignKeys) { - const valuesSet = new Set(); - for (const row of rowsRO.rows) { - const value = row[foreignKey.column_name]; - if (value !== undefined && value !== null) { - valuesSet.add(value as string | number); - } - } - - const key = `${foreignKey.referenced_table_name}:${foreignKey.referenced_column_name}`; - if (foreignKeyDataMap.has(key)) { - const existing = foreignKeyDataMap.get(key); - valuesSet.forEach((v) => existing!.values.add(v)); - } else { - foreignKeyDataMap.set(key, { foreignKey, values: valuesSet }); - } - } - - const identityPromises = Array.from(foreignKeyDataMap.values()).map(async ({ foreignKey, values }) => { - const foreignTableSettings = foreignTableSettingsMap.get(foreignKey.referenced_table_name); - const builtDAOsForeignTableSettings = buildDAOsTableSettingsDs(foreignTableSettings, {} as any); - const identityColumns = await this.getBatchedIdentityColumns( - Array.from(values), - foreignKey, - dao, - builtDAOsForeignTableSettings, - userEmail, - ); - return { foreignKey, identityColumns }; - }); - - const identityResults = await Promise.all(identityPromises); - - for (const { foreignKey, identityColumns } of identityResults) { - if (!identitiesMap.has(foreignKey.referenced_table_name)) { - identitiesMap.set(foreignKey.referenced_table_name, []); - } - identitiesMap.get(foreignKey.referenced_table_name)?.push(...identityColumns); - } - } - - const identities = Array.from(identitiesMap, ([referenced_table_name, identity_columns]) => ({ - referenced_table_name, - identity_columns, - })); - - const foreignKeysConformity = tableForeignKeys.map((key) => ({ - currentFKeyName: key.column_name, - realFKeyName: key.referenced_column_name, - referencedTableName: key.referenced_table_name, - })); - - const identityLookupMaps = new Map>(); - for (const element of foreignKeysConformity) { - const identityForCurrentTable = identities.find( - (el) => el.referenced_table_name === element.referencedTableName, - ); - - if (!identityForCurrentTable) continue; - - const lookupKey = `${element.referencedTableName}:${element.realFKeyName}`; - if (!identityLookupMaps.has(lookupKey)) { - const identityColumnsMap = new Map( - identityForCurrentTable.identity_columns.map((col) => [col[element.realFKeyName], col]), - ); - identityLookupMaps.set(lookupKey, identityColumnsMap); - } - } - - for (const row of rowsRO.rows) { - for (const element of foreignKeysConformity) { - const lookupKey = `${element.referencedTableName}:${element.realFKeyName}`; - const identityColumnsMap = identityLookupMaps.get(lookupKey); - - if (!identityColumnsMap) continue; - - const identityForCurrentValue = identityColumnsMap.get(row[element.currentFKeyName]); - row[element.currentFKeyName] = - typeof identityForCurrentValue === 'object' && identityForCurrentValue !== null - ? identityForCurrentValue - : {}; - } - } - - operationResult = OperationResultStatusEnum.successfully; - return rowsRO; - } catch (e) { - Sentry.captureException(e); - operationResult = OperationResultStatusEnum.unsuccessfully; - if (e instanceof HttpException) { - throw e; - } - throw new HttpException( - { - message: `${Messages.FAILED_GET_TABLE_ROWS} ${Messages.ERROR_MESSAGE} + constructor( + @Inject(BaseType.GLOBAL_DB_CONTEXT) + protected _dbContext: IGlobalDatabaseContext, + private amplitudeService: AmplitudeService, + private tableLogsService: TableLogsService, + ) { + super(); + } + + protected async implementation(inputData: GetTableRowsDs): Promise { + let operationResult = OperationResultStatusEnum.unknown; + + const { connectionId, masterPwd, page, perPage, query, tableName, userId, filters } = inputData; + let { searchingFieldValue } = inputData; + const connection = await this._dbContext.connectionRepository.findAndDecryptConnection(connectionId, masterPwd); + if (!connection) { + throw new HttpException( + { + message: Messages.CONNECTION_NOT_FOUND, + }, + HttpStatus.BAD_REQUEST, + ); + } + + if (connection.is_frozen) { + throw new NonAvailableInFreePlanException(Messages.CONNECTION_IS_FROZEN); + } + + try { + const dao = getDataAccessObject(connection); + const tablesInConnection = await dao.getTablesFromDB(); + const tableNames = tablesInConnection.map((table) => table.tableName); + + if (!tableNames.includes(tableName)) { + throw new BadRequestException(Messages.TABLE_NOT_FOUND); + } + + let userEmail: string; + if (isConnectionTypeAgent(connection.type)) { + userEmail = await this._dbContext.userRepository.getUserEmailOrReturnNull(userId); + } + + // eslint-disable-next-line prefer-const + let { tableSettings, tableCustomFields, tableWidgets } = + await this._dbContext.tableSettingsRepository.findTableCustoms(connectionId, tableName); + + /* eslint-disable */ + let [ + tablePrimaryColumns, + tableForeignKeys, + tableStructure, + userTablePermissions, + customActionEvents, + savedTableFilters, + personalTableSettings, + /* eslint-enable */ + ] = await Promise.all([ + dao.getTablePrimaryColumns(tableName, userEmail), + dao.getTableForeignKeys(tableName, userEmail), + dao.getTableStructure(tableName, userEmail), + this._dbContext.userAccessRepository.getUserTablePermissions(userId, connectionId, tableName, masterPwd), + this._dbContext.actionEventsRepository.findCustomEventsForTable(connectionId, tableName), + this._dbContext.tableFiltersRepository.findTableFiltersForTableInConnection(tableName, connectionId), + this._dbContext.personalTableSettingsRepository.findUserTableSettings(userId, connectionId, tableName), + ]); + const filteringFields: Array = isObjectEmpty(filters) + ? findFilteringFieldsUtil(query, tableStructure) + : parseFilteringFieldsFromBodyData(filters, tableStructure); + + const orderingField = findOrderingFieldUtil(query, tableStructure, tableSettings); + + const configured = !!tableSettings; + + const allowCsvExport = tableSettings?.allow_csv_export ?? true; + const allowCsvImport = tableSettings?.allow_csv_import ?? true; + const can_delete = tableSettings?.can_delete ?? true; + const can_update = tableSettings?.can_update ?? true; + const can_add = tableSettings?.can_add ?? true; + + //todo rework in daos + tableSettings = tableSettings ? tableSettings : ({} as TableSettingsEntity); + personalTableSettings = personalTableSettings ? personalTableSettings : ({} as PersonalTableSettingsEntity); + + const { autocomplete, referencedColumn } = query; + + const autocompleteFields = + autocomplete && referencedColumn + ? findAutocompleteFieldsUtil(query, tableStructure, tableSettings, referencedColumn) + : undefined; + + const builtDAOsTableSettings = buildDAOsTableSettingsDs(tableSettings, personalTableSettings); + if (orderingField) { + builtDAOsTableSettings.ordering_field = orderingField.field; + builtDAOsTableSettings.ordering = orderingField.value; + } + if ( + isHexString(searchingFieldValue) && + (tableStructure.some((field) => isBinary(field.data_type)) || + connection.type === ConnectionTypesEnum.mongodb || + connection.type === ConnectionTypesEnum.agent_mongodb) + ) { + searchingFieldValue = hexToBinary(searchingFieldValue) as any; + builtDAOsTableSettings.search_fields = tableStructure + .filter((field) => isBinary(field.data_type)) + .map((field) => field.column_name); + if (connection.type === 'mongodb' || connection.type === 'agent_mongodb') { + builtDAOsTableSettings.search_fields.push('_id'); + } + } + + let rows: FoundRowsDS; + try { + rows = await dao.getRowsFromTable( + tableName, + builtDAOsTableSettings, + page, + perPage, + searchingFieldValue, + filteringFields, + autocompleteFields, + tableStructure, + userEmail, + ); + } catch (e) { + Sentry.captureException(e); + throw new UnknownSQLException(e.message, ExceptionOperations.FAILED_TO_GET_ROWS_FROM_TABLE); + } + rows = processRowsUtil(rows, tableWidgets, tableStructure, tableCustomFields); + + const foreignKeysFromWidgets: Array = tableWidgets + .filter((widget) => widget.widget_type === WidgetTypeEnum.Foreign_key) + .reduce>((acc, widget) => { + if (widget.widget_params) { + try { + const widgetParams = JSON5.parse(widget.widget_params) as ForeignKeyDSInfo; + acc.push(widgetParams); + } catch (_e) { + // + } + } + return acc; + }, []); + + tableForeignKeys = [...tableForeignKeys, ...foreignKeysFromWidgets]; + + const canUserReadForeignTables = await Promise.all( + tableForeignKeys.map((foreignKey) => + this._dbContext.userAccessRepository + .improvedCheckTableRead(userId, connectionId, foreignKey.referenced_table_name, masterPwd) + .then((canRead) => ({ + tableName: foreignKey.referenced_table_name, + canRead, + })), + ), + ); + + const readableForeignTables = new Set( + canUserReadForeignTables.filter(({ canRead }) => canRead).map(({ tableName }) => tableName), + ); + + tableForeignKeys = tableForeignKeys.filter(({ referenced_table_name }) => + readableForeignTables.has(referenced_table_name), + ); + + if (tableForeignKeys && tableForeignKeys.length > 0) { + tableForeignKeys = await Promise.all( + tableForeignKeys.map((el) => this.attachForeignColumnNames(el, userId, connectionId, dao).catch(() => el)), + ); + } + + const formedTableStructure = formFullTableStructure(tableStructure, tableSettings); + + const largeDataset = rows.large_dataset || rows.pagination.total > Constants.LARGE_DATASET_ROW_LIMIT; + + const listFields = findAvailableFields(builtDAOsTableSettings, tableStructure); + const actionEventsDtos = customActionEvents.map((el) => buildActionEventDto(el)); + const savedFiltersRO = savedTableFilters.map((el) => buildCreatedTableFilterRO(el)); + + const rowsRO = { + rows: rows.data, + primaryColumns: tablePrimaryColumns, + pagination: rows.pagination, + sortable_by: builtDAOsTableSettings?.sortable_by?.length > 0 ? builtDAOsTableSettings.sortable_by : [], + ordering_field: personalTableSettings.ordering_field ? personalTableSettings.ordering_field : undefined, + ordering: personalTableSettings.ordering ? personalTableSettings.ordering : undefined, + columns_view: builtDAOsTableSettings.columns_view ? builtDAOsTableSettings.columns_view : undefined, + structure: formedTableStructure, + foreignKeys: tableForeignKeys, + configured: configured, + widgets: tableWidgets, + identity_column: builtDAOsTableSettings.identity_column ? builtDAOsTableSettings.identity_column : null, + table_permissions: userTablePermissions, + list_fields: listFields, + action_events: actionEventsDtos, + table_actions: actionEventsDtos, + large_dataset: largeDataset, + allow_csv_export: allowCsvExport, + allow_csv_import: allowCsvImport, + saved_filters: savedFiltersRO, + can_delete: can_delete, + can_update: can_update, + can_add: can_add, + table_settings: { + sortable_by: builtDAOsTableSettings?.sortable_by?.length > 0 ? builtDAOsTableSettings.sortable_by : [], + ordering: personalTableSettings.ordering ? personalTableSettings.ordering : undefined, + identity_column: builtDAOsTableSettings.identity_column ? builtDAOsTableSettings.identity_column : null, + list_fields: personalTableSettings?.list_fields?.length > 0 ? personalTableSettings.list_fields : [], + allow_csv_export: allowCsvExport, + allow_csv_import: allowCsvImport, + can_delete: can_delete, + can_update: can_update, + can_add: can_add, + columns_view: personalTableSettings?.columns_view ? personalTableSettings.columns_view : [], + }, + }; + + const identitiesMap = new Map(); + + if (tableForeignKeys?.length > 0) { + const uniqueReferencedTables = [...new Set(tableForeignKeys.map((fk) => fk.referenced_table_name))]; + const foreignTableSettingsPromises = uniqueReferencedTables.map((tableName) => + this._dbContext.tableSettingsRepository.findTableSettings(connectionId, tableName).then((settings) => ({ + tableName, + settings, + })), + ); + const foreignTableSettingsResults = await Promise.all(foreignTableSettingsPromises); + const foreignTableSettingsMap = new Map( + foreignTableSettingsResults.map((result) => [result.tableName, result.settings]), + ); + + const foreignKeyDataMap = new Map }>(); + + for (const foreignKey of tableForeignKeys) { + const valuesSet = new Set(); + for (const row of rowsRO.rows) { + const value = row[foreignKey.column_name]; + if (value !== undefined && value !== null) { + valuesSet.add(value as string | number); + } + } + + const key = `${foreignKey.referenced_table_name}:${foreignKey.referenced_column_name}`; + if (foreignKeyDataMap.has(key)) { + const existing = foreignKeyDataMap.get(key); + valuesSet.forEach((v) => existing!.values.add(v)); + } else { + foreignKeyDataMap.set(key, { foreignKey, values: valuesSet }); + } + } + + const identityPromises = Array.from(foreignKeyDataMap.values()).map(async ({ foreignKey, values }) => { + const foreignTableSettings = foreignTableSettingsMap.get(foreignKey.referenced_table_name); + const builtDAOsForeignTableSettings = buildDAOsTableSettingsDs(foreignTableSettings, {} as any); + const identityColumns = await this.getBatchedIdentityColumns( + Array.from(values), + foreignKey, + dao, + builtDAOsForeignTableSettings, + userEmail, + ); + return { foreignKey, identityColumns }; + }); + + const identityResults = await Promise.all(identityPromises); + + for (const { foreignKey, identityColumns } of identityResults) { + if (!identitiesMap.has(foreignKey.referenced_table_name)) { + identitiesMap.set(foreignKey.referenced_table_name, []); + } + identitiesMap.get(foreignKey.referenced_table_name)?.push(...identityColumns); + } + } + + const identities = Array.from(identitiesMap, ([referenced_table_name, identity_columns]) => ({ + referenced_table_name, + identity_columns, + })); + + const foreignKeysConformity = tableForeignKeys.map((key) => ({ + currentFKeyName: key.column_name, + realFKeyName: key.referenced_column_name, + referencedTableName: key.referenced_table_name, + })); + + const identityLookupMaps = new Map>(); + for (const element of foreignKeysConformity) { + const identityForCurrentTable = identities.find( + (el) => el.referenced_table_name === element.referencedTableName, + ); + + if (!identityForCurrentTable) continue; + + const lookupKey = `${element.referencedTableName}:${element.realFKeyName}`; + if (!identityLookupMaps.has(lookupKey)) { + const identityColumnsMap = new Map( + identityForCurrentTable.identity_columns.map((col) => [col[element.realFKeyName], col]), + ); + identityLookupMaps.set(lookupKey, identityColumnsMap); + } + } + + for (const row of rowsRO.rows) { + for (const element of foreignKeysConformity) { + const lookupKey = `${element.referencedTableName}:${element.realFKeyName}`; + const identityColumnsMap = identityLookupMaps.get(lookupKey); + + if (!identityColumnsMap) continue; + + const identityForCurrentValue = identityColumnsMap.get(row[element.currentFKeyName]); + row[element.currentFKeyName] = + typeof identityForCurrentValue === 'object' && identityForCurrentValue !== null + ? identityForCurrentValue + : {}; + } + } + + operationResult = OperationResultStatusEnum.successfully; + return rowsRO; + } catch (e) { + Sentry.captureException(e); + operationResult = OperationResultStatusEnum.unsuccessfully; + if (e instanceof HttpException) { + throw e; + } + throw new HttpException( + { + message: `${Messages.FAILED_GET_TABLE_ROWS} ${Messages.ERROR_MESSAGE} ${e.message} ${Messages.TRY_AGAIN_LATER}`, - originalMessage: e.originalMessage ? `${Messages.ERROR_MESSAGE_ORIGINAL} ${e.originalMessage}` : undefined, - }, - HttpStatus.INTERNAL_SERVER_ERROR, - ); - } finally { - const logRecord = { - table_name: tableName, - userId: userId, - connection: connection, - operationType: LogOperationTypeEnum.rowsReceived, - operationStatusResult: operationResult, - }; - await this.tableLogsService.crateAndSaveNewLogUtil(logRecord); - const isTest = await this._dbContext.connectionRepository.isTestConnectionById(connectionId); - await this.amplitudeService.formAndSendLogRecord( - isTest ? AmplitudeEventTypeEnum.tableRowsReceivedTest : AmplitudeEventTypeEnum.tableRowsReceived, - userId, - ); - } - } - - private async attachForeignColumnNames( - foreignKey: ForeignKeyDS, - userId: string, - connectionId: string, - dao: IDataAccessObject | IDataAccessObjectAgent, - ): Promise { - try { - const [foreignTableSettings, foreignTableStructure] = await Promise.all([ - this._dbContext.tableSettingsRepository.findTableSettingsPure(connectionId, foreignKey.referenced_table_name), - dao.getTableStructure(foreignKey.referenced_table_name, userId), - ]); - - const columnNames = foreignTableStructure - .map((el) => el.column_name) - .filter((el) => foreignTableSettings?.autocomplete_columns.includes(el)); - - return { - ...foreignKey, - autocomplete_columns: columnNames, - }; - } catch (_e) { - return { - ...foreignKey, - autocomplete_columns: [], - }; - } - } - - private chunkArray(array: T[], chunkSize: number): T[][] { - const results = []; - for (let i = 0; i < array.length; i += chunkSize) { - results.push(array.slice(i, i + chunkSize)); - } - return results; - } - - private async getBatchedIdentityColumns( - foreignKeysValuesCollection: Array, - foreignKey: ForeignKeyDS, - dao: IDataAccessObject | IDataAccessObjectAgent, - foreignTableSettings: TableSettingsDS, - userEmail: string, - ): Promise>> { - const uniqueValues = [...new Set(foreignKeysValuesCollection)]; - - if (uniqueValues.length === 0) { - return []; - } - - const batchSize = 50; - - if (uniqueValues.length <= batchSize) { - return await dao.getIdentityColumns( - foreignKey.referenced_table_name, - foreignKey.referenced_column_name, - foreignTableSettings?.identity_column, - uniqueValues, - userEmail, - ); - } - - const chunkedValues = this.chunkArray(uniqueValues, batchSize); - const results = await Promise.all( - chunkedValues.map((chunk) => - dao.getIdentityColumns( - foreignKey.referenced_table_name, - foreignKey.referenced_column_name, - foreignTableSettings?.identity_column, - chunk, - userEmail, - ), - ), - ); - - return results.flat(); - } + originalMessage: e.originalMessage ? `${Messages.ERROR_MESSAGE_ORIGINAL} ${e.originalMessage}` : undefined, + }, + HttpStatus.INTERNAL_SERVER_ERROR, + ); + } finally { + const logRecord = { + table_name: tableName, + userId: userId, + connection: connection, + operationType: LogOperationTypeEnum.rowsReceived, + operationStatusResult: operationResult, + }; + await this.tableLogsService.crateAndSaveNewLogUtil(logRecord); + const isTest = await this._dbContext.connectionRepository.isTestConnectionById(connectionId); + await this.amplitudeService.formAndSendLogRecord( + isTest ? AmplitudeEventTypeEnum.tableRowsReceivedTest : AmplitudeEventTypeEnum.tableRowsReceived, + userId, + ); + } + } + + private async attachForeignColumnNames( + foreignKey: ForeignKeyDS, + userId: string, + connectionId: string, + dao: IDataAccessObject | IDataAccessObjectAgent, + ): Promise { + try { + const [foreignTableSettings, foreignTableStructure] = await Promise.all([ + this._dbContext.tableSettingsRepository.findTableSettingsPure(connectionId, foreignKey.referenced_table_name), + dao.getTableStructure(foreignKey.referenced_table_name, userId), + ]); + + const columnNames = foreignTableStructure + .map((el) => el.column_name) + .filter((el) => foreignTableSettings?.autocomplete_columns.includes(el)); + + return { + ...foreignKey, + autocomplete_columns: columnNames, + }; + } catch (_e) { + return { + ...foreignKey, + autocomplete_columns: [], + }; + } + } + + private chunkArray(array: T[], chunkSize: number): T[][] { + const results = []; + for (let i = 0; i < array.length; i += chunkSize) { + results.push(array.slice(i, i + chunkSize)); + } + return results; + } + + private async getBatchedIdentityColumns( + foreignKeysValuesCollection: Array, + foreignKey: ForeignKeyDS, + dao: IDataAccessObject | IDataAccessObjectAgent, + foreignTableSettings: TableSettingsDS, + userEmail: string, + ): Promise>> { + const uniqueValues = [...new Set(foreignKeysValuesCollection)]; + + if (uniqueValues.length === 0) { + return []; + } + + const batchSize = 50; + + if (uniqueValues.length <= batchSize) { + return await dao.getIdentityColumns( + foreignKey.referenced_table_name, + foreignKey.referenced_column_name, + foreignTableSettings?.identity_column, + uniqueValues, + userEmail, + ); + } + + const chunkedValues = this.chunkArray(uniqueValues, batchSize); + const results = await Promise.all( + chunkedValues.map((chunk) => + dao.getIdentityColumns( + foreignKey.referenced_table_name, + foreignKey.referenced_column_name, + foreignTableSettings?.identity_column, + chunk, + userEmail, + ), + ), + ); + + return results.flat(); + } }