From bf12cade5feec71e2acea552fc583d852ff574cc Mon Sep 17 00:00:00 2001 From: SaibazKhan Date: Tue, 23 Jun 2026 13:23:33 +0530 Subject: [PATCH 1/2] remove computed field from instruction add one remove field command fix many-to-many on chatter log --- .../migrate-removed-fields.command.ts | 60 +++ .../seed-data/solid-core-metadata.json | 140 +++++-- ...-entity-computed-field-provider.service.ts | 16 +- src/services/crud.service.ts | 27 ++ src/services/import-transaction.service.ts | 5 + .../removed-field-migration.service.ts | 386 ++++++++++++++++++ src/solid-core.module.ts | 4 + src/subscribers/audit.subscriber.ts | 64 ++- 8 files changed, 665 insertions(+), 37 deletions(-) create mode 100644 src/commands/migrate-removed-fields.command.ts create mode 100644 src/services/removed-field-migration.service.ts diff --git a/src/commands/migrate-removed-fields.command.ts b/src/commands/migrate-removed-fields.command.ts new file mode 100644 index 00000000..d31185e1 --- /dev/null +++ b/src/commands/migrate-removed-fields.command.ts @@ -0,0 +1,60 @@ +import { Logger } from "@nestjs/common"; +import { Command, CommandRunner, Option } from "nest-commander"; +import { RemovedFieldMigrationService } from "src/services/removed-field-migration.service"; +import { CommandError } from "./helper"; + +interface CommandOptions { + name: string; + dryRun?: boolean; +} + +@Command({ + name: "migrate-removed-fields", + description: "Drops live database artifacts for fields marked for removal and cleans the related metadata.", +}) +export class MigrateRemovedFieldsCommand extends CommandRunner { + constructor( + private readonly removedFieldMigrationService: RemovedFieldMigrationService, + ) { + super(); + } + + private readonly logger = new Logger(MigrateRemovedFieldsCommand.name); + + async run(_passedParam: string[], options?: CommandOptions): Promise { + const errors = this.validate(options); + if (errors.length) { + errors.forEach((error) => this.logger.error(error)); + return; + } + + const dryRun = options?.dryRun ?? true; + const result = await this.removedFieldMigrationService.migrateMarkedFields(options.name, dryRun); + result.operations.forEach((operation) => this.logger.log(operation)); + this.logger.log(`Processed ${result.removedFieldNames.length} field(s) for model "${result.modelName}".`); + } + + @Option({ + flags: "-n, --name ", + description: "Model name (singularName) from the ss_model_metadata table", + }) + parseName(val: string): string { + return val; + } + + @Option({ + flags: "-d, --dryRun [dry run]", + description: "Dry run the command", + }) + parseDryRun(val: string): boolean { + this.logger.debug(`Dry run : ${val}`); + return val === "false" ? false : true; + } + + private validate(options: CommandOptions): CommandError[] { + if (!options?.name) { + return [new CommandError("Model Name is required")]; + } + return []; + } +} diff --git a/src/seeders/seed-data/solid-core-metadata.json b/src/seeders/seed-data/solid-core-metadata.json index e92ca711..9d28f62a 100755 --- a/src/seeders/seed-data/solid-core-metadata.json +++ b/src/seeders/seed-data/solid-core-metadata.json @@ -8987,12 +8987,15 @@ 48 ], "enableGlobalSearch": true, - "create": true, - "edit": true, + "create": false, + "edit": false, "delete": true, + "import": false, + "export": false, "allowedViews": [ "list", - "card" + "card", + "tree" ] }, "children": [ @@ -9253,7 +9256,13 @@ "enableGlobalSearch": true, "create": false, "edit": false, - "delete": false + "delete": false, + "import": false, + "export": false, + "allowedViews": [ + "list", + "tree" + ] }, "children": [ { @@ -10345,15 +10354,11 @@ "swimlanesCount": 5, "recordsInSwimlane": 10, "enableGlobalSearch": true, - "create": true, - "edit": true, - "delete": true, + "create": false, + "edit": false, + "delete": false, "groupBy": "stage", - "draggable": false, - "allowedViews": [ - "list", - "kanban" - ] + "draggable": false }, "children": [ { @@ -12179,7 +12184,11 @@ "edit": false, "delete": false, "import": false, - "export": false + "export": false, + "allowedViews": [ + "list", + "tree" + ] }, "children": [ { @@ -13991,7 +14000,13 @@ "enableGlobalSearch": true, "create": false, "edit": true, - "delete": false + "delete": false, + "import": false, + "export": false, + "allowedViews": [ + "list", + "tree" + ] }, "children": [ { @@ -14066,7 +14081,13 @@ "enableGlobalSearch": true, "create": false, "edit": false, - "delete": false + "delete": false, + "import": false, + "export": false, + "allowedViews": [ + "list", + "tree" + ] }, "children": [ { @@ -14150,7 +14171,13 @@ "enableGlobalSearch": true, "create": false, "edit": false, - "delete": false + "delete": false, + "import": false, + "export": false, + "allowedViews": [ + "list", + "tree" + ] }, "children": [ { @@ -14212,7 +14239,13 @@ "enableGlobalSearch": true, "create": false, "edit": false, - "delete": false + "delete": false, + "import": false, + "export": false, + "allowedViews": [ + "list", + "tree" + ] }, "children": [ { @@ -14281,7 +14314,13 @@ "enableGlobalSearch": true, "create": false, "edit": false, - "delete": false + "delete": true, + "import": false, + "export": false, + "allowedViews": [ + "list", + "tree" + ] }, "children": [ { @@ -14351,7 +14390,14 @@ "enableGlobalSearch": true, "create": false, "edit": false, - "delete": false + "delete": true, + "import": false, + "export": false, + "allowedViews": [ + "list", + "card", + "tree" + ] }, "children": [ { @@ -14417,11 +14463,14 @@ 50 ], "enableGlobalSearch": true, - "create": false, - "edit": false, - "delete": false, - "import": false, - "export": false + "create": true, + "edit": true, + "delete": true, + "import": true, + "export": true, + "allowedViews": [ + "list" + ] }, "children": [ { @@ -14456,7 +14505,11 @@ "edit": false, "delete": false, "import": false, - "export": false + "export": false, + "allowedViews": [ + "list", + "tree" + ] }, "children": [ { @@ -14518,7 +14571,11 @@ "edit": false, "delete": false, "import": false, - "export": false + "export": false, + "allowedViews": [ + "list", + "tree" + ] }, "children": [ { @@ -14587,7 +14644,11 @@ "edit": true, "delete": true, "import": true, - "export": true + "export": true, + "allowedViews": [ + "list", + "tree" + ] }, "children": [ { @@ -14647,7 +14708,11 @@ "edit": false, "delete": false, "import": false, - "export": false + "export": false, + "allowedViews": [ + "list", + "tree" + ] }, "children": [ { @@ -14707,7 +14772,11 @@ "edit": false, "delete": false, "import": false, - "export": false + "export": false, + "allowedViews": [ + "list", + "tree" + ] }, "children": [ { @@ -14777,7 +14846,11 @@ "edit": false, "delete": false, "import": false, - "export": false + "export": false, + "allowedViews": [ + "list", + "tree" + ] }, "children": [ { @@ -14846,7 +14919,12 @@ "enableGlobalSearch": true, "create": true, "edit": true, - "delete": true + "delete": true, + "import": true, + "export": true, + "allowedViews": [ + "list" + ] }, "children": [ { diff --git a/src/services/computed-fields/entity/concat-entity-computed-field-provider.service.ts b/src/services/computed-fields/entity/concat-entity-computed-field-provider.service.ts index fae2fbdf..f43b42e4 100644 --- a/src/services/computed-fields/entity/concat-entity-computed-field-provider.service.ts +++ b/src/services/computed-fields/entity/concat-entity-computed-field-provider.service.ts @@ -1,21 +1,29 @@ +import dayjs from "dayjs"; import { Injectable } from "@nestjs/common"; import { kebabCase, get } from "lodash"; import { ComputedFieldProvider } from "src/decorators/computed-field-provider.decorator"; import { CommonEntity } from "src/entities/common.entity"; import { ComputedFieldMetadata } from "src/helpers/solid-registry"; import { IEntityPreComputeFieldProvider } from "src/interfaces"; +import { SettingService } from "src/services/setting.service"; +import type { SolidCoreSetting } from "src/services/settings/default-settings-provider.service"; export interface ConcatComputedFieldContext { separator: string; // The separator to use between concatenated values fields: string[]; // The fields to concatenate slugify?: boolean; // Optional: if true, slugify each field value before concatenation + dateFormat?: string; // Optional: format to use for Date values. Defaults to the configured dateFormat setting } @ComputedFieldProvider() @Injectable() export class ConcatEntityComputedFieldProvider implements IEntityPreComputeFieldProvider { + constructor( + private readonly settingService: SettingService, + ) { } + name(): string { return "ConcatEntityComputedFieldProvider"; } @@ -29,6 +37,8 @@ export class ConcatEntityComputedFieldProvider implement const separator = computedFieldValueProviderCtxt.separator ?? ' '; const fields: string[] = computedFieldValueProviderCtxt.fields ?? []; const slugify = computedFieldValueProviderCtxt.slugify ?? false; + const dateFormat = computedFieldValueProviderCtxt.dateFormat + ?? this.settingService.getConfigValue("dateFormat"); const parts: string[] = []; @@ -38,6 +48,10 @@ export class ConcatEntityComputedFieldProvider implement // normalize to string (skip null/undefined) if (val == null) continue; + if (val instanceof Date) { + val = dayjs(val).format(dateFormat); + } + if (typeof val !== 'string') { val = String(val); } @@ -58,4 +72,4 @@ export class ConcatEntityComputedFieldProvider implement (triggerEntity as any)[computedFieldMetadata.fieldName] = concatenatedString; } -} \ No newline at end of file +} diff --git a/src/services/crud.service.ts b/src/services/crud.service.ts index a2f074ce..8a7a8bc8 100755 --- a/src/services/crud.service.ts +++ b/src/services/crud.service.ts @@ -34,6 +34,7 @@ import { SelectionDynamicFieldCrudManager } from "../helpers/field-crud-managers import { SelectionStaticFieldCrudManager } from "../helpers/field-crud-managers/SelectionStaticFieldCrudManager"; import { ShortTextFieldCrudManager } from "../helpers/field-crud-managers/ShortTextFieldCrudManager"; import { UUIDFieldCrudManager } from "../helpers/field-crud-managers/UUIDFieldCrudManager"; +import { ModelMetadataHelperService } from "src/helpers/model-metadata-helper.service"; import { FieldCrudManager, MediaWithFullUrl } from "../interfaces"; import { CrudHelperService, FilterCombinator, UserIdFields } from "./crud-helper.service"; import { HashingService } from "./hashing.service"; @@ -42,6 +43,7 @@ import { getMediaStorageProvider } from "./mediaStorageProviders"; import { ModelMetadataService } from "./model-metadata.service"; import { RequestContextService } from "./request-context.service"; +const AUDIT_BEFORE_SNAPSHOT = '__auditBeforeSnapshot'; export class CRUDService { // Add two generic value i.e Person,CreatePersonDto, so we get the proper types in our service @@ -213,6 +215,31 @@ export class CRUDService { // Add two generic value i.e ); } + const modelMetadataHelperService = this.moduleRef.get(ModelMetadataHelperService, { strict: false }); + const auditRelationFields = (await modelMetadataHelperService.loadFieldHierarchy(model.singularName)).filter(field => + field.enableAuditTracking && + field.type === 'relation' && + field.relationType !== 'one-to-many' + ); + if (auditRelationFields.length > 0) { + const relations: any = {}; + auditRelationFields.forEach(field => relations[field.name] = true); + const auditBeforeEntity = await this.repo.findOne({ + where: { + id: id, + } as unknown as FindOptionsWhere, + relations: relations as any, + }); + if (auditBeforeEntity) { + Object.defineProperty(entity, AUDIT_BEFORE_SNAPSHOT, { + configurable: true, + enumerable: false, + value: auditBeforeEntity, + writable: true, + }); + } + } + // // In some instances for legacy tables sometimes id is mapped as a bigint. // // in these cases the update method ends up attempting to insert records due to some type orm type mismatch issue. // const idFieldMetadata = model.fields.find(f => f.name === 'id'); diff --git a/src/services/import-transaction.service.ts b/src/services/import-transaction.service.ts index e110b79b..a113536f 100644 --- a/src/services/import-transaction.service.ts +++ b/src/services/import-transaction.service.ts @@ -194,6 +194,11 @@ export class ImportTransactionService extends CRUDService { // Iterate through the fields and populate the standard instructions for (const field of allFields) { + + // Skip Computed fields + if (field.type === SolidFieldType.computed) { + continue; + } // Skip system fields if (systemFieldNames.includes(field.name)) { continue; diff --git a/src/services/removed-field-migration.service.ts b/src/services/removed-field-migration.service.ts new file mode 100644 index 00000000..9e4ee734 --- /dev/null +++ b/src/services/removed-field-migration.service.ts @@ -0,0 +1,386 @@ +import * as fs from "fs/promises"; +import { BadRequestException, Injectable, Logger, NotFoundException } from "@nestjs/common"; +import { ModuleRef } from "@nestjs/core"; +import { getDataSourceToken } from "@nestjs/typeorm"; +import { kebabCase, snakeCase } from "lodash"; +import * as path from "path"; +import { ERROR_MESSAGES } from "src/constants/error-messages"; +import { RelationType, SolidFieldType } from "src/dtos/create-field-metadata.dto"; +import { FieldMetadata } from "src/entities/field-metadata.entity"; +import { ModelMetadata } from "src/entities/model-metadata.entity"; +import { ModuleMetadataHelperService } from "src/helpers/module-metadata-helper.service"; +import { classify } from "src/helpers/string.helper"; +import { FieldMetadataRepository } from "src/repository/field-metadata.repository"; +import { ModelMetadataRepository } from "src/repository/model-metadata.repository"; +import { DataSource, EntityMetadata, QueryRunner, Table } from "typeorm"; + +interface MetadataFileUpdate { + filePath: string; + originalContent: string; + updatedContent: string; +} + +export interface RemovedFieldMigrationResult { + dryRun: boolean; + modelName: string; + operations: string[]; + removedFieldNames: string[]; +} + +@Injectable() +export class RemovedFieldMigrationService { + constructor( + private readonly modelMetadataRepo: ModelMetadataRepository, + private readonly fieldMetadataRepo: FieldMetadataRepository, + private readonly moduleMetadataHelperService: ModuleMetadataHelperService, + private readonly moduleRef: ModuleRef, + ) { } + + private readonly logger = new Logger(RemovedFieldMigrationService.name); + + // Cleans fields marked for removal by updating schema state and metadata for a single model. + async migrateMarkedFields(modelUserKey: string, dryRun: boolean = false): Promise { + if (!modelUserKey) { + throw new BadRequestException("Model name is required"); + } + + const model = await this.modelMetadataRepo.findOne({ + where: { singularName: modelUserKey }, + relations: { fields: true, module: true }, + }); + + if (!model) { + throw new NotFoundException(ERROR_MESSAGES.MODEL_NOT_FOUND(modelUserKey)); + } + + const fieldsForRemoval = model.fields.filter((field) => field.isMarkedForRemoval); + const operations: string[] = []; + + if (fieldsForRemoval.length === 0) { + const message = `No fields marked for removal were found for model "${model.singularName}".`; + this.logger.log(message); + operations.push(message); + return { + dryRun, + modelName: model.singularName, + operations, + removedFieldNames: [], + }; + } + + const metadataFileUpdate = dryRun ? null : await this.prepareMetadataFileUpdate(model, fieldsForRemoval); + const dataSource = await this.resolveDataSource(model.dataSource); + const entityMetadata = this.resolveEntityMetadata(dataSource, model); + const queryRunner = dataSource.createQueryRunner(); + + try { + await queryRunner.connect(); + + if (!dryRun) { + await queryRunner.startTransaction(); + } + + for (const field of fieldsForRemoval) { + await this.cleanupMarkedField({ + field, + model, + entityMetadata, + queryRunner, + dryRun, + operations, + }); + } + + if (!dryRun) { + await queryRunner.commitTransaction(); + } + } catch (error) { + if (!dryRun && queryRunner.isTransactionActive) { + await queryRunner.rollbackTransaction(); + } + throw error; + } finally { + await queryRunner.release(); + } + + if (!dryRun && metadataFileUpdate) { + await fs.writeFile(metadataFileUpdate.filePath, metadataFileUpdate.updatedContent, "utf8"); + + try { + await this.fieldMetadataRepo.manager.transaction(async (manager) => { + await manager.remove(FieldMetadata, fieldsForRemoval); + }); + } catch (error) { + await fs.writeFile(metadataFileUpdate.filePath, metadataFileUpdate.originalContent, "utf8"); + throw error; + } + + operations.push(`Updated metadata file at ${metadataFileUpdate.filePath}`); + operations.push(`Removed ${fieldsForRemoval.length} field metadata record(s) from ss_field_metadata`); + } else if (dryRun) { + operations.push(`Would update module metadata for model "${model.singularName}" and remove ${fieldsForRemoval.length} field metadata record(s).`); + } + + return { + dryRun, + modelName: model.singularName, + operations, + removedFieldNames: fieldsForRemoval.map((field) => field.name), + }; + } + + private async cleanupMarkedField(params: { field: FieldMetadata; model: ModelMetadata; entityMetadata?: EntityMetadata; queryRunner: QueryRunner; dryRun: boolean; operations: string[]; }): Promise { + const { field, model, entityMetadata, queryRunner, dryRun, operations } = params; + const relationMetadata = entityMetadata?.relations.find((relation) => relation.propertyName === field.name); + const resolvedTableName = entityMetadata?.tableName || model.tableName; + + if (field.type !== SolidFieldType.relation) { + const columnCandidates = this.buildColumnCandidates(field, entityMetadata, relationMetadata); + await this.dropColumnsForField(resolvedTableName, field, columnCandidates, queryRunner, dryRun, operations); + return; + } + + if (field.relationType === RelationType.manyToOne) { + const columnCandidates = this.buildColumnCandidates(field, entityMetadata, relationMetadata); + await this.dropColumnsForField(resolvedTableName, field, columnCandidates, queryRunner, dryRun, operations); + return; + } + + if (field.relationType === RelationType.manyTomany) { + const joinTableName = relationMetadata?.junctionEntityMetadata?.tableName || field.relationJoinTableName; + const ownsJoinTable = relationMetadata?.isOwning || field.isRelationManyToManyOwner; + + if (!ownsJoinTable) { + operations.push(`No direct database cleanup required for inverse many-to-many field "${field.name}".`); + return; + } + + if (!joinTableName) { + operations.push(`Skipping join-table cleanup for "${field.name}" because no join table name could be resolved.`); + return; + } + + await this.dropJoinTable(joinTableName, field, queryRunner, dryRun, operations); + return; + } + + operations.push(`No direct database cleanup required for relation field "${field.name}" with type "${field.relationType}".`); + } + + private async dropColumnsForField(tableName: string, field: FieldMetadata, columnCandidates: string[], queryRunner: QueryRunner, dryRun: boolean, operations: string[],): Promise { + if (!tableName) { + operations.push(`Skipping field "${field.name}" because the model table name could not be resolved.`); + return; + } + + if (columnCandidates.length === 0) { + operations.push(`Skipping field "${field.name}" because no column candidates could be resolved.`); + return; + } + + const handledColumns = new Set(); + let droppedAnyColumn = false; + + for (const columnName of columnCandidates) { + if (!columnName || handledColumns.has(columnName)) { + continue; + } + + handledColumns.add(columnName); + const table = await this.loadTable(queryRunner, tableName); + const column = table?.columns.find((tableColumn) => tableColumn.name === columnName); + + if (!column) { + continue; + } + + droppedAnyColumn = true; + await this.dropColumnArtifacts(table, columnName, queryRunner, dryRun, operations); + } + + if (!droppedAnyColumn) { + operations.push(`No database column found for field "${field.name}" on table "${tableName}". Metadata cleanup will still proceed.`); + } + } + + private async dropColumnArtifacts(table: Table, columnName: string, queryRunner: QueryRunner, dryRun: boolean, operations: string[],): Promise { + for (const foreignKey of table.foreignKeys.filter((item) => item.columnNames.includes(columnName))) { + operations.push(`Drop foreign key "${foreignKey.name}" on "${table.name}.${columnName}"`); + if (!dryRun) { + await queryRunner.dropForeignKey(table, foreignKey); + } + } + + for (const uniqueConstraint of table.uniques.filter((item) => item.columnNames.includes(columnName))) { + operations.push(`Drop unique constraint "${uniqueConstraint.name}" on "${table.name}.${columnName}"`); + if (!dryRun) { + await queryRunner.dropUniqueConstraint(table, uniqueConstraint); + } + } + + for (const index of table.indices.filter((item) => item.columnNames.includes(columnName))) { + operations.push(`Drop index "${index.name}" on "${table.name}.${columnName}"`); + if (!dryRun) { + await queryRunner.dropIndex(table, index); + } + } + + operations.push(`Drop column "${table.name}.${columnName}"`); + if (!dryRun) { + await queryRunner.dropColumn(table, columnName); + } + } + + private async dropJoinTable(joinTableName: string, field: FieldMetadata, queryRunner: QueryRunner, dryRun: boolean, operations: string[]): Promise { + const table = await this.loadTable(queryRunner, joinTableName); + if (!table) { + operations.push(`Join table "${joinTableName}" for field "${field.name}" does not exist. Metadata cleanup will still proceed.`); + return; + } + + operations.push(`Drop join table "${joinTableName}" for field "${field.name}"`); + if (!dryRun) { + await queryRunner.dropTable(joinTableName); + } + } + + private buildColumnCandidates(field: FieldMetadata, entityMetadata?: EntityMetadata, relationMetadata?: EntityMetadata["relations"][number]): string[] { + const columnCandidates = new Set(); + + relationMetadata?.joinColumns?.forEach((column) => { + if (column.databaseName) { + columnCandidates.add(column.databaseName); + } + }); + + entityMetadata?.columns + .filter((column) => column.propertyName === field.name) + .forEach((column) => { + if (column.databaseName) { + columnCandidates.add(column.databaseName); + } + }); + + if (field.columnName) { + columnCandidates.add(field.columnName); + } + + if (field.relationCoModelColumnName) { + columnCandidates.add(field.relationCoModelColumnName); + } + + if (field.type === SolidFieldType.relation && field.relationType === RelationType.manyToOne) { + columnCandidates.add(`${snakeCase(field.name)}_id`); + } else { + columnCandidates.add(snakeCase(field.name)); + } + + return [...columnCandidates].filter(Boolean); + } + + private async prepareMetadataFileUpdate(model: ModelMetadata, fieldsForRemoval: FieldMetadata[]): Promise { + const filePath = await this.resolveMetadataFilePath(model.module.name); + const originalContent = await fs.readFile(filePath, "utf8"); + const metadata = JSON.parse(originalContent); + const models = metadata?.moduleMetadata?.models; + + if (!Array.isArray(models)) { + throw new BadRequestException(`Invalid module metadata structure in ${filePath}`); + } + + const existingModelIndex = models.findIndex((existingModel: any) => existingModel.singularName === model.singularName); + if (existingModelIndex === -1) { + throw new NotFoundException(`Model "${model.singularName}" not found in metadata file ${filePath}`); + } + + const fieldNamesForRemoval = new Set(fieldsForRemoval.map((field) => field.name)); + const existingModel = models[existingModelIndex]; + existingModel.fields = (existingModel.fields || []).filter((field: any) => !fieldNamesForRemoval.has(field.name)); + models[existingModelIndex] = existingModel; + + return { + filePath, + originalContent, + updatedContent: JSON.stringify(metadata, null, 2), + }; + } + + private async resolveDataSource(dataSourceName?: string): Promise { + const normalizedDataSourceName = dataSourceName && dataSourceName !== "default" ? dataSourceName : undefined; + const token = normalizedDataSourceName ? getDataSourceToken(normalizedDataSourceName) : getDataSourceToken(); + let dataSource: DataSource | undefined; + + try { + dataSource = this.moduleRef.get(token, { strict: false }); + } catch (error: any) { + throw new NotFoundException(`Datasource "${normalizedDataSourceName ?? "default"}" could not be resolved: ${error?.message ?? error}`); + } + + if (!dataSource) { + throw new NotFoundException(`Datasource "${normalizedDataSourceName ?? "default"}" could not be resolved.`); + } + + if (!dataSource.isInitialized) { + await dataSource.initialize(); + } + + return dataSource; + } + + private resolveEntityMetadata(dataSource: DataSource, model: ModelMetadata): EntityMetadata | undefined { + const candidates = [classify(model.singularName), model.singularName, model.tableName].filter(Boolean); + + for (const candidate of candidates) { + try { + return dataSource.getMetadata(candidate); + } catch { + // Try the next candidate. + } + } + + return dataSource.entityMetadatas.find((metadata) => metadata.tableName === model.tableName); + } + + private async resolveMetadataFilePath(moduleName: string): Promise { + const defaultPath = await this.moduleMetadataHelperService.getModuleMetadataFilePath(moduleName); + if (await this.fileExists(defaultPath)) { + return defaultPath; + } + + const dashModuleName = kebabCase(moduleName); + const moduleMetadataFilePath = path.resolve( + process.cwd(), + "module-metadata", + dashModuleName, + `${dashModuleName}-metadata.json`, + ); + + if (await this.fileExists(moduleMetadataFilePath)) { + return moduleMetadataFilePath; + } + + return defaultPath; + } + + private async fileExists(filePath: string): Promise { + try { + await fs.access(filePath); + return true; + } catch { + return false; + } + } + + private async loadTable(queryRunner: QueryRunner, tableName: string): Promise { + if (!tableName) { + return undefined; + } + + const hasTable = await queryRunner.hasTable(tableName); + if (!hasTable) { + return undefined; + } + + return queryRunner.getTable(tableName) ?? undefined; + } +} diff --git a/src/solid-core.module.ts b/src/solid-core.module.ts index 147583ab..d8a6a253 100644 --- a/src/solid-core.module.ts +++ b/src/solid-core.module.ts @@ -13,6 +13,7 @@ import { import { MulterModule } from "@nestjs/platform-express"; import { TypeOrmModule } from "@nestjs/typeorm"; import { RemoveFieldsCommand } from "./commands/remove-fields.command"; +import { MigrateRemovedFieldsCommand } from "./commands/migrate-removed-fields.command"; import { FieldMetadataController } from "./controllers/field-metadata.controller"; import { DashboardController } from "./controllers/dashboard.controller"; import { MediaStorageProviderMetadataController } from "./controllers/media-storage-provider-metadata.controller"; @@ -43,6 +44,7 @@ import { ListOfValuesService } from "./services/list-of-values.service"; import { MediaStorageProviderMetadataService } from "./services/media-storage-provider-metadata.service"; import { MediaService } from "./services/media.service"; import { ModelMetadataService } from "./services/model-metadata.service"; +import { RemovedFieldMigrationService } from "./services/removed-field-migration.service"; import { ModuleMetadataExplorerService } from "./services/module-metadata-explorer.service"; import { ModuleMetadataService } from "./services/module-metadata.service"; import { ModulePackageService } from "./services/module-package.service"; @@ -547,6 +549,7 @@ import { DashboardUserLayoutRepository } from './repositories/dashboard-user-lay FieldMetadataService, DashboardRuntimeService, RemoveFieldsCommand, + MigrateRemovedFieldsCommand, RefreshModelCommand, RefreshModuleCommand, InfoCommand, @@ -802,6 +805,7 @@ import { DashboardUserLayoutRepository } from './repositories/dashboard-user-lay UserViewMetadataRepository, ModelMetadataRepository, ModuleMetadataRepository, + RemovedFieldMigrationService, ActionMetadataRepository, MediaStorageProviderMetadataRepository, FixturesService, diff --git a/src/subscribers/audit.subscriber.ts b/src/subscribers/audit.subscriber.ts index 63e76bf1..ba70006b 100644 --- a/src/subscribers/audit.subscriber.ts +++ b/src/subscribers/audit.subscriber.ts @@ -1,12 +1,14 @@ import { Injectable, Logger, Scope } from '@nestjs/common'; +import { ModelMetadataHelperService } from 'src/helpers/model-metadata-helper.service'; import { lowerFirst } from 'src/helpers/string.helper'; import { SolidRegistry } from 'src/helpers/solid-registry'; import { DataSource, EntityMetadata, EntitySubscriberInterface, InsertEvent, RemoveEvent, UpdateEvent } from 'typeorm'; import { AuditQueuePayload } from 'src/interfaces'; import { RequestContextService } from 'src/services/request-context.service'; import { PublisherFactory } from 'src/services/queues/publisher-factory.service'; +const AUDIT_BEFORE_SNAPSHOT = '__auditBeforeSnapshot'; -@Injectable({scope: Scope.TRANSIENT}) +@Injectable({ scope: Scope.TRANSIENT }) export class AuditSubscriber implements EntitySubscriberInterface { private readonly logger = new Logger(AuditSubscriber.name); private dataSource: DataSource; @@ -14,6 +16,7 @@ export class AuditSubscriber implements EntitySubscriberInterface { private readonly publisherFactory: PublisherFactory, private readonly solidRegistry: SolidRegistry, private readonly requestContextService: RequestContextService, + private readonly modelMetadataHelperService: ModelMetadataHelperService, ) { } bindToDataSource(dataSource: DataSource) { @@ -53,16 +56,67 @@ export class AuditSubscriber implements EntitySubscriberInterface { async afterUpdate(event: UpdateEvent) { if (!this.shouldTrackAudit(event.metadata)) return; + const entityId = event.entity?.id ?? event.databaseEntity?.id ?? (event as any).entityId ?? null; + let before = event.databaseEntity ?? null; + let after = event.entity ?? null; + const updatedColumnNames = (event.updatedColumns ?? []).map(c => c.propertyName); + + const auditRelationFields = (await this.modelMetadataHelperService.loadFieldHierarchy(lowerFirst(event.metadata.name))).filter(field => + field.enableAuditTracking && + field.type === 'relation' && + field.relationType !== 'one-to-many' + ); + if (entityId && auditRelationFields.length > 0) { + const relations: any = {}; + auditRelationFields.forEach(field => relations[field.name] = true); + const relationBefore = event.entity?.[AUDIT_BEFORE_SNAPSHOT] ?? null; + const relationAfter = await event.manager.getRepository(event.metadata.target as any).findOne({ + where: { id: entityId } as any, + relations: relations as any, + }); + + if (relationBefore) { + before = relationBefore; + } + if (relationAfter) { + after = relationAfter; + } + + if (relationBefore && relationAfter) { + auditRelationFields.forEach(field => { + if (field.relationType === 'many-to-one') { + const oldId = relationBefore[field.name]?.id ?? null; + const newId = relationAfter[field.name]?.id ?? null; + if (oldId !== newId && !updatedColumnNames.includes(field.name)) { + updatedColumnNames.push(field.name); + } + } + else if (field.relationType === 'many-to-many') { + const oldIds = Array.isArray(relationBefore[field.name]) + ? relationBefore[field.name].map(item => item.id).sort() + : []; + + const newIds = Array.isArray(relationAfter[field.name]) + ? relationAfter[field.name].map(item => item.id).sort() + : []; + if ((oldIds.length !== newIds.length || JSON.stringify(oldIds) !== JSON.stringify(newIds)) && !updatedColumnNames.includes(field.name)) { + updatedColumnNames.push(field.name); + } + } + }); + } + } + this.enqueue(event, { eventType: 'update', modelName: event.metadata.name, - entityId: event.entity?.id ?? null, + entityId: entityId, occurredAt: new Date().toISOString(), - after: event.entity ?? null, + after: after, // databaseEntity is only populated when the entity was fetched first (save() path). // QueryBuilder update() leaves this undefined; postAuditMessageOnUpdate guards for it. - before: event.databaseEntity ?? null, - updatedColumnNames: (event.updatedColumns ?? []).map(c => c.propertyName), + before: before, + updatedColumnNames: updatedColumnNames, userId: this.activeUserId(), }); } From de3501b69eb02fa09c3ce80c47f1155b92a2def3 Mon Sep 17 00:00:00 2001 From: SaibazKhan Date: Thu, 25 Jun 2026 12:05:36 +0530 Subject: [PATCH 2/2] refactor and add migration command --- .../migrate-removed-fields.command.ts | 25 ++++- src/services/crud.service.ts | 70 +++++++------ src/services/model-metadata.service.ts | 25 ++++- .../removed-field-migration.service.ts | 54 +---------- src/subscribers/audit.subscriber.ts | 97 +++++++++++-------- 5 files changed, 136 insertions(+), 135 deletions(-) diff --git a/src/commands/migrate-removed-fields.command.ts b/src/commands/migrate-removed-fields.command.ts index d31185e1..1dbf784e 100644 --- a/src/commands/migrate-removed-fields.command.ts +++ b/src/commands/migrate-removed-fields.command.ts @@ -2,6 +2,7 @@ import { Logger } from "@nestjs/common"; import { Command, CommandRunner, Option } from "nest-commander"; import { RemovedFieldMigrationService } from "src/services/removed-field-migration.service"; import { CommandError } from "./helper"; +import { ModelMetadataService } from "src/services/model-metadata.service"; interface CommandOptions { name: string; @@ -15,6 +16,8 @@ interface CommandOptions { export class MigrateRemovedFieldsCommand extends CommandRunner { constructor( private readonly removedFieldMigrationService: RemovedFieldMigrationService, + private readonly modelMetadataService: ModelMetadataService, + ) { super(); } @@ -29,9 +32,27 @@ export class MigrateRemovedFieldsCommand extends CommandRunner { } const dryRun = options?.dryRun ?? true; - const result = await this.removedFieldMigrationService.migrateMarkedFields(options.name, dryRun); + + // STEP 1: Capture fields BEFORE migration deletes metadata + // const model = await this.modelMetadataService.findOneByUserKey( + // options.name, + // ["module", "fields"], + // ); + + // const fieldsForRemoval = model.fields.filter( + // field => field.isMarkedForRemoval, + // ); + + // // STEP 2: Run remove-fields schematic first + // if (!dryRun && fieldsForRemoval.length > 0) { + // // await this.modelMetadataService.executeRemoveFieldsOnly(options.name,fieldsForRemoval.map(f => f.name),false,); + // await this.modelMetadataService.executeRemoveFieldsWithModel(model, fieldsForRemoval.map(f => f.name),false,); + // } + + // STEP 3: Then perform DB + metadata cleanup + const result = await this.removedFieldMigrationService.migrateMarkedFields(options.name, dryRun,); + result.operations.forEach((operation) => this.logger.log(operation)); - this.logger.log(`Processed ${result.removedFieldNames.length} field(s) for model "${result.modelName}".`); } @Option({ diff --git a/src/services/crud.service.ts b/src/services/crud.service.ts index 8a7a8bc8..33d2cf84 100755 --- a/src/services/crud.service.ts +++ b/src/services/crud.service.ts @@ -187,7 +187,7 @@ export class CRUDService { // Add two generic value i.e })); } - //TODO: Will the updates be partial i.e PATCH or full i.e PUT + //TODO: Will the updates be partial i.e PATCH or full i.e PUT async update(id: number, updateDto: any, files: Express.Multer.File[] = [], isPartialUpdate: boolean = false, solidRequestContext: any = {}, isUpdate: boolean = false): Promise { if (!id) { throw new Error(ERROR_MESSAGES.ID_REQUIRED_FOR_UPDATE); @@ -215,41 +215,8 @@ export class CRUDService { // Add two generic value i.e ); } - const modelMetadataHelperService = this.moduleRef.get(ModelMetadataHelperService, { strict: false }); - const auditRelationFields = (await modelMetadataHelperService.loadFieldHierarchy(model.singularName)).filter(field => - field.enableAuditTracking && - field.type === 'relation' && - field.relationType !== 'one-to-many' - ); - if (auditRelationFields.length > 0) { - const relations: any = {}; - auditRelationFields.forEach(field => relations[field.name] = true); - const auditBeforeEntity = await this.repo.findOne({ - where: { - id: id, - } as unknown as FindOptionsWhere, - relations: relations as any, - }); - if (auditBeforeEntity) { - Object.defineProperty(entity, AUDIT_BEFORE_SNAPSHOT, { - configurable: true, - enumerable: false, - value: auditBeforeEntity, - writable: true, - }); - } - } - - // // In some instances for legacy tables sometimes id is mapped as a bigint. - // // in these cases the update method ends up attempting to insert records due to some type orm type mismatch issue. - // const idFieldMetadata = model.fields.find(f => f.name === 'id'); - // updateDto.id = idFieldMetadata?.type === 'bigint' ? BigInt(id) : id; - - // This class will be extended by the generated service class i.e PersonService - // The data required to identify the model and module name will be passed from the generate CrudService subclass - //TODO: Algorithm to create the entity - // 1. Fire a query and load all the fields in the provided model name for a particular module - // FIXME This can be optimized to take in module name i.e (handle scenario wherein same model exists in multiple modules) + // Capture pre-update many-to-many relation state for audit tracking + await this.prepareManyToManyAuditSnapshot(entity,id,model.singularName); let hasMediaFields = false; const fieldsToProcess = [...model.fields]; @@ -276,6 +243,37 @@ export class CRUDService { // Add two generic value i.e return savedEntity; } +/** + * Captures the current state of audit-enabled many-to-many relations + * before the entity is updated. The snapshot is attached to the entity + * and can be used later for audit comparison. + */ +private async prepareManyToManyAuditSnapshot(entity: T,id: number,modelSingularName: string): Promise { + const modelMetadataHelperService = this.moduleRef.get(ModelMetadataHelperService, { strict: false }); + const auditRelationFields = (await modelMetadataHelperService.loadFieldHierarchy(modelSingularName)).filter(field => + field.enableAuditTracking && + field.type === 'relation' && + field.relationType !== 'one-to-many' + ); + if (auditRelationFields.length > 0) { + const relations: any = {}; + auditRelationFields.forEach(field => relations[field.name] = true); + const auditBeforeEntity = await this.repo.findOne({ + where: { + id: id, + } as unknown as FindOptionsWhere, + relations: relations as any, + }); + if (auditBeforeEntity) { + Object.defineProperty(entity, AUDIT_BEFORE_SNAPSHOT, { + configurable: true, + enumerable: false, + value: auditBeforeEntity, + writable: true, + }); + } + } + } //TODO: Will the updates be partial i.e PATCH or full i.e PUT async delete(id: number, solidRequestContext: any = {}) { if (!id) { diff --git a/src/services/model-metadata.service.ts b/src/services/model-metadata.service.ts index 055a7bae..2277c2a8 100755 --- a/src/services/model-metadata.service.ts +++ b/src/services/model-metadata.service.ts @@ -1478,7 +1478,7 @@ export class ModelMetadataService { //Filter out the fields by id const fieldsForRemoval = model.fields.filter((field) => options.fieldIdsForRemoval.includes(+field.id)); const removeOutput = await this.executeRemoveFieldsCommand(model, fieldsForRemoval, options.dryRun); - + const migrationOutput = await this.executeMigrationRemoveFieldsCommand(model, fieldsForRemoval, options.dryRun); // Remove the fields from the database as well. This also checks, if the field is marked for removal for (const field of fieldsForRemoval) { if (field.isMarkedForRemoval) { @@ -1515,6 +1515,13 @@ export class ModelMetadataService { return removeOutput; } + // model-metadata.service.ts mein naya method add karo + // async executeRemoveFieldsWithModel(model: ModelMetadata, fieldNames: string[], dryRun = false,) { + // const fieldsForRemoval = model.fields.filter( + // field => fieldNames.includes(field.name) + // ); + // return this.executeRemoveFieldsCommand(model, fieldsForRemoval, dryRun); + // } async generateModelCode(options: CodeGenerationOptions): Promise { if (!options.modelId && !options.modelUserKey) { @@ -1563,6 +1570,22 @@ export class ModelMetadataService { return output; } + private async executeMigrationRemoveFieldsCommand(model: ModelMetadata, fieldsForRemoval: FieldMetadata[], dryRun: boolean = false,): Promise { + if (!fieldsForRemoval || fieldsForRemoval.length === 0) { + return ""; + } + + const output = await this.commandService.executeCommandWithArgs({ + command: 'npx', + args: ['@solidxai/solidctl@latest', 'migration', '-n', model.singularName, 'remove-field', ...(dryRun ? [] : ['--apply'])], + cwd: path.join(process.cwd(), '..'), + }); + + this.logger.debug(`Schematic output : ${output}`); + return output; + } + + async updateUserKey(data: any) { const { modelName, fieldName } = data; diff --git a/src/services/removed-field-migration.service.ts b/src/services/removed-field-migration.service.ts index 9e4ee734..9f305177 100644 --- a/src/services/removed-field-migration.service.ts +++ b/src/services/removed-field-migration.service.ts @@ -14,12 +14,6 @@ import { FieldMetadataRepository } from "src/repository/field-metadata.repositor import { ModelMetadataRepository } from "src/repository/model-metadata.repository"; import { DataSource, EntityMetadata, QueryRunner, Table } from "typeorm"; -interface MetadataFileUpdate { - filePath: string; - originalContent: string; - updatedContent: string; -} - export interface RemovedFieldMigrationResult { dryRun: boolean; modelName: string; @@ -68,7 +62,6 @@ export class RemovedFieldMigrationService { }; } - const metadataFileUpdate = dryRun ? null : await this.prepareMetadataFileUpdate(model, fieldsForRemoval); const dataSource = await this.resolveDataSource(model.dataSource); const entityMetadata = this.resolveEntityMetadata(dataSource, model); const queryRunner = dataSource.createQueryRunner(); @@ -103,24 +96,6 @@ export class RemovedFieldMigrationService { await queryRunner.release(); } - if (!dryRun && metadataFileUpdate) { - await fs.writeFile(metadataFileUpdate.filePath, metadataFileUpdate.updatedContent, "utf8"); - - try { - await this.fieldMetadataRepo.manager.transaction(async (manager) => { - await manager.remove(FieldMetadata, fieldsForRemoval); - }); - } catch (error) { - await fs.writeFile(metadataFileUpdate.filePath, metadataFileUpdate.originalContent, "utf8"); - throw error; - } - - operations.push(`Updated metadata file at ${metadataFileUpdate.filePath}`); - operations.push(`Removed ${fieldsForRemoval.length} field metadata record(s) from ss_field_metadata`); - } else if (dryRun) { - operations.push(`Would update module metadata for model "${model.singularName}" and remove ${fieldsForRemoval.length} field metadata record(s).`); - } - return { dryRun, modelName: model.singularName, @@ -278,33 +253,6 @@ export class RemovedFieldMigrationService { return [...columnCandidates].filter(Boolean); } - private async prepareMetadataFileUpdate(model: ModelMetadata, fieldsForRemoval: FieldMetadata[]): Promise { - const filePath = await this.resolveMetadataFilePath(model.module.name); - const originalContent = await fs.readFile(filePath, "utf8"); - const metadata = JSON.parse(originalContent); - const models = metadata?.moduleMetadata?.models; - - if (!Array.isArray(models)) { - throw new BadRequestException(`Invalid module metadata structure in ${filePath}`); - } - - const existingModelIndex = models.findIndex((existingModel: any) => existingModel.singularName === model.singularName); - if (existingModelIndex === -1) { - throw new NotFoundException(`Model "${model.singularName}" not found in metadata file ${filePath}`); - } - - const fieldNamesForRemoval = new Set(fieldsForRemoval.map((field) => field.name)); - const existingModel = models[existingModelIndex]; - existingModel.fields = (existingModel.fields || []).filter((field: any) => !fieldNamesForRemoval.has(field.name)); - models[existingModelIndex] = existingModel; - - return { - filePath, - originalContent, - updatedContent: JSON.stringify(metadata, null, 2), - }; - } - private async resolveDataSource(dataSourceName?: string): Promise { const normalizedDataSourceName = dataSourceName && dataSourceName !== "default" ? dataSourceName : undefined; const token = normalizedDataSourceName ? getDataSourceToken(normalizedDataSourceName) : getDataSourceToken(); @@ -383,4 +331,4 @@ export class RemovedFieldMigrationService { return queryRunner.getTable(tableName) ?? undefined; } -} +} \ No newline at end of file diff --git a/src/subscribers/audit.subscriber.ts b/src/subscribers/audit.subscriber.ts index ba70006b..3a7b131a 100644 --- a/src/subscribers/audit.subscriber.ts +++ b/src/subscribers/audit.subscriber.ts @@ -61,50 +61,11 @@ export class AuditSubscriber implements EntitySubscriberInterface { let after = event.entity ?? null; const updatedColumnNames = (event.updatedColumns ?? []).map(c => c.propertyName); - const auditRelationFields = (await this.modelMetadataHelperService.loadFieldHierarchy(lowerFirst(event.metadata.name))).filter(field => - field.enableAuditTracking && - field.type === 'relation' && - field.relationType !== 'one-to-many' - ); - if (entityId && auditRelationFields.length > 0) { - const relations: any = {}; - auditRelationFields.forEach(field => relations[field.name] = true); - const relationBefore = event.entity?.[AUDIT_BEFORE_SNAPSHOT] ?? null; - const relationAfter = await event.manager.getRepository(event.metadata.target as any).findOne({ - where: { id: entityId } as any, - relations: relations as any, - }); + const auditResult = await this.prepareManyToManyAuditUpdateSnapshot(event, entityId, updatedColumnNames); - if (relationBefore) { - before = relationBefore; - } - if (relationAfter) { - after = relationAfter; - } - - if (relationBefore && relationAfter) { - auditRelationFields.forEach(field => { - if (field.relationType === 'many-to-one') { - const oldId = relationBefore[field.name]?.id ?? null; - const newId = relationAfter[field.name]?.id ?? null; - if (oldId !== newId && !updatedColumnNames.includes(field.name)) { - updatedColumnNames.push(field.name); - } - } - else if (field.relationType === 'many-to-many') { - const oldIds = Array.isArray(relationBefore[field.name]) - ? relationBefore[field.name].map(item => item.id).sort() - : []; - - const newIds = Array.isArray(relationAfter[field.name]) - ? relationAfter[field.name].map(item => item.id).sort() - : []; - if ((oldIds.length !== newIds.length || JSON.stringify(oldIds) !== JSON.stringify(newIds)) && !updatedColumnNames.includes(field.name)) { - updatedColumnNames.push(field.name); - } - } - }); - } + if (auditResult) { + before = auditResult.before; + after = auditResult.after; } this.enqueue(event, { @@ -121,6 +82,56 @@ export class AuditSubscriber implements EntitySubscriberInterface { }); } + /** + * Resolves before/after snapshots for audit-enabled many-to-many relations + * and updates changed relation field names for audit tracking. + */ + private async prepareManyToManyAuditUpdateSnapshot(event: UpdateEvent, entityId: number | null, updatedColumnNames: string[]): Promise<{ before: any; after: any } | null> { + + const auditRelationFields = (await this.modelMetadataHelperService.loadFieldHierarchy(lowerFirst(event.metadata.name))).filter(field => + field.enableAuditTracking && + field.type === 'relation' && + field.relationType !== 'one-to-many' + ); + + if (!entityId || auditRelationFields.length === 0) { + return null; + } + + const relations: Record = {}; + + auditRelationFields.forEach(field => { + relations[field.name] = true; + }); + + const relationBefore = event.entity?.[AUDIT_BEFORE_SNAPSHOT] ?? null; + + const relationAfter = await event.manager.getRepository(event.metadata.target as any).findOne({ + where: { id: entityId } as any, + relations: relations as any, + }); + + if (relationBefore && relationAfter) { + auditRelationFields.forEach(field => { + const oldIds = Array.isArray(relationBefore[field.name]) + ? relationBefore[field.name].map(item => item.id).sort() + : []; + + const newIds = Array.isArray(relationAfter[field.name]) + ? relationAfter[field.name].map(item => item.id).sort() + : []; + + if ((oldIds.length !== newIds.length || JSON.stringify(oldIds) !== JSON.stringify(newIds)) + && !updatedColumnNames.includes(field.name)) { + updatedColumnNames.push(field.name); + } + }); + } + return { + before: relationBefore ?? event.databaseEntity ?? null, + after: relationAfter ?? event.entity ?? null, + }; + } async afterRemove(event: RemoveEvent) { if (!this.shouldTrackAudit(event.metadata)) return; this.enqueue(event, {